From 4083de2a514b91ffd24791e12e63a04622109190 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 9 Dec 2025 10:26:55 +0100 Subject: [PATCH 001/341] Add public dashboard and route, add DoubleMetricPlot and GQL queries - add roofline legend display switch - small fixes --- api/schema.graphqls | 19 + internal/graph/generated/generated.go | 643 +++++++++++++++++ internal/graph/model/models_gen.go | 12 + internal/graph/schema.resolvers.go | 81 +++ internal/routerConfig/routes.go | 31 +- web/frontend/rollup.config.mjs | 1 + web/frontend/src/DashPublic.root.svelte | 671 ++++++++++++++++++ web/frontend/src/Header.svelte | 2 +- web/frontend/src/Status.root.svelte | 68 +- web/frontend/src/dashpublic.entrypoint.js | 13 + .../src/generic/plots/DoubleMetricPlot.svelte | 640 +++++++++++++++++ .../src/generic/plots/Roofline.svelte | 3 +- web/frontend/src/generic/plots/Stacked.svelte | 2 +- web/frontend/src/header/NavbarLinks.svelte | 28 + web/frontend/src/status.entrypoint.js | 1 + web/frontend/src/status/DashDetails.svelte | 82 +++ web/frontend/src/status/DashInternal.svelte | 605 ++++++++++++++++ .../{ => dashdetails}/StatisticsDash.svelte | 10 +- .../{ => dashdetails}/StatusDash.svelte | 14 +- .../status/{ => dashdetails}/UsageDash.svelte | 8 +- web/templates/base.tmpl | 65 +- web/templates/monitoring/dashboard.tmpl | 14 + web/templates/monitoring/status.tmpl | 1 + 23 files changed, 2918 insertions(+), 96 deletions(-) create mode 100644 web/frontend/src/DashPublic.root.svelte create mode 100644 web/frontend/src/dashpublic.entrypoint.js create mode 100644 web/frontend/src/generic/plots/DoubleMetricPlot.svelte create mode 100644 web/frontend/src/status/DashDetails.svelte create mode 100644 web/frontend/src/status/DashInternal.svelte rename web/frontend/src/status/{ => dashdetails}/StatisticsDash.svelte (92%) rename web/frontend/src/status/{ => dashdetails}/StatusDash.svelte (97%) rename web/frontend/src/status/{ => dashdetails}/UsageDash.svelte (98%) create mode 100644 web/templates/monitoring/dashboard.tmpl diff --git a/api/schema.graphqls b/api/schema.graphqls index 8f5e1c7c..1c81e6b6 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -164,6 +164,13 @@ type JobMetricWithName { metric: JobMetric! } +type ClusterMetricWithName { + name: String! + unit: Unit + timestep: Int! + data: [NullableFloat!]! +} + type JobMetric { unit: Unit timestep: Int! @@ -267,6 +274,11 @@ type NodeMetrics { metrics: [JobMetricWithName!]! } +type ClusterMetrics { + nodeCount: Int! + metrics: [ClusterMetricWithName!]! +} + type NodesResultList { items: [NodeMetrics!]! offset: Int @@ -385,6 +397,13 @@ type Query { page: PageRequest resolution: Int ): NodesResultList! + + clusterMetrics( + cluster: String! + metrics: [String!] + from: Time! + to: Time! + ): ClusterMetrics! } type Mutation { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index a3b1a1d3..b1489420 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -66,6 +66,18 @@ type ComplexityRoot struct { SubClusters func(childComplexity int) int } + ClusterMetricWithName struct { + Data func(childComplexity int) int + Name func(childComplexity int) int + Timestep func(childComplexity int) int + Unit func(childComplexity int) int + } + + ClusterMetrics struct { + Metrics func(childComplexity int) int + NodeCount func(childComplexity int) int + } + ClusterSupport struct { Cluster func(childComplexity int) int SubClusters func(childComplexity int) int @@ -319,6 +331,7 @@ type ComplexityRoot struct { Query struct { AllocatedNodes func(childComplexity int, cluster string) int + ClusterMetrics func(childComplexity int, cluster string, metrics []string, from time.Time, to time.Time) int Clusters func(childComplexity int) int GlobalMetrics func(childComplexity int) int Job func(childComplexity int, id string) int @@ -485,6 +498,7 @@ type QueryResolver interface { RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) + ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) } type SubClusterResolver interface { NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) @@ -551,6 +565,48 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return e.complexity.Cluster.SubClusters(childComplexity), true + case "ClusterMetricWithName.data": + if e.complexity.ClusterMetricWithName.Data == nil { + break + } + + return e.complexity.ClusterMetricWithName.Data(childComplexity), true + + case "ClusterMetricWithName.name": + if e.complexity.ClusterMetricWithName.Name == nil { + break + } + + return e.complexity.ClusterMetricWithName.Name(childComplexity), true + + case "ClusterMetricWithName.timestep": + if e.complexity.ClusterMetricWithName.Timestep == nil { + break + } + + return e.complexity.ClusterMetricWithName.Timestep(childComplexity), true + + case "ClusterMetricWithName.unit": + if e.complexity.ClusterMetricWithName.Unit == nil { + break + } + + return e.complexity.ClusterMetricWithName.Unit(childComplexity), true + + case "ClusterMetrics.metrics": + if e.complexity.ClusterMetrics.Metrics == nil { + break + } + + return e.complexity.ClusterMetrics.Metrics(childComplexity), true + + case "ClusterMetrics.nodeCount": + if e.complexity.ClusterMetrics.NodeCount == nil { + break + } + + return e.complexity.ClusterMetrics.NodeCount(childComplexity), true + case "ClusterSupport.cluster": if e.complexity.ClusterSupport.Cluster == nil { break @@ -1699,6 +1755,18 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return e.complexity.Query.AllocatedNodes(childComplexity, args["cluster"].(string)), true + case "Query.clusterMetrics": + if e.complexity.Query.ClusterMetrics == nil { + break + } + + args, err := ec.field_Query_clusterMetrics_args(ctx, rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.ClusterMetrics(childComplexity, args["cluster"].(string), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true + case "Query.clusters": if e.complexity.Query.Clusters == nil { break @@ -2577,6 +2645,13 @@ type JobMetricWithName { metric: JobMetric! } +type ClusterMetricWithName { + name: String! + unit: Unit + timestep: Int! + data: [NullableFloat!]! +} + type JobMetric { unit: Unit timestep: Int! @@ -2680,6 +2755,11 @@ type NodeMetrics { metrics: [JobMetricWithName!]! } +type ClusterMetrics { + nodeCount: Int! + metrics: [ClusterMetricWithName!]! +} + type NodesResultList { items: [NodeMetrics!]! offset: Int @@ -2798,6 +2878,13 @@ type Query { page: PageRequest resolution: Int ): NodesResultList! + + clusterMetrics( + cluster: String! + metrics: [String!] + from: Time! + to: Time! + ): ClusterMetrics! } type Mutation { @@ -3074,6 +3161,32 @@ func (ec *executionContext) field_Query_allocatedNodes_args(ctx context.Context, return args, nil } +func (ec *executionContext) field_Query_clusterMetrics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := graphql.ProcessArgField(ctx, rawArgs, "cluster", ec.unmarshalNString2string) + if err != nil { + return nil, err + } + args["cluster"] = arg0 + arg1, err := graphql.ProcessArgField(ctx, rawArgs, "metrics", ec.unmarshalOString2ᚕstringᚄ) + if err != nil { + return nil, err + } + args["metrics"] = arg1 + arg2, err := graphql.ProcessArgField(ctx, rawArgs, "from", ec.unmarshalNTime2timeᚐTime) + if err != nil { + return nil, err + } + args["from"] = arg2 + arg3, err := graphql.ProcessArgField(ctx, rawArgs, "to", ec.unmarshalNTime2timeᚐTime) + if err != nil { + return nil, err + } + args["to"] = arg3 + return args, nil +} + func (ec *executionContext) field_Query_jobMetrics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -3784,6 +3897,283 @@ func (ec *executionContext) fieldContext_Cluster_subClusters(_ context.Context, return fc, nil } +func (ec *executionContext) _ClusterMetricWithName_name(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_unit(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_unit(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Unit, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*schema.Unit) + fc.Result = res + return ec.marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_unit(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "base": + return ec.fieldContext_Unit_base(ctx, field) + case "prefix": + return ec.fieldContext_Unit_prefix(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type Unit", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_timestep(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_timestep(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Timestep, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_timestep(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_data(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_data(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Data, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_data(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type NullableFloat does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetrics_nodeCount(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetrics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetrics_nodeCount(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.NodeCount, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetrics_nodeCount(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetrics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetrics_metrics(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetrics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetrics_metrics(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Metrics, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.ClusterMetricWithName) + fc.Result = res + return ec.marshalNClusterMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithNameᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetrics_metrics(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetrics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "name": + return ec.fieldContext_ClusterMetricWithName_name(ctx, field) + case "unit": + return ec.fieldContext_ClusterMetricWithName_unit(ctx, field) + case "timestep": + return ec.fieldContext_ClusterMetricWithName_timestep(ctx, field) + case "data": + return ec.fieldContext_ClusterMetricWithName_data(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ClusterMetricWithName", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _ClusterSupport_cluster(ctx context.Context, field graphql.CollectedField, obj *schema.ClusterSupport) (ret graphql.Marshaler) { fc, err := ec.fieldContext_ClusterSupport_cluster(ctx, field) if err != nil { @@ -12353,6 +12743,67 @@ func (ec *executionContext) fieldContext_Query_nodeMetricsList(ctx context.Conte return fc, nil } +func (ec *executionContext) _Query_clusterMetrics(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_clusterMetrics(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().ClusterMetrics(rctx, fc.Args["cluster"].(string), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*model.ClusterMetrics) + fc.Result = res + return ec.marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_clusterMetrics(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "nodeCount": + return ec.fieldContext_ClusterMetrics_nodeCount(ctx, field) + case "metrics": + return ec.fieldContext_ClusterMetrics_metrics(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ClusterMetrics", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_clusterMetrics_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query___type(ctx, field) if err != nil { @@ -17527,6 +17978,101 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, return out } +var clusterMetricWithNameImplementors = []string{"ClusterMetricWithName"} + +func (ec *executionContext) _ClusterMetricWithName(ctx context.Context, sel ast.SelectionSet, obj *model.ClusterMetricWithName) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, clusterMetricWithNameImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ClusterMetricWithName") + case "name": + out.Values[i] = ec._ClusterMetricWithName_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "unit": + out.Values[i] = ec._ClusterMetricWithName_unit(ctx, field, obj) + case "timestep": + out.Values[i] = ec._ClusterMetricWithName_timestep(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "data": + out.Values[i] = ec._ClusterMetricWithName_data(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + +var clusterMetricsImplementors = []string{"ClusterMetrics"} + +func (ec *executionContext) _ClusterMetrics(ctx context.Context, sel ast.SelectionSet, obj *model.ClusterMetrics) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, clusterMetricsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ClusterMetrics") + case "nodeCount": + out.Values[i] = ec._ClusterMetrics_nodeCount(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "metrics": + out.Values[i] = ec._ClusterMetrics_metrics(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var clusterSupportImplementors = []string{"ClusterSupport"} func (ec *executionContext) _ClusterSupport(ctx context.Context, sel ast.SelectionSet, obj *schema.ClusterSupport) graphql.Marshaler { @@ -20101,6 +20647,28 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "clusterMetrics": + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_clusterMetrics(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "__type": out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { @@ -21205,6 +21773,74 @@ func (ec *executionContext) marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋc return ec._Cluster(ctx, sel, v) } +func (ec *executionContext) marshalNClusterMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.ClusterMetricWithName) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx context.Context, sel ast.SelectionSet, v *model.ClusterMetricWithName) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._ClusterMetricWithName(ctx, sel, v) +} + +func (ec *executionContext) marshalNClusterMetrics2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx context.Context, sel ast.SelectionSet, v model.ClusterMetrics) graphql.Marshaler { + return ec._ClusterMetrics(ctx, sel, &v) +} + +func (ec *executionContext) marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx context.Context, sel ast.SelectionSet, v *model.ClusterMetrics) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._ClusterMetrics(ctx, sel, v) +} + func (ec *executionContext) marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupport(ctx context.Context, sel ast.SelectionSet, v schema.ClusterSupport) graphql.Marshaler { return ec._ClusterSupport(ctx, sel, &v) } @@ -24142,6 +24778,13 @@ func (ec *executionContext) marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑli return ec._Unit(ctx, sel, &v) } +func (ec *executionContext) marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v *schema.Unit) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return ec._Unit(ctx, sel, v) +} + func (ec *executionContext) marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐUser(ctx context.Context, sel ast.SelectionSet, v *model.User) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 4cb414eb..63b2da5d 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -13,6 +13,18 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) +type ClusterMetricWithName struct { + Name string `json:"name"` + Unit *schema.Unit `json:"unit,omitempty"` + Timestep int `json:"timestep"` + Data []schema.Float `json:"data"` +} + +type ClusterMetrics struct { + NodeCount int `json:"nodeCount"` + Metrics []*ClusterMetricWithName `json:"metrics"` +} + type Count struct { Name string `json:"name"` Count int `json:"count"` diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index e4901c46..624ddc64 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "math" "regexp" "slices" "strconv" @@ -973,6 +974,86 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub return nodeMetricsListResult, nil } +// ClusterMetrics is the resolver for the clusterMetrics field. +func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) { + + user := repository.GetUserFromContext(ctx) + if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { + return nil, errors.New("you need to be administrator or support staff for this query") + } + + if metrics == nil { + for _, mc := range archive.GetCluster(cluster).MetricConfig { + metrics = append(metrics, mc.Name) + } + } + + // 'nodes' == nil -> Defaults to all nodes of cluster for existing query workflow + scopes := []schema.MetricScope{"node"} + data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx) + if err != nil { + cclog.Warn("error while loading node data") + return nil, err + } + + clusterMetricData := make([]*model.ClusterMetricWithName, 0) + clusterMetrics := model.ClusterMetrics{NodeCount: 0, Metrics: clusterMetricData} + + collectorTimestep := make(map[string]int) + collectorUnit := make(map[string]schema.Unit) + collectorData := make(map[string][]schema.Float) + + for _, metrics := range data { + clusterMetrics.NodeCount += 1 + for metric, scopedMetrics := range metrics { + _, ok := collectorData[metric] + if !ok { + collectorData[metric] = make([]schema.Float, 0) + for _, scopedMetric := range scopedMetrics { + // Collect Info + collectorTimestep[metric] = scopedMetric.Timestep + collectorUnit[metric] = scopedMetric.Unit + // Collect Initial Data + for _, ser := range scopedMetric.Series { + for _, val := range ser.Data { + collectorData[metric] = append(collectorData[metric], val) + } + } + } + } else { + // Sum up values by index + for _, scopedMetric := range scopedMetrics { + // For This Purpose (Cluster_Wide-Sum of Node Metrics) OK + for _, ser := range scopedMetric.Series { + for i, val := range ser.Data { + collectorData[metric][i] += val + } + } + } + } + } + } + + for metricName, data := range collectorData { + cu := collectorUnit[metricName] + roundedData := make([]schema.Float, 0) + for _, val := range data { + roundedData = append(roundedData, schema.Float((math.Round(float64(val)*100.0) / 100.0))) + } + + cm := model.ClusterMetricWithName{ + Name: metricName, + Unit: &cu, + Timestep: collectorTimestep[metricName], + Data: roundedData, + } + + clusterMetrics.Metrics = append(clusterMetrics.Metrics, &cm) + } + + return &clusterMetrics, nil +} + // NumberOfNodes is the resolver for the numberOfNodes field. func (r *subClusterResolver) NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) { nodeList, err := archive.ParseNodeList(obj.Nodes) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 9c19de52..71edeefb 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -47,7 +47,9 @@ var routes []Route = []Route{ {"/monitoring/systems/list/{cluster}/{subcluster}", "monitoring/systems.tmpl", "Cluster Node List - ClusterCockpit", false, setupClusterListRoute}, {"/monitoring/node/{cluster}/{hostname}", "monitoring/node.tmpl", "Node - ClusterCockpit", false, setupNodeRoute}, {"/monitoring/analysis/{cluster}", "monitoring/analysis.tmpl", "Analysis - ClusterCockpit", true, setupAnalysisRoute}, - {"/monitoring/status/{cluster}", "monitoring/status.tmpl", "Status of - ClusterCockpit", false, setupClusterStatusRoute}, + {"/monitoring/status/{cluster}", "monitoring/status.tmpl", " Dashboard - ClusterCockpit", false, setupClusterStatusRoute}, + {"/monitoring/status/detail/{cluster}", "monitoring/status.tmpl", "Status of - ClusterCockpit", false, setupClusterDetailRoute}, + {"/monitoring/dashboard/{cluster}", "monitoring/dashboard.tmpl", " Dashboard - ClusterCockpit", false, setupDashboardRoute}, } func setupHomeRoute(i InfoType, r *http.Request) InfoType { @@ -117,6 +119,33 @@ func setupClusterStatusRoute(i InfoType, r *http.Request) InfoType { vars := mux.Vars(r) i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] + i["displayType"] = "DASHBOARD" + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupClusterDetailRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["id"] = vars["cluster"] + i["cluster"] = vars["cluster"] + i["displayType"] = "DETAILS" + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupDashboardRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["id"] = vars["cluster"] + i["cluster"] = vars["cluster"] + i["displayType"] = "PUBLIC" from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") if from != "" || to != "" { i["from"] = from diff --git a/web/frontend/rollup.config.mjs b/web/frontend/rollup.config.mjs index c92d8155..6b7cf884 100644 --- a/web/frontend/rollup.config.mjs +++ b/web/frontend/rollup.config.mjs @@ -74,5 +74,6 @@ export default [ entrypoint('node', 'src/node.entrypoint.js'), entrypoint('analysis', 'src/analysis.entrypoint.js'), entrypoint('status', 'src/status.entrypoint.js'), + entrypoint('dashpublic', 'src/dashpublic.entrypoint.js'), entrypoint('config', 'src/config.entrypoint.js') ]; diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte new file mode 100644 index 00000000..e344aa42 --- /dev/null +++ b/web/frontend/src/DashPublic.root.svelte @@ -0,0 +1,671 @@ + + + + + + +

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

+ + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + + + + + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + + {#if $statusQuery.error} + + Error Requesting StatusQuery: {$statusQuery.error.message} + + {/if} + {#if $statesTimed.error} + + Error Requesting StatesTimed: {$statesTimed.error.message} + + {/if} + {#if $topJobsQuery.error} + + Error Requesting TopJobsQuery: {$topJobsQuery.error.message} + + {/if} + {#if $nodeStatusQuery.error} + + Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + + {/if} + + + {:else} + + + + + Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" + {[...clusterInfo?.processorTypes].toString()} + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + {#if clusterInfo?.totalAccs !== 0} + + + + + + {/if} +
{clusterInfo?.runningJobs} Running Jobs{clusterInfo?.activeUsers} Active Users
+ Flop Rate (Any) + + Memory BW Rate +
+ {clusterInfo?.flopRate} + {clusterInfo?.flopRateUnit} + + {clusterInfo?.memBwRate} + {clusterInfo?.memBwRateUnit} +
Allocated Nodes
+ +
{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes} + Nodes
Allocated Cores
+ +
{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)} + Cores
Allocated Accelerators
+ +
{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs} + Accelerators
+ + + + + + +
+ {#key refinedStateData} +

+ Current Node States +

+ sd.count, + )} + entities={refinedStateData.map( + (sd) => sd.state, + )} + /> + {/key} +
+ + + {#key refinedStateData} + + + + + + + {#each refinedStateData as sd, i} + + + + + + {/each} +
Current StateNodes
{sd.state}{sd.count}
+ {/key} + + + + + + + + Infos + + + Contents + + + + +
+ {#key $statusQuery?.data?.nodeMetrics} + + {/key} +
+ + +
+ +
+ + + +
+ {#key $statesTimed?.data?.nodeStates} + + {/key} +
+ + + {/if} + + diff --git a/web/frontend/src/Header.svelte b/web/frontend/src/Header.svelte index 98a796a2..f7ceac2e 100644 --- a/web/frontend/src/Header.svelte +++ b/web/frontend/src/Header.svelte @@ -120,7 +120,7 @@ href: "/monitoring/status/", icon: "clipboard-data", perCluster: true, - listOptions: false, + listOptions: true, menu: "Info", }, ]; diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 3d9002a1..8175681e 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -6,77 +6,43 @@ --> - - - + - -{#if $initq.fetching} - +{#if displayType !== "DASHBOARD" && displayType !== "DETAILS"} + - - - -{:else if $initq.error} - - - {$initq.error.message} + Unknown displayList type! {:else} - - - - - - - - - - - - - - - - - - - - - + {#if displayStatusDetail} + + + {:else} + + + {/if} {/if} diff --git a/web/frontend/src/dashpublic.entrypoint.js b/web/frontend/src/dashpublic.entrypoint.js new file mode 100644 index 00000000..47287c7a --- /dev/null +++ b/web/frontend/src/dashpublic.entrypoint.js @@ -0,0 +1,13 @@ +import { mount } from 'svelte'; +// import {} from './header.entrypoint.js' +import DashPublic from './DashPublic.root.svelte' + +mount(DashPublic, { + target: document.getElementById('svelte-app'), + props: { + presetCluster: infos.cluster, + }, + context: new Map([ + ['cc-config', clusterCockpitConfig] + ]) +}) diff --git a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte new file mode 100644 index 00000000..94acf45a --- /dev/null +++ b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte @@ -0,0 +1,640 @@ + + + + + +{#if metricData[0]?.data && metricData[0]?.data?.length > 0} +
+{:else} + Cannot render plot: No series data returned for {cluster} +{/if} diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index 79ece220..6425275a 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -36,6 +36,7 @@ subCluster = null, allowSizeChange = false, useColors = true, + useLegend = true, width = 600, height = 380, } = $props(); @@ -534,7 +535,7 @@ width: width, height: height, legend: { - show: true, + show: useLegend, }, cursor: { dataIdx: (u, seriesIdx) => { diff --git a/web/frontend/src/generic/plots/Stacked.svelte b/web/frontend/src/generic/plots/Stacked.svelte index 4c532db1..2616f9ae 100644 --- a/web/frontend/src/generic/plots/Stacked.svelte +++ b/web/frontend/src/generic/plots/Stacked.svelte @@ -156,7 +156,7 @@ { scale: "y", grid: { show: true }, - labelFont: "sans-serif", + // labelFont: "sans-serif", label: ylabel + (yunit ? ` (${yunit})` : ''), // values: (u, vals) => vals.map((v) => formatNumber(v)), }, diff --git a/web/frontend/src/header/NavbarLinks.svelte b/web/frontend/src/header/NavbarLinks.svelte index 41454d12..04fec0ea 100644 --- a/web/frontend/src/header/NavbarLinks.svelte +++ b/web/frontend/src/header/NavbarLinks.svelte @@ -64,6 +64,34 @@ {/each} + {:else if item.title === 'Status'} + + + + {item.title} + + + {#each clusters as cluster} + + + {cluster.name} + + + + Status Dashboard + + + Status Details + + + + {/each} + + {:else} diff --git a/web/frontend/src/status.entrypoint.js b/web/frontend/src/status.entrypoint.js index c3407c13..e21b3612 100644 --- a/web/frontend/src/status.entrypoint.js +++ b/web/frontend/src/status.entrypoint.js @@ -6,6 +6,7 @@ mount(Status, { target: document.getElementById('svelte-app'), props: { presetCluster: infos.cluster, + displayType: displayType, }, context: new Map([ ['cc-config', clusterCockpitConfig] diff --git a/web/frontend/src/status/DashDetails.svelte b/web/frontend/src/status/DashDetails.svelte new file mode 100644 index 00000000..410d8df4 --- /dev/null +++ b/web/frontend/src/status/DashDetails.svelte @@ -0,0 +1,82 @@ + + + + + + + + +

Current Status of Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}"

+ + + + +{#if $initq.fetching} + + + + + +{:else if $initq.error} + + + {$initq.error.message} + + +{:else} + + + + + + + + + + + + + + + + + + + + + +{/if} diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte new file mode 100644 index 00000000..73fa639d --- /dev/null +++ b/web/frontend/src/status/DashInternal.svelte @@ -0,0 +1,605 @@ + + + + + + +

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

+ + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + + + + + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + + {#if $statusQuery.error} + + Error Requesting StatusQuery: {$statusQuery.error.message} + + {/if} + {#if $statesTimed.error} + + Error Requesting StatesTimed: {$statesTimed.error.message} + + {/if} + {#if $topJobsQuery.error} + + Error Requesting TopJobsQuery: {$topJobsQuery.error.message} + + {/if} + {#if $nodeStatusQuery.error} + + Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + + {/if} + + + {:else} + + + + + Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" + {[...clusterInfo?.processorTypes].toString()} + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + {#if clusterInfo?.totalAccs !== 0} + + + + + + {/if} +
{clusterInfo?.runningJobs} Running Jobs{clusterInfo?.activeUsers} Active Users
+ Flop Rate (Any) + + Memory BW Rate +
+ {clusterInfo?.flopRate} + {clusterInfo?.flopRateUnit} + + {clusterInfo?.memBwRate} + {clusterInfo?.memBwRateUnit} +
Allocated Nodes
+ +
{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes} + Nodes
Allocated Cores
+ +
{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)} + Cores
Allocated Accelerators
+ +
{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs} + Accelerators
+ + + + + + +
+

+ Top Projects: Jobs +

+ tp['totalJobs'], + )} + entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} + /> +
+ + + + + + + + + {#each $topJobsQuery.data.jobsStatistics as tp, i} + + + + + + {/each} +
ProjectJobs
+ {scrambleNames ? scramble(tp.id) : tp.id} + + {tp['totalJobs']}
+ + + + +
+ {#key $statusQuery?.data?.jobsMetricStats} + + {/key} +
+ + + {#if clusterInfo?.totalAccs == 0} + + {:else} + + {/if} + + +
+ {#key $statesTimed?.data?.nodeStates} + + {/key} +
+ + +
+ {#key $statesTimed?.data?.healthStates} + + {/key} +
+ + + {/if} + + diff --git a/web/frontend/src/status/StatisticsDash.svelte b/web/frontend/src/status/dashdetails/StatisticsDash.svelte similarity index 92% rename from web/frontend/src/status/StatisticsDash.svelte rename to web/frontend/src/status/dashdetails/StatisticsDash.svelte index efd7a4cb..0a1cd46f 100644 --- a/web/frontend/src/status/StatisticsDash.svelte +++ b/web/frontend/src/status/dashdetails/StatisticsDash.svelte @@ -22,11 +22,11 @@ } from "@urql/svelte"; import { convert2uplot, - } from "../generic/utils.js"; - import PlotGrid from "../generic/PlotGrid.svelte"; - import Histogram from "../generic/plots/Histogram.svelte"; - import HistogramSelection from "../generic/select/HistogramSelection.svelte"; - import Refresher from "../generic/helper/Refresher.svelte"; + } from "../../generic/utils.js"; + import PlotGrid from "../../generic/PlotGrid.svelte"; + import Histogram from "../../generic/plots/Histogram.svelte"; + import HistogramSelection from "../../generic/select/HistogramSelection.svelte"; + import Refresher from "../../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { diff --git a/web/frontend/src/status/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte similarity index 97% rename from web/frontend/src/status/StatusDash.svelte rename to web/frontend/src/status/dashdetails/StatusDash.svelte index 03a8cc4a..ee604836 100644 --- a/web/frontend/src/status/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -22,12 +22,12 @@ gql, getContextClient, } from "@urql/svelte"; - import { formatDurationTime } from "../generic/units.js"; - import Refresher from "../generic/helper/Refresher.svelte"; - import TimeSelection from "../generic/select/TimeSelection.svelte"; - import Roofline from "../generic/plots/Roofline.svelte"; - import Pie, { colors } from "../generic/plots/Pie.svelte"; - import Stacked from "../generic/plots/Stacked.svelte"; + import { formatDurationTime } from "../../generic/units.js"; + import Refresher from "../../generic/helper/Refresher.svelte"; + import TimeSelection from "../../generic/select/TimeSelection.svelte"; + import Roofline from "../../generic/plots/Roofline.svelte"; + import Pie, { colors } from "../../generic/plots/Pie.svelte"; + import Stacked from "../../generic/plots/Stacked.svelte"; /* Svelte 5 Props */ let { @@ -83,7 +83,7 @@ } `, variables: { - filter: { cluster: { eq: cluster }, timeStart: stackedFrom}, + filter: { cluster: { eq: cluster }, timeStart: 1760096999}, typeNode: "node", typeHealth: "health" }, diff --git a/web/frontend/src/status/UsageDash.svelte b/web/frontend/src/status/dashdetails/UsageDash.svelte similarity index 98% rename from web/frontend/src/status/UsageDash.svelte rename to web/frontend/src/status/dashdetails/UsageDash.svelte index 74dd7a99..2071465a 100644 --- a/web/frontend/src/status/UsageDash.svelte +++ b/web/frontend/src/status/dashdetails/UsageDash.svelte @@ -27,10 +27,10 @@ scramble, scrambleNames, convert2uplot, - } from "../generic/utils.js"; - import Pie, { colors } from "../generic/plots/Pie.svelte"; - import Histogram from "../generic/plots/Histogram.svelte"; - import Refresher from "../generic/helper/Refresher.svelte"; + } from "../../generic/utils.js"; + import Pie, { colors } from "../../generic/plots/Pie.svelte"; + import Histogram from "../../generic/plots/Histogram.svelte"; + import Refresher from "../../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { diff --git a/web/templates/base.tmpl b/web/templates/base.tmpl index 7fd35a0a..28eab339 100644 --- a/web/templates/base.tmpl +++ b/web/templates/base.tmpl @@ -23,34 +23,49 @@ - {{block "navigation" .}} -
- {{end}} + {{if eq .Infos.displayType "PUBLIC"}} +
+
+ {{block "content-public" .}} + Whoops, you should not see this... [MAIN] + {{end}} +
+
-
-
- {{block "content" .}} - Whoops, you should not see this... - {{end}} -
-
+ {{block "javascript-public" .}} + Whoops, you should not see this... [JS] + {{end}} - {{block "footer" .}} -
- -
    -
  • Version {{ .Build.Version }}
    • -
  • Hash {{ .Build.Hash }}
    • -
  • Built {{ .Build.Buildtime }}
    • -
-
- {{end}} + {{else}} + {{block "navigation" .}} +
+ {{end}} - {{block "javascript" .}} - +
+
+ {{block "content" .}} + Whoops, you should not see this... [MAIN] + {{end}} +
+
+ + {{block "footer" .}} +
+ +
    +
  • Version {{ .Build.Version }}
    • +
  • Hash {{ .Build.Hash }}
    • +
  • Built {{ .Build.Buildtime }}
    • +
+
+ {{end}} + + {{block "javascript" .}} + + {{end}} {{end}} diff --git a/web/templates/monitoring/dashboard.tmpl b/web/templates/monitoring/dashboard.tmpl new file mode 100644 index 00000000..06666cde --- /dev/null +++ b/web/templates/monitoring/dashboard.tmpl @@ -0,0 +1,14 @@ +{{define "content-public"}} +
+{{end}} + +{{define "stylesheets"}} + +{{end}} +{{define "javascript-public"}} + + +{{end}} diff --git a/web/templates/monitoring/status.tmpl b/web/templates/monitoring/status.tmpl index 15aff693..b6a84140 100644 --- a/web/templates/monitoring/status.tmpl +++ b/web/templates/monitoring/status.tmpl @@ -8,6 +8,7 @@ {{define "javascript"}} From 6e385db378b314f0ba31e0b74d05a4baa30b7da1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 11 Dec 2025 18:51:19 +0100 Subject: [PATCH 002/341] color roofline plot, add option to match pie and table color for ndoestate --- web/frontend/src/DashPublic.root.svelte | 201 +++++++++--------- .../src/generic/plots/DoubleMetricPlot.svelte | 4 +- web/frontend/src/generic/plots/Pie.svelte | 13 +- .../src/generic/plots/Roofline.svelte | 189 ++++++++++++++-- .../src/generic/plots/RooflineLegacy.svelte | 8 +- web/frontend/src/generic/plots/Stacked.svelte | 18 +- 6 files changed, 304 insertions(+), 129 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index e344aa42..2e060126 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -82,7 +82,7 @@ } `, variables: { - filter: { cluster: { eq: presetCluster }, timeStart: 1760096999}, // DEBUG VALUE, use StackedFrom + filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, // DEBUG VALUE 1760096999, use StackedFrom typeNode: "node", typeHealth: "health" }, @@ -97,6 +97,7 @@ query ( $cluster: String! $metrics: [String!] + # $nmetrics: [String!] $from: Time! $to: Time! $clusterFrom: Time! @@ -183,7 +184,7 @@ totalCores totalAccs } - # TEST + # ClusterMetrics for doubleMetricPlot clusterMetrics( cluster: $cluster metrics: $metrics @@ -205,7 +206,8 @@ `, variables: { cluster: presetCluster, - metrics: ["flops_any", "mem_bw"], // Fixed names for roofline and status bars + metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline + // nmetrics: ["cpu_load", "acc_utilization"], // Metrics for Node Graph from: from.toISOString(), clusterFrom: clusterFrom.toISOString(), to: to.toISOString(), @@ -349,18 +351,18 @@ }); /* Functions */ - function legendColors(targetIdx, useAltColors) { - // Reuses first color if targetIdx overflows - let c; - if (useCbColors) { - c = [...colors['colorblind']]; - } else if (useAltColors) { - c = [...colors['alternative']]; - } else { - c = [...colors['default']]; - } - return c[(c.length + targetIdx) % c.length]; - } + // function legendColors(targetIdx, useAltColors) { + // // Reuses first color if targetIdx overflows + // let c; + // if (useCbColors) { + // c = [...colors['colorblind']]; + // } else if (useAltColors) { + // c = [...colors['alternative']]; + // } else { + // c = [...colors['default']]; + // } + // return c[(c.length + targetIdx) % c.length]; + // } function transformNodesStatsToData(subclusterData) { let data = null @@ -425,10 +427,10 @@ - + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} @@ -461,13 +463,24 @@ {:else} - - - - + + + + +

Cluster {presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}

+ + +

CPU(s)

{[...clusterInfo?.processorTypes].join(', ')}

+ + + + + + + @@ -545,77 +558,7 @@ - - - -
- {#key refinedStateData} -

- Current Node States -

- sd.count, - )} - entities={refinedStateData.map( - (sd) => sd.state, - )} - /> - {/key} -
- - - {#key refinedStateData} -
- - - - - - {#each refinedStateData as sd, i} - - - - - - {/each} -
Current StateNodes
{sd.state}{sd.count}
- {/key} - - - - - - - - Infos - - - Contents - - - - -
- {#key $statusQuery?.data?.nodeMetrics} - - {/key} -
- +
+ + +
+ {#key $statusQuery?.data?.nodeMetrics} + + {/key} +
+ + + + +
+ {#key refinedStateData} + + sd.count, + )} + entities={refinedStateData.map( + (sd) => sd.state, + )} + fixColors={refinedStateData.map( + (sd) => colors['nodeStates'][sd.state], + )} + /> + {/key} +
+ + + {#key refinedStateData} + + + + + + + {#each refinedStateData as sd, i} + + + + + + {/each} +
StateCount
{sd.state.charAt(0).toUpperCase() + sd.state.slice(1)}{sd.count}
+ {/key} + + + +
{#key $statesTimed?.data?.nodeStates} @@ -659,7 +670,7 @@ xlabel="Time" ylabel="Nodes" yunit = "#Count" - title = "Node States" + title = "Cluster Status" stateType = "Node" /> {/key} diff --git a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte index 94acf45a..9579f36b 100644 --- a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte +++ b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte @@ -33,6 +33,7 @@ // metric, width = 0, height = 300, + fixLinewidth = null, timestep, numNodes, metricData, @@ -52,7 +53,7 @@ // const subClusterTopology = getContext("getHardwareTopology")(cluster, subCluster); // const metricConfig = getContext("getMetricConfig")(cluster, subCluster, metric); const lineColors = clusterCockpitConfig.plotConfiguration_colorScheme; - const lineWidth = clusterCockpitConfig.plotConfiguration_lineWidth / window.devicePixelRatio; + const lineWidth = fixLinewidth ? fixLinewidth : clusterCockpitConfig.plotConfiguration_lineWidth / window.devicePixelRatio; // const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false; const renderSleepTime = 200; // const normalLineColor = "#000000"; @@ -444,6 +445,7 @@ const opts = { width, height, + title: 'Cluster Utilization', plugins: [legendAsTooltipPlugin()], series: plotSeries, axes: [ diff --git a/web/frontend/src/generic/plots/Pie.svelte b/web/frontend/src/generic/plots/Pie.svelte index 45be1f59..5e31b7dd 100644 --- a/web/frontend/src/generic/plots/Pie.svelte +++ b/web/frontend/src/generic/plots/Pie.svelte @@ -59,7 +59,15 @@ 'rgb(135,133,0)', 'rgb(0,167,108)', 'rgb(189,189,189)', - ] + ], + nodeStates: { + allocated: "rgba(0, 128, 0, 0.75)", + down: "rgba(255, 0, 0, 0.75)", + idle: "rgba(0, 0, 255, 0.75)", + reserved: "rgba(255, 0, 255, 0.75)", + mixed: "rgba(255, 215, 0, 0.75)", + unknown: "rgba(0, 0, 0, 0.75)" + } } @@ -77,6 +85,7 @@ entities, displayLegend = false, useAltColors = false, + fixColors = null } = $props(); /* Const Init */ @@ -98,6 +107,8 @@ c = [...colors['colorblind']]; } else if (useAltColors) { c = [...colors['alternative']]; + } else if (fixColors?.length > 0) { + c = [...fixColors]; } else { c = [...colors['default']]; } diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index 6425275a..c28ab1fa 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -34,15 +34,18 @@ nodesData = null, cluster = null, subCluster = null, + fixTitle = null, + yMinimum = null, allowSizeChange = false, useColors = true, useLegend = true, + colorBackground = false, width = 600, height = 380, } = $props(); /* Const Init */ - const lineWidth = clusterCockpitConfig.plotConfiguration_lineWidth; + const lineWidth = 2 // clusterCockpitConfig.plotConfiguration_lineWidth; const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false; /* Var Init */ @@ -294,7 +297,7 @@ } else { // No Colors: Use Black u.ctx.strokeStyle = "rgb(0, 0, 0)"; - u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)"; + u.ctx.fillStyle = colorBackground ? "rgb(0, 0, 0)" : "rgba(0, 0, 0, 0.5)"; } // Get Values @@ -527,6 +530,7 @@ let plotTitle = "CPU Roofline Diagram"; if (jobsData) plotTitle = "Job Average Roofline Diagram"; if (nodesData) plotTitle = "Node Average Roofline Diagram"; + if (fixTitle) plotTitle = fixTitle if (roofData) { const opts = { @@ -617,7 +621,7 @@ }, y: { range: [ - 0.01, + yMinimum ? yMinimum : 0.01, subCluster?.flopRateSimd?.value ? nearestThousand(subCluster.flopRateSimd.value) : 10000, @@ -669,6 +673,7 @@ u.ctx.lineWidth = lineWidth; u.ctx.beginPath(); + // Get Values const ycut = 0.01 * subCluster.memoryBandwidth.value; const scalarKnee = (subCluster.flopRateScalar.value - ycut) / @@ -676,19 +681,27 @@ const simdKnee = (subCluster.flopRateSimd.value - ycut) / subCluster.memoryBandwidth.value; - const scalarKneeX = u.valToPos(scalarKnee, "x", true), // Value, axis, toCanvasPixels - simdKneeX = u.valToPos(simdKnee, "x", true), - flopRateScalarY = u.valToPos( - subCluster.flopRateScalar.value, - "y", - true, - ), - flopRateSimdY = u.valToPos( - subCluster.flopRateSimd.value, - "y", - true, - ); + // Get Const Coords + const originX = u.valToPos(0.01, "x", true); + const originY = u.valToPos(yMinimum ? yMinimum : 0.01, "y", true); + + const outerX = u.valToPos(1000, "x", true); // rightmost x in plot coords + const outerY = u.valToPos( + subCluster?.flopRateSimd?.value + ? nearestThousand(subCluster.flopRateSimd.value) + : 10000, + "y", + true + ); + + const scalarKneeX = u.valToPos(scalarKnee, "x", true) // Value, axis, toCanvasPixels + const simdKneeX = u.valToPos(simdKnee, "x", true) + + const flopRateScalarY = u.valToPos(subCluster.flopRateScalar.value, "y", true) + const flopRateSimdY = u.valToPos(subCluster.flopRateSimd.value, "y", true); + + /* Render Lines */ if ( scalarKneeX < width * window.devicePixelRatio - @@ -728,10 +741,10 @@ y1, x2, y2, - u.valToPos(0.01, "x", true), - u.valToPos(1.0, "y", true), // X-Axis Start Coords - u.valToPos(1000, "x", true), - u.valToPos(1.0, "y", true), // X-Axis End Coords + originX, // x3; X-Axis Start Coord-X + originY, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + originY, // y4; X-Axis End Coord-Y ); if (xAxisIntersect.x > x1) { @@ -746,6 +759,144 @@ u.ctx.stroke(); // Reset grid lineWidth u.ctx.lineWidth = 0.15; + + /* Render Area */ + if (colorBackground) { + + u.ctx.beginPath(); + // Additional Coords for Colored Regions + const yhalf = u.valToPos(ycut/2, "y", true) + const simdShift = u.valToPos(simdKnee*1.75, "x", true) + + let upperBorderIntersect = lineIntersect( + x1, + y1, + x2, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateSimdY*1.667, // y4; X-Axis End Coord-Y + ); + + let lowerBorderIntersect = lineIntersect( + x1, + y1, + x2, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y + ); + + let helperUpperBorderIntersect = lineIntersect( + x1, + yhalf, + simdShift, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateSimdY*1.667, // y4; X-Axis End Coord-Y + ); + + let helperLowerBorderIntersect = lineIntersect( + x1, + yhalf, + simdShift, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y + ); + + let helperLowerBorderIntersectTop = lineIntersect( + x1, + yhalf, + simdShift, + y2, + scalarKneeX, // x3; X-Axis Start Coord-X + flopRateScalarY, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY, // y4; X-Axis End Coord-Y + ); + + // Diagonal Helper + u.ctx.moveTo(x1, yhalf); + u.ctx.lineTo(simdShift, y2); + // Upper Simd Helper + u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667); + u.ctx.lineTo(outerX, flopRateSimdY*1.667); + // Lower Scalar Helper + u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667); + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); + + u.ctx.stroke(); + + /* Color Regions */ + // MemoryBound + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.lineTo(x1, y1); // YCut + u.ctx.lineTo(x2, y2); // Upper Knee + u.ctx.lineTo(simdShift, y2); // Upper Helper Knee + u.ctx.lineTo(x1, yhalf); // Half yCut + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 200, 0, 0.4)"; // Yellow + u.ctx.fill(); + u.ctx.restore(); + + // Compute Lower + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Helper Knee + u.ctx.lineTo(scalarKneeX, flopRateScalarY); // Lower Knee + u.ctx.lineTo(outerX, flopRateScalarY); // Outer Border + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); // Outer Lower Helper Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue + u.ctx.fill(); + u.ctx.restore(); + + // Compute Upper + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee + u.ctx.lineTo(simdKneeX, flopRateSimdY); // Upper Knee + u.ctx.lineTo(outerX, flopRateSimdY); // Outer Border + u.ctx.lineTo(outerX, flopRateSimdY*1.667); // Outer Upper Helper Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue + u.ctx.fill(); + u.ctx.restore(); + + // Nomansland Lower + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(originX, originY); // Origin + u.ctx.lineTo(originX, yhalf); // YCut Half + u.ctx.lineTo(helperLowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Inner Helper Knee + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); // Lower Inner Border + u.ctx.lineTo(outerX, originY); // Lower Right Corner + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light + u.ctx.fill(); + u.ctx.restore(); + + // Nomansland Upper + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(helperLowerBorderIntersectTop.x, flopRateScalarY); // Lower Knee Top + u.ctx.lineTo(helperUpperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee + u.ctx.lineTo(outerX, flopRateSimdY*1.667); // Upper Inner Border + u.ctx.lineTo(outerX, flopRateScalarY); // Lower Knee Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light + u.ctx.fill(); + u.ctx.restore(); + } } /* Render Scales */ diff --git a/web/frontend/src/generic/plots/RooflineLegacy.svelte b/web/frontend/src/generic/plots/RooflineLegacy.svelte index 624253bf..6ee96ef1 100644 --- a/web/frontend/src/generic/plots/RooflineLegacy.svelte +++ b/web/frontend/src/generic/plots/RooflineLegacy.svelte @@ -315,10 +315,10 @@ y1, x2, y2, - u.valToPos(0.01, "x", true), - u.valToPos(1.0, "y", true), // X-Axis Start Coords - u.valToPos(1000, "x", true), - u.valToPos(1.0, "y", true), // X-Axis End Coords + u.valToPos(0.01, "x", true), // x3; X-Axis Start Coord-X + u.valToPos(1.0, "y", true), // y3; X-Axis Start Coord-Y + u.valToPos(1000, "x", true), // x4; X-Axis End Coord-X + u.valToPos(1.0, "y", true), // y4; X-Axis End Coord-Y ); if (xAxisIntersect.x > x1) { diff --git a/web/frontend/src/generic/plots/Stacked.svelte b/web/frontend/src/generic/plots/Stacked.svelte index 2616f9ae..c9ec1d72 100644 --- a/web/frontend/src/generic/plots/Stacked.svelte +++ b/web/frontend/src/generic/plots/Stacked.svelte @@ -39,63 +39,63 @@ label: "Full", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)", + fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)", stroke: cbmode ? "rgb(0, 110, 0)" : "green", }, partial: { label: "Partial", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)", + fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)", stroke: cbmode ? "rgb(235, 172, 35)" : "gold", }, failed: { label: "Failed", scale: "y", width: lineWidth, - fill: cbmode ? "rgb(181, 29, 20, 0.4)" : "rgba(255, 0, 0, 0.4)", + fill: cbmode ? "rgb(181, 29, 20, 0.6)" : "rgba(255, 0, 0, 0.6)", stroke: cbmode ? "rgb(181, 29, 20)" : "red", }, idle: { label: "Idle", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 140, 249, 0.4)" : "rgba(0, 0, 255, 0.4)", + fill: cbmode ? "rgba(0, 140, 249, 0.6)" : "rgba(0, 0, 255, 0.6)", stroke: cbmode ? "rgb(0, 140, 249)" : "blue", }, allocated: { label: "Allocated", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)", + fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)", stroke: cbmode ? "rgb(0, 110, 0)" : "green", }, reserved: { label: "Reserved", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(209, 99, 230, 0.4)" : "rgba(255, 0, 255, 0.4)", + fill: cbmode ? "rgba(209, 99, 230, 0.6)" : "rgba(255, 0, 255, 0.6)", stroke: cbmode ? "rgb(209, 99, 230)" : "magenta", }, mixed: { label: "Mixed", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)", + fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)", stroke: cbmode ? "rgb(235, 172, 35)" : "gold", }, down: { label: "Down", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(181, 29 ,20, 0.4)" : "rgba(255, 0, 0, 0.4)", + fill: cbmode ? "rgba(181, 29 ,20, 0.6)" : "rgba(255, 0, 0, 0.6)", stroke: cbmode ? "rgb(181, 29, 20)" : "red", }, unknown: { label: "Unknown", scale: "y", width: lineWidth, - fill: "rgba(0, 0, 0, 0.4)", + fill: "rgba(0, 0, 0, 0.6)", stroke: "black", } }; From 0d62181272e3d406b2543aab2d8fd765d2245594 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 12 Dec 2025 11:19:37 +0100 Subject: [PATCH 003/341] move roofline elements below series data render --- web/frontend/src/generic/plots/Roofline.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index c28ab1fa..3b53a662 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -651,7 +651,7 @@ hooks: { // setSeries: [ (u, seriesIdx) => console.log('setSeries', seriesIdx) ], // setLegend: [ u => console.log('setLegend', u.legend.idxs) ], - drawClear: [ + drawClear: [ // drawClear hook which fires before anything exists, so will render under the grid (u) => { qt = qt || new Quadtree(0, 0, u.bbox.width, u.bbox.height); qt.clear(); @@ -663,7 +663,7 @@ }); }, ], - draw: [ + drawAxes: [ // drawAxes hook, which fires after axes and grid have been rendered (u) => { // draw roofs when subCluster set if (subCluster != null) { From 79e1c236fe576b98ae8495ea897aa9a6af63f889 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 12 Dec 2025 17:51:54 +0100 Subject: [PATCH 004/341] cleanup, adapt internalDash, remove debug query value --- web/frontend/src/DashPublic.root.svelte | 120 ++---------------- .../src/generic/plots/Roofline.svelte | 7 - web/frontend/src/status/DashInternal.svelte | 79 ++++++------ .../src/status/dashdetails/StatusDash.svelte | 2 +- 4 files changed, 54 insertions(+), 154 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 2e060126..055e4899 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -16,19 +16,14 @@ } from "@urql/svelte"; import { init, - scramble, - scrambleNames, - convert2uplot } from "./generic/utils.js"; import { - formatDurationTime, formatNumber, } from "./generic/units.js"; import { Row, Col, Card, - CardTitle, CardHeader, CardBody, Spinner, @@ -39,9 +34,10 @@ import Roofline from "./generic/plots/Roofline.svelte"; import Pie, { colors } from "./generic/plots/Pie.svelte"; import Stacked from "./generic/plots/Stacked.svelte"; - // import Histogram from "./generic/plots/Histogram.svelte"; import DoubleMetric from "./generic/plots/DoubleMetricPlot.svelte"; + // Todo: Refresher-Tick + /* Svelte 5 Props */ let { presetCluster, @@ -53,7 +49,6 @@ const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false /* States */ - let pagingState = $state({page: 1, itemsPerPage: 10}) // Top 10 let from = $state(new Date(Date.now() - (5 * 60 * 1000))); let clusterFrom = $state(new Date(Date.now() - (8 * 60 * 60 * 1000))); let to = $state(new Date(Date.now())); @@ -68,13 +63,8 @@ const statesTimed = $derived(queryStore({ client: client, query: gql` - query ($filter: [NodeFilter!], $typeNode: String!, $typeHealth: String!) { - nodeStates: nodeStatesTimed(filter: $filter, type: $typeNode) { - state - counts - times - } - healthStates: nodeStatesTimed(filter: $filter, type: $typeHealth) { + query ($filter: [NodeFilter!], $type: String!) { + nodeStatesTimed(filter: $filter, type: $type) { state counts times @@ -82,9 +72,8 @@ } `, variables: { - filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, // DEBUG VALUE 1760096999, use StackedFrom - typeNode: "node", - typeHealth: "health" + filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, + type: "node", }, requestPolicy: "network-only" })); @@ -97,7 +86,6 @@ query ( $cluster: String! $metrics: [String!] - # $nmetrics: [String!] $from: Time! $to: Time! $clusterFrom: Time! @@ -207,7 +195,6 @@ variables: { cluster: presetCluster, metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline - // nmetrics: ["cpu_load", "acc_utilization"], // Metrics for Node Graph from: from.toISOString(), clusterFrom: clusterFrom.toISOString(), to: to.toISOString(), @@ -219,31 +206,6 @@ requestPolicy: "network-only" })); - const topJobsQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $paging: PageRequest! - ) { - jobsStatistics( - filter: $filter - page: $paging - sortBy: TOTALJOBS - groupBy: PROJECT - ) { - id - totalJobs - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster} }], - paging: pagingState // Top 10 - }, - requestPolicy: "network-only" - })); - // Note: nodeMetrics are requested on configured $timestep resolution const nodeStatusQuery = $derived(queryStore({ client: client, @@ -351,19 +313,6 @@ }); /* Functions */ - // function legendColors(targetIdx, useAltColors) { - // // Reuses first color if targetIdx overflows - // let c; - // if (useCbColors) { - // c = [...colors['colorblind']]; - // } else if (useAltColors) { - // c = [...colors['alternative']]; - // } else { - // c = [...colors['default']]; - // } - // return c[(c.length + targetIdx) % c.length]; - // } - function transformNodesStatsToData(subclusterData) { let data = null const x = [], y = [] @@ -415,30 +364,18 @@ return result } - /* Inspect */ - $inspect(clusterInfo).with((type, clusterInfo) => { - console.log(type, 'clusterInfo', clusterInfo) - }); - - $inspect($statusQuery?.data?.clusterMetrics).with((type, clusterMetrics) => { - console.log(type, 'clusterMetrics', clusterMetrics) - }); - - - {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + {#if $statusQuery.fetching || $statesTimed.fetching || $nodeStatusQuery.fetching} - {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + {:else if $statusQuery.error || $statesTimed.error || $nodeStatusQuery.error} {#if $statusQuery.error} @@ -450,11 +387,6 @@ Error Requesting StatesTimed: {$statesTimed.error.message} {/if} - {#if $topJobsQuery.error} - - Error Requesting TopJobsQuery: {$topJobsQuery.error.message} - - {/if} {#if $nodeStatusQuery.error} Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} @@ -477,10 +409,6 @@ - @@ -559,7 +487,7 @@ - +
- @@ -620,9 +525,6 @@
{#key refinedStateData} -
- {#key $statesTimed?.data?.nodeStates} + {#key $statesTimed?.data?.nodeStatesTimed} { - console.log(type, 'clusterInfo', clusterInfo) - }); - - - -

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

- - + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching}
@@ -409,7 +422,7 @@ Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" - {[...clusterInfo?.processorTypes].toString()} + {[...clusterInfo?.processorTypes].join(', ')}
@@ -488,6 +501,7 @@ + @@ -529,6 +543,7 @@ +
{#key $statusQuery?.data?.jobsMetricStats} @@ -544,31 +559,20 @@ {/key}
- - {#if clusterInfo?.totalAccs == 0} - +
+ - {:else} - - {/if} +
+
{#key $statesTimed?.data?.nodeStates} @@ -584,6 +588,7 @@ {/key}
+
{#key $statesTimed?.data?.healthStates} diff --git a/web/frontend/src/status/dashdetails/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte index ee604836..f1f5a1a1 100644 --- a/web/frontend/src/status/dashdetails/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -83,7 +83,7 @@ } `, variables: { - filter: { cluster: { eq: cluster }, timeStart: 1760096999}, + filter: { cluster: { eq: cluster }, timeStart: stackedFrom}, typeNode: "node", typeHealth: "health" }, From c5aff1a2ca39fc8363be8c9d56070278700ea45e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 15 Dec 2025 13:55:02 +0100 Subject: [PATCH 005/341] add autorefresh, remove leftover query --- web/frontend/src/DashPublic.root.svelte | 61 ++++++------------ .../src/generic/helper/Refresher.svelte | 43 +++++++------ web/frontend/src/status/DashInternal.svelte | 64 +++++++------------ .../src/status/dashdetails/StatusDash.svelte | 2 +- 4 files changed, 69 insertions(+), 101 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 055e4899..d3eb57bc 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -35,8 +35,7 @@ import Pie, { colors } from "./generic/plots/Pie.svelte"; import Stacked from "./generic/plots/Stacked.svelte"; import DoubleMetric from "./generic/plots/DoubleMetricPlot.svelte"; - - // Todo: Refresher-Tick + import Refresher from "./generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { @@ -206,35 +205,6 @@ requestPolicy: "network-only" })); - // Note: nodeMetrics are requested on configured $timestep resolution - const nodeStatusQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $selectedHistograms: [String!] - $numDurationBins: String - ) { - jobsStatistics(filter: $filter, metrics: $selectedHistograms, numDurationBins: $numDurationBins) { - histNumCores { - count - value - } - histNumAccs { - count - value - } - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }], - selectedHistograms: [], // No Metrics requested for node hardware stats - Empty Array can be used for refresh - numDurationBins: "1h", // Hardcode or selector? - }, - requestPolicy: "network-only" - })); - const clusterInfo = $derived.by(() => { if ($initq?.data?.clusters) { let rawInfos = {}; @@ -368,28 +338,39 @@ - {#if $statusQuery.fetching || $statesTimed.fetching || $nodeStatusQuery.fetching} + +
+ { + from = new Date(Date.now() - 5 * 60 * 1000); + to = new Date(Date.now()); + clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000)) + + if (interval) stackedFrom += Math.floor(interval / 1000); + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh + }} + /> + + + {#if $statusQuery.fetching || $statesTimed.fetching} - {:else if $statusQuery.error || $statesTimed.error || $nodeStatusQuery.error} + {:else if $statusQuery.error || $statesTimed.error} {#if $statusQuery.error} - Error Requesting StatusQuery: {$statusQuery.error.message} + Error Requesting Status Data: {$statusQuery.error.message} {/if} {#if $statesTimed.error} - Error Requesting StatesTimed: {$statesTimed.error.message} - - {/if} - {#if $nodeStatusQuery.error} - - Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + Error Requesting Node Scheduler States: {$statesTimed.error.message} {/if} diff --git a/web/frontend/src/generic/helper/Refresher.svelte b/web/frontend/src/generic/helper/Refresher.svelte index 7f568bfb..ca05bf6d 100644 --- a/web/frontend/src/generic/helper/Refresher.svelte +++ b/web/frontend/src/generic/helper/Refresher.svelte @@ -14,6 +14,7 @@ let { initially = null, presetClass = "", + hideSelector = false, onRefresh } = $props(); @@ -36,25 +37,27 @@ }); - - - - - - - - - - + + + + + + + + +{/if} diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte index 495f9983..c6abf068 100644 --- a/web/frontend/src/status/DashInternal.svelte +++ b/web/frontend/src/status/DashInternal.svelte @@ -39,6 +39,7 @@ import Pie, { colors } from "../generic/plots/Pie.svelte"; import Stacked from "../generic/plots/Stacked.svelte"; import DoubleMetric from "../generic/plots/DoubleMetricPlot.svelte"; + import Refresher from "../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { @@ -224,35 +225,6 @@ requestPolicy: "network-only" })); - // Note: nodeMetrics are requested on configured $timestep resolution - const nodeStatusQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $selectedHistograms: [String!] - $numDurationBins: String - ) { - jobsStatistics(filter: $filter, metrics: $selectedHistograms, numDurationBins: $numDurationBins) { - histNumCores { - count - value - } - histNumAccs { - count - value - } - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }], - selectedHistograms: [], // No Metrics requested for node hardware stats - Empty Array can be used for refresh - numDurationBins: "1h", // Hardcode or selector? - }, - requestPolicy: "network-only" - })); - const clusterInfo = $derived.by(() => { if ($initq?.data?.clusters) { let rawInfos = {}; @@ -385,33 +357,45 @@ - {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + { + from = new Date(Date.now() - 5 * 60 * 1000); + to = new Date(Date.now()); + clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000)) + pagingState = { page:1, itemsPerPage: 10 }; + + if (interval) stackedFrom += Math.floor(interval / 1000); + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh + }} + /> + + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching} - {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} - + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error} + {#if $statusQuery.error} - Error Requesting StatusQuery: {$statusQuery.error.message} + Error Requesting Status Data: {$statusQuery.error.message} {/if} {#if $statesTimed.error} - Error Requesting StatesTimed: {$statesTimed.error.message} + Error Requesting Node Scheduler States: {$statesTimed.error.message} {/if} {#if $topJobsQuery.error} - Error Requesting TopJobsQuery: {$topJobsQuery.error.message} - - {/if} - {#if $nodeStatusQuery.error} - - Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + Error Requesting Jobs By Project: {$topJobsQuery.error.message} {/if} diff --git a/web/frontend/src/status/dashdetails/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte index f1f5a1a1..f858969f 100644 --- a/web/frontend/src/status/dashdetails/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -402,7 +402,7 @@ to = new Date(Date.now()); if (interval) stackedFrom += Math.floor(interval / 1000); - else stackedFrom += 1 // Workaround: TineSelection not linked, just trigger new data on manual refresh + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh }} /> From d56b0e93db21d2cff011f2f5afcf43c1cbd3e840 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 15 Dec 2025 15:10:10 +0100 Subject: [PATCH 006/341] cleanup routes, cleanup root components --- internal/routerConfig/routes.go | 17 +------------ web/frontend/src/DashPublic.root.svelte | 8 +++--- web/frontend/src/Status.root.svelte | 31 +++++++++-------------- web/frontend/src/dashpublic.entrypoint.js | 2 +- web/templates/base.tmpl | 4 +-- web/templates/monitoring/dashboard.tmpl | 2 +- 6 files changed, 21 insertions(+), 43 deletions(-) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 71edeefb..c2126cd0 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -120,11 +120,6 @@ func setupClusterStatusRoute(i InfoType, r *http.Request) InfoType { i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] i["displayType"] = "DASHBOARD" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } return i } @@ -133,11 +128,6 @@ func setupClusterDetailRoute(i InfoType, r *http.Request) InfoType { i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] i["displayType"] = "DETAILS" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } return i } @@ -145,12 +135,7 @@ func setupDashboardRoute(i InfoType, r *http.Request) InfoType { vars := mux.Vars(r) i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] - i["displayType"] = "PUBLIC" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } + i["displayType"] = "PUBLIC" // Used in Main Template return i } diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index d3eb57bc..dac3f9aa 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -6,9 +6,9 @@ --> - - -{#if displayType !== "DASHBOARD" && displayType !== "DETAILS"} +{#if displayType === 'DETAILS'} + +{:else if displayType === 'DASHBOARD'} + +{:else} - Unknown displayList type! + + + Unknown DisplayType for Status View! + + -{:else} - {#if displayStatusDetail} - - - {:else} - - - {/if} {/if} diff --git a/web/frontend/src/dashpublic.entrypoint.js b/web/frontend/src/dashpublic.entrypoint.js index 47287c7a..b9e92ffb 100644 --- a/web/frontend/src/dashpublic.entrypoint.js +++ b/web/frontend/src/dashpublic.entrypoint.js @@ -5,7 +5,7 @@ import DashPublic from './DashPublic.root.svelte' mount(DashPublic, { target: document.getElementById('svelte-app'), props: { - presetCluster: infos.cluster, + presetCluster: presetCluster, }, context: new Map([ ['cc-config', clusterCockpitConfig] diff --git a/web/templates/base.tmpl b/web/templates/base.tmpl index 28eab339..a1bd4134 100644 --- a/web/templates/base.tmpl +++ b/web/templates/base.tmpl @@ -27,13 +27,13 @@
{{block "content-public" .}} - Whoops, you should not see this... [MAIN] + Whoops, you should not see this... [PUBLIC] {{end}}
{{block "javascript-public" .}} - Whoops, you should not see this... [JS] + Whoops, you should not see this... [JS PUBLIC] {{end}} {{else}} diff --git a/web/templates/monitoring/dashboard.tmpl b/web/templates/monitoring/dashboard.tmpl index 06666cde..1ef94558 100644 --- a/web/templates/monitoring/dashboard.tmpl +++ b/web/templates/monitoring/dashboard.tmpl @@ -7,7 +7,7 @@ {{end}} {{define "javascript-public"}} From 46351389b6c39186d9a71cd254ff55e1d2700c5a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 15 Dec 2025 21:25:00 +0100 Subject: [PATCH 007/341] Add ai agent guidelines --- AGENTS.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..847bc094 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# ClusterCockpit Backend - Agent Guidelines + +## Build/Test Commands + +- Build: `make` or `go build ./cmd/cc-backend` +- Run all tests: `make test` (runs: `go clean -testcache && go build ./... && go vet ./... && go test ./...`) +- Run single test: `go test -run TestName ./path/to/package` +- Run single test file: `go test ./path/to/package -run TestName` +- Frontend build: `cd web/frontend && npm install && npm run build` +- Generate GraphQL: `make graphql` (uses gqlgen) +- Generate Swagger: `make swagger` (uses swaggo/swag) + +## Code Style + +- **Formatting**: Use `gofumpt` for all Go files (strict requirement) +- **Copyright header**: All files must include copyright header (see existing files) +- **Package docs**: Document packages with comprehensive package-level comments explaining purpose, usage, configuration +- **Imports**: Standard library first, then external packages, then internal packages (grouped with blank lines) +- **Naming**: Use camelCase for private, PascalCase for exported; descriptive names (e.g., `JobRepository`, `handleError`) +- **Error handling**: Return errors, don't panic; use custom error types where appropriate; log with cclog package +- **Logging**: Use `cclog` package (e.g., `cclog.Errorf()`, `cclog.Warnf()`, `cclog.Debugf()`) +- **Testing**: Use standard `testing` package; use `testify/assert` for assertions; name tests `TestFunctionName` +- **Comments**: Document all exported functions/types with godoc-style comments +- **Structs**: Document fields with inline comments, especially for complex configurations +- **HTTP handlers**: Return proper status codes; use `handleError()` helper for consistent error responses +- **JSON**: Use struct tags for JSON marshaling; `DisallowUnknownFields()` for strict decoding From 33c38f94646e94abb364823bd2280892af8c164f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 15 Dec 2025 21:25:30 +0100 Subject: [PATCH 008/341] Fix start time in tasks --- internal/taskmanager/compressionService.go | 2 +- internal/taskmanager/retentionService.go | 4 ++-- internal/taskmanager/stopJobsExceedTime.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/taskmanager/compressionService.go b/internal/taskmanager/compressionService.go index c2df852d..1da2f68d 100644 --- a/internal/taskmanager/compressionService.go +++ b/internal/taskmanager/compressionService.go @@ -17,7 +17,7 @@ import ( func RegisterCompressionService(compressOlderThan int) { cclog.Info("Register compression service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o5, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(5, 0, 0))), gocron.NewTask( func() { var jobs []*schema.Job diff --git a/internal/taskmanager/retentionService.go b/internal/taskmanager/retentionService.go index e452ffd0..92cd36ab 100644 --- a/internal/taskmanager/retentionService.go +++ b/internal/taskmanager/retentionService.go @@ -16,7 +16,7 @@ import ( func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { cclog.Info("Register retention delete service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(14, 30, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) @@ -43,7 +43,7 @@ func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { func RegisterRetentionMoveService(age int, includeDB bool, location string, omitTagged bool) { cclog.Info("Register retention move service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(4, 0, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) diff --git a/internal/taskmanager/stopJobsExceedTime.go b/internal/taskmanager/stopJobsExceedTime.go index e59b3aee..b763f561 100644 --- a/internal/taskmanager/stopJobsExceedTime.go +++ b/internal/taskmanager/stopJobsExceedTime.go @@ -16,7 +16,7 @@ import ( func RegisterStopJobsExceedTime() { cclog.Info("Register undead jobs service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o3, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))), gocron.NewTask( func() { err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) From 6f49998ad38332e1d671cae96f4fbab61edeb0a9 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:49:17 +0100 Subject: [PATCH 009/341] Switch to new go tool pattern for external tool deps --- Makefile | 4 ++-- go.mod | 5 +++++ tools.go | 9 --------- 3 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 tools.go diff --git a/Makefile b/Makefile index 0e19095a..0378b700 100644 --- a/Makefile +++ b/Makefile @@ -50,12 +50,12 @@ frontend: swagger: $(info ===> GENERATE swagger) - @go run github.com/swaggo/swag/cmd/swag init --parseDependency -d ./internal/api -g rest.go -o ./api + @go tool github.com/swaggo/swag/cmd/swag init --parseDependency -d ./internal/api -g rest.go -o ./api @mv ./api/docs.go ./internal/api/docs.go graphql: $(info ===> GENERATE graphql) - @go run github.com/99designs/gqlgen + @go tool github.com/99designs/gqlgen clean: $(info ===> CLEAN) diff --git a/go.mod b/go.mod index 3b3583bd..75e62f1e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,11 @@ go 1.24.0 toolchain go1.24.1 +tool ( + github.com/99designs/gqlgen + github.com/swaggo/swag/cmd/swag +) + require ( github.com/99designs/gqlgen v0.17.84 github.com/ClusterCockpit/cc-lib v1.0.0 diff --git a/tools.go b/tools.go deleted file mode 100644 index 950056c4..00000000 --- a/tools.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build tools -// +build tools - -package tools - -import ( - _ "github.com/99designs/gqlgen" - _ "github.com/swaggo/swag/cmd/swag" -) From 0306723307a2ddb25ae27284e4143320de393858 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:55:31 +0100 Subject: [PATCH 010/341] Introduce transparent compression for importJob function in all archive backends --- pkg/archive/fsBackend.go | 53 +++++++++++++++++++++++++++++------- pkg/archive/s3Backend.go | 26 +++++++++++++++--- pkg/archive/sqliteBackend.go | 25 +++++++++++++++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 1e9d7db3..22c1772a 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -603,19 +603,52 @@ func (fsa *FsArchive) ImportJob( return err } - f, err = os.Create(path.Join(dir, "data.json")) - if err != nil { - cclog.Error("Error while creating filepath for data.json") + var dataBuf bytes.Buffer + if err := EncodeJobData(&dataBuf, jobData); err != nil { + cclog.Error("Error while encoding job metricdata") return err } - if err := EncodeJobData(f, jobData); err != nil { - cclog.Error("Error while encoding job metricdata to data.json file") - return err + + if dataBuf.Len() > 2000 { + f, err = os.Create(path.Join(dir, "data.json.gz")) + if err != nil { + cclog.Error("Error while creating filepath for data.json.gz") + return err + } + gzipWriter := gzip.NewWriter(f) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Error("Error while writing compressed job data") + gzipWriter.Close() + f.Close() + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Warn("Error while closing gzip writer") + f.Close() + return err + } + if err := f.Close(); err != nil { + cclog.Warn("Error while closing data.json.gz file") + return err + } + } else { + f, err = os.Create(path.Join(dir, "data.json")) + if err != nil { + cclog.Error("Error while creating filepath for data.json") + return err + } + if _, err := f.Write(dataBuf.Bytes()); err != nil { + cclog.Error("Error while writing job metricdata to data.json file") + f.Close() + return err + } + if err := f.Close(); err != nil { + cclog.Warn("Error while closing data.json file") + return err + } } - if err := f.Close(); err != nil { - cclog.Warn("Error while closing data.json file") - } - return err + + return nil } func (fsa *FsArchive) StoreClusterCfg(name string, config *schema.Cluster) error { diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index 5b3d9f02..eacdde76 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -467,7 +467,6 @@ func (s3a *S3Archive) StoreJobMeta(job *schema.Job) error { func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) error { ctx := context.Background() - // Upload meta.json metaKey := getS3Key(jobMeta, "meta.json") var metaBuf bytes.Buffer if err := EncodeJobMeta(&metaBuf, jobMeta); err != nil { @@ -485,18 +484,37 @@ func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) er return err } - // Upload data.json - dataKey := getS3Key(jobMeta, "data.json") var dataBuf bytes.Buffer if err := EncodeJobData(&dataBuf, jobData); err != nil { cclog.Error("S3Archive ImportJob() > encoding data error") return err } + var dataKey string + var dataBytes []byte + + if dataBuf.Len() > 2000 { + dataKey = getS3Key(jobMeta, "data.json.gz") + var compressedBuf bytes.Buffer + gzipWriter := gzip.NewWriter(&compressedBuf) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Errorf("S3Archive ImportJob() > gzip write error: %v", err) + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Errorf("S3Archive ImportJob() > gzip close error: %v", err) + return err + } + dataBytes = compressedBuf.Bytes() + } else { + dataKey = getS3Key(jobMeta, "data.json") + dataBytes = dataBuf.Bytes() + } + _, err = s3a.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s3a.bucket), Key: aws.String(dataKey), - Body: bytes.NewReader(dataBuf.Bytes()), + Body: bytes.NewReader(dataBytes), }) if err != nil { cclog.Errorf("S3Archive ImportJob() > PutObject data error: %v", err) diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 49aeb79d..589beea4 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -361,16 +361,37 @@ func (sa *SqliteArchive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) return err } + var dataBytes []byte + var compressed bool + + if dataBuf.Len() > 2000 { + var compressedBuf bytes.Buffer + gzipWriter := gzip.NewWriter(&compressedBuf) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Errorf("SqliteArchive ImportJob() > gzip write error: %v", err) + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Errorf("SqliteArchive ImportJob() > gzip close error: %v", err) + return err + } + dataBytes = compressedBuf.Bytes() + compressed = true + } else { + dataBytes = dataBuf.Bytes() + compressed = false + } + now := time.Now().Unix() _, err := sa.db.Exec(` INSERT INTO jobs (job_id, cluster, start_time, meta_json, data_json, data_compressed, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, 0, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_id, cluster, start_time) DO UPDATE SET meta_json = excluded.meta_json, data_json = excluded.data_json, data_compressed = excluded.data_compressed, updated_at = excluded.updated_at - `, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBuf.Bytes(), now, now) + `, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBytes, compressed, now, now) if err != nil { cclog.Errorf("SqliteArchive ImportJob() > insert error: %v", err) return err From e6286768a74fc323c60346e2a45ec271b9d44d20 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:56:48 +0100 Subject: [PATCH 011/341] Refactor variabel naming and update doc comments --- internal/api/job.go | 14 +++--- internal/graph/schema.resolvers.go | 20 ++++---- internal/graph/util.go | 10 ++-- internal/repository/job.go | 17 ++++--- internal/repository/jobCreate.go | 13 ++--- internal/repository/jobFind.go | 67 +++++++++++++------------- internal/repository/job_test.go | 2 +- internal/repository/node.go | 4 +- internal/repository/repository_test.go | 2 +- internal/repository/tags.go | 19 ++++---- internal/tagger/detectApp_test.go | 2 +- internal/tagger/tagger.go | 4 +- internal/tagger/tagger_test.go | 2 +- 13 files changed, 88 insertions(+), 88 deletions(-) diff --git a/internal/api/job.go b/internal/api/job.go index 7701374a..919772f4 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -253,7 +253,7 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request) return } - job, err = api.JobRepository.FindById(r.Context(), id) // Get Job from Repo by ID + job, err = api.JobRepository.FindByID(r.Context(), id) // Get Job from Repo by ID } else { handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -346,7 +346,7 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) { return } - job, err = api.JobRepository.FindById(r.Context(), id) + job, err = api.JobRepository.FindByID(r.Context(), id) } else { handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -445,7 +445,7 @@ func (api *RestAPI) editMeta(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -493,7 +493,7 @@ func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -557,7 +557,7 @@ func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -796,7 +796,7 @@ func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) { return } - err = api.JobRepository.DeleteJobById(id) + err = api.JobRepository.DeleteJobByID(id) } else { handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -852,7 +852,7 @@ func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) return } - err = api.JobRepository.DeleteJobById(*job.ID) + err = api.JobRepository.DeleteJobByID(*job.ID) if err != nil { handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 75556938..418d0ee3 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -376,7 +376,7 @@ func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, erro cclog.Warn("Error while parsing job id") return nil, err } - return repo.GetNodeById(numericId, false) + return repo.GetNodeByID(numericId, false) } // Nodes is the resolver for the nodes field. @@ -442,7 +442,7 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) return nil, err } - job, err := r.Repo.FindById(ctx, numericId) + job, err := r.Repo.FindByID(ctx, numericId) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -1003,10 +1003,12 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } // SubCluster returns generated.SubClusterResolver implementation. func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} } -type clusterResolver struct{ *Resolver } -type jobResolver struct{ *Resolver } -type metricValueResolver struct{ *Resolver } -type mutationResolver struct{ *Resolver } -type nodeResolver struct{ *Resolver } -type queryResolver struct{ *Resolver } -type subClusterResolver struct{ *Resolver } +type ( + clusterResolver struct{ *Resolver } + jobResolver struct{ *Resolver } + metricValueResolver struct{ *Resolver } + mutationResolver struct{ *Resolver } + nodeResolver struct{ *Resolver } + queryResolver struct{ *Resolver } + subClusterResolver struct{ *Resolver } +) diff --git a/internal/graph/util.go b/internal/graph/util.go index 38c4914f..220c3a84 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -2,12 +2,14 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package graph import ( "context" "fmt" "math" + "slices" "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" @@ -185,11 +187,5 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF func requireField(ctx context.Context, name string) bool { fields := graphql.CollectAllFields(ctx) - for _, f := range fields { - if f == name { - return true - } - } - - return false + return slices.Contains(fields, name) } diff --git a/internal/repository/job.go b/internal/repository/job.go index 2f003e3b..f23a14cf 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -376,7 +376,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged bool) (int, return cnt, err } -func (r *JobRepository) DeleteJobById(id int64) error { +func (r *JobRepository) DeleteJobByID(id int64) error { // Invalidate cache entries before deletion r.cache.Del(fmt.Sprintf("metadata:%d", id)) r.cache.Del(fmt.Sprintf("energyFootprint:%d", id)) @@ -577,10 +577,10 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { return nil } -func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) { +func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) { query := sq.Select("job.id").From("job"). Join("jobtag ON jobtag.job_id = job.id"). - Where(sq.Eq{"jobtag.tag_id": tagId}).Distinct() + Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct() rows, err := query.RunWith(r.stmtCache).Query() if err != nil { cclog.Error("Error while running query") @@ -589,15 +589,15 @@ func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) { jobIds := make([]int64, 0, 100) for rows.Next() { - var jobId int64 + var jobID int64 - if err := rows.Scan(&jobId); err != nil { + if err := rows.Scan(&jobID); err != nil { rows.Close() cclog.Warn("Error while scanning rows") return nil, err } - jobIds = append(jobIds, jobId) + jobIds = append(jobIds, jobID) } return jobIds, nil @@ -731,10 +731,11 @@ func (r *JobRepository) UpdateEnergy( metricEnergy := 0.0 if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh - if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh) + switch sc.MetricConfig[i].Energy { + case "energy": // this metric has energy as unit (Joules or Wh) cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp) // FIXME: Needs sum as stats type - } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) + case "power": // this metric has power as unit (Watt) // Energy: Power (in Watts) * Time (in Seconds) // Unit: (W * (s / 3600)) / 1000 = kWh // Round 2 Digits: round(Energy * 100) / 100 diff --git a/internal/repository/jobCreate.go b/internal/repository/jobCreate.go index 2fcc69e9..efd262b8 100644 --- a/internal/repository/jobCreate.go +++ b/internal/repository/jobCreate.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -109,27 +110,27 @@ func (r *JobRepository) Start(job *schema.Job) (id int64, err error) { // Stop updates the job with the database id jobId using the provided arguments. func (r *JobRepository) Stop( - jobId int64, + jobID int64, duration int32, state schema.JobState, monitoringStatus int32, ) (err error) { // Invalidate cache entries as job state is changing - r.cache.Del(fmt.Sprintf("metadata:%d", jobId)) - r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobId)) + r.cache.Del(fmt.Sprintf("metadata:%d", jobID)) + r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID)) stmt := sq.Update("job"). Set("job_state", state). Set("duration", duration). Set("monitoring_status", monitoringStatus). - Where("job.id = ?", jobId) + Where("job.id = ?", jobID) _, err = stmt.RunWith(r.stmtCache).Exec() return err } func (r *JobRepository) StopCached( - jobId int64, + jobID int64, duration int32, state schema.JobState, monitoringStatus int32, @@ -140,7 +141,7 @@ func (r *JobRepository) StopCached( Set("job_state", state). Set("duration", duration). Set("monitoring_status", monitoringStatus). - Where("job_cache.id = ?", jobId) + Where("job_cache.id = ?", jobID) _, err = stmt.RunWith(r.stmtCache).Exec() return err diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 11f66c40..c4051e7f 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -22,13 +23,13 @@ import ( // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows func (r *JobRepository) Find( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) (*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). - Where("job.job_id = ?", *jobId) + Where("job.job_id = ?", *jobID) if cluster != nil { q = q.Where("job.cluster = ?", *cluster) @@ -44,12 +45,12 @@ func (r *JobRepository) Find( } func (r *JobRepository) FindCached( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) (*schema.Job, error) { q := sq.Select(jobCacheColumns...).From("job_cache"). - Where("job_cache.job_id = ?", *jobId) + Where("job_cache.job_id = ?", *jobID) if cluster != nil { q = q.Where("job_cache.cluster = ?", *cluster) @@ -63,19 +64,19 @@ func (r *JobRepository) FindCached( return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// Find executes a SQL query to find a specific batch job. -// The job is queried using the batch job id, the cluster name, -// and the start time of the job in UNIX epoch time seconds. -// It returns a pointer to a schema.Job data structure and an error variable. -// To check if no job was found test err == sql.ErrNoRows +// FindAll executes a SQL query to find all batch jobs matching the given criteria. +// Jobs are queried using the batch job id, and optionally filtered by cluster name +// and start time (UNIX epoch time seconds). +// It returns a slice of pointers to schema.Job data structures and an error variable. +// An empty slice is returned if no matching jobs are found. func (r *JobRepository) FindAll( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) ([]*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). - Where("job.job_id = ?", *jobId) + Where("job.job_id = ?", *jobID) if cluster != nil { q = q.Where("job.cluster = ?", *cluster) @@ -139,13 +140,13 @@ func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) { return jl, nil } -// FindById executes a SQL query to find a specific batch job. +// FindByID executes a SQL query to find a specific batch job. // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) q, qerr := SecurityCheck(ctx, q) if qerr != nil { @@ -155,14 +156,14 @@ func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByIdWithUser executes a SQL query to find a specific batch job. +// FindByIDWithUser executes a SQL query to find a specific batch job. // The job is queried using the database id. The user is passed directly, // instead as part of the context. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) q, qerr := SecurityCheckWithUser(user, q) if qerr != nil { @@ -172,24 +173,24 @@ func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schem return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByIdDirect executes a SQL query to find a specific batch job. +// FindByIDDirect executes a SQL query to find a specific batch job. // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByIdDirect(jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByJobId executes a SQL query to find a specific batch job. +// FindByJobID executes a SQL query to find a specific batch job. // The job is queried using the slurm id and the clustername. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime int64, cluster string) (*schema.Job, error) { +func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) { q := sq.Select(jobColumns...). From("job"). - Where("job.job_id = ?", jobId). + Where("job.job_id = ?", jobID). Where("job.cluster = ?", cluster). Where("job.start_time = ?", startTime) @@ -205,10 +206,10 @@ func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime // The job is queried using the slurm id,a username and the cluster. // It returns a bool. // If job was found, user is owner: test err != sql.ErrNoRows -func (r *JobRepository) IsJobOwner(jobId int64, startTime int64, user string, cluster string) bool { +func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool { q := sq.Select("id"). From("job"). - Where("job.job_id = ?", jobId). + Where("job.job_id = ?", jobID). Where("job.hpc_user = ?", user). Where("job.cluster = ?", cluster). Where("job.start_time = ?", startTime) @@ -269,19 +270,19 @@ func (r *JobRepository) FindConcurrentJobs( queryString := fmt.Sprintf("cluster=%s", job.Cluster) for rows.Next() { - var id, jobId, startTime sql.NullInt64 + var id, jobID, startTime sql.NullInt64 - if err = rows.Scan(&id, &jobId, &startTime); err != nil { + if err = rows.Scan(&id, &jobID, &startTime); err != nil { cclog.Warn("Error while scanning rows") return nil, err } if id.Valid { - queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64)) + queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64)) items = append(items, &model.JobLink{ ID: fmt.Sprint(id.Int64), - JobID: int(jobId.Int64), + JobID: int(jobID.Int64), }) } } @@ -294,19 +295,19 @@ func (r *JobRepository) FindConcurrentJobs( defer rows.Close() for rows.Next() { - var id, jobId, startTime sql.NullInt64 + var id, jobID, startTime sql.NullInt64 - if err := rows.Scan(&id, &jobId, &startTime); err != nil { + if err := rows.Scan(&id, &jobID, &startTime); err != nil { cclog.Warn("Error while scanning rows") return nil, err } if id.Valid { - queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64)) + queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64)) items = append(items, &model.JobLink{ ID: fmt.Sprint(id.Int64), - JobID: int(jobId.Int64), + JobID: int(jobID.Int64), }) } } diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index 9415bf98..c89225b3 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -33,7 +33,7 @@ func TestFind(t *testing.T) { func TestFindById(t *testing.T) { r := setup(t) - job, err := r.FindById(getContext(t), 338) + job, err := r.FindByID(getContext(t), 338) if err != nil { t.Fatal(err) } diff --git a/internal/repository/node.go b/internal/repository/node.go index 1e302704..9a1f3530 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -106,7 +106,7 @@ func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) return node, nil } -func (r *NodeRepository) GetNodeById(id int64, withMeta bool) (*schema.Node, error) { +func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) { node := &schema.Node{} var timestamp int if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state", @@ -240,7 +240,6 @@ func (r *NodeRepository) QueryNodes( page *model.PageRequest, order *model.OrderByInput, // Currently unused! ) ([]*schema.Node, error) { - query, qerr := AccessCheck(ctx, sq.Select("hostname", "cluster", "subcluster", "node_state", "health_state", "MAX(time_stamp) as time"). From("node"). @@ -309,7 +308,6 @@ func (r *NodeRepository) CountNodes( ctx context.Context, filters []*model.NodeFilter, ) (int, error) { - query, qerr := AccessCheck(ctx, sq.Select("time_stamp", "count(*) as countRes"). From("node"). diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 5603c31c..1346e4da 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -55,7 +55,7 @@ func BenchmarkDB_FindJobById(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := db.FindById(getContext(b), jobId) + _, err := db.FindByID(getContext(b), jobId) noErr(b, err) } }) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 52bd9076..5ca13382 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -18,7 +19,7 @@ import ( // AddTag adds the tag with id `tagId` to the job with the database id `jobId`. // Requires user authentication for security checks. func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warnf("Error finding job %d for user %s: %v", job, user.Username, err) return nil, err @@ -32,7 +33,7 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche // AddTagDirect adds a tag without user security checks. // Use only for internal/admin operations. func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdDirect(job) + j, err := r.FindByIDDirect(job) if err != nil { cclog.Warnf("Error finding job %d: %v", job, err) return nil, err @@ -43,10 +44,10 @@ func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error }) } -// Removes a tag from a job by tag id. -// Used by GraphQL API +// RemoveTag removes the tag with the database id `tag` from the job with the database id `job`. +// Requires user authentication for security checks. Used by GraphQL API. func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -75,8 +76,8 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema. return tags, archive.UpdateTags(j, archiveTags) } -// Removes a tag from a job by tag info -// Used by REST API +// RemoveJobTagByRequest removes a tag from the job with the database id `job` by tag type, name, and scope. +// Requires user authentication for security checks. Used by REST API. func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) { // Get Tag ID to delete tagID, exists := r.TagId(tagType, tagName, tagScope) @@ -86,7 +87,7 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT } // Get Job - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -124,7 +125,7 @@ func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) { continue } - job, err := r.FindByIdDirect(j) + job, err := r.FindByIDDirect(j) if err != nil { cclog.Warnf("Error while getting job %d", j) continue diff --git a/internal/tagger/detectApp_test.go b/internal/tagger/detectApp_test.go index 295ee97c..f9fc91d0 100644 --- a/internal/tagger/detectApp_test.go +++ b/internal/tagger/detectApp_test.go @@ -43,7 +43,7 @@ func TestRegister(t *testing.T) { func TestMatch(t *testing.T) { r := setup(t) - job, err := r.FindByIdDirect(317) + job, err := r.FindByIDDirect(317) noErr(t, err) var tagger AppTagger diff --git a/internal/tagger/tagger.go b/internal/tagger/tagger.go index 028d9efe..2ba18a14 100644 --- a/internal/tagger/tagger.go +++ b/internal/tagger/tagger.go @@ -40,7 +40,7 @@ type JobTagger struct { // startTaggers are applied when a job starts (e.g., application detection) startTaggers []Tagger // stopTaggers are applied when a job completes (e.g., job classification) - stopTaggers []Tagger + stopTaggers []Tagger } func newTagger() { @@ -98,7 +98,7 @@ func RunTaggers() error { } for _, id := range jl { - job, err := r.FindByIdDirect(id) + job, err := r.FindByIDDirect(id) if err != nil { cclog.Errorf("Error while getting job %s", err) return err diff --git a/internal/tagger/tagger_test.go b/internal/tagger/tagger_test.go index c81fac4a..fb4bc54e 100644 --- a/internal/tagger/tagger_test.go +++ b/internal/tagger/tagger_test.go @@ -18,7 +18,7 @@ func TestInit(t *testing.T) { func TestJobStartCallback(t *testing.T) { Init() r := setup(t) - job, err := r.FindByIdDirect(525) + job, err := r.FindByIDDirect(525) noErr(t, err) jobs := make([]*schema.Job, 0, 1) From 7fce6fa401acb9cf443b561ca2e44bb436287d19 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:11:09 +0100 Subject: [PATCH 012/341] Parallelize the Iter function in all archive backends --- pkg/archive/fsBackend.go | 69 +++++++++++++++-------- pkg/archive/s3Backend.go | 56 ++++++++++++------- pkg/archive/sqliteBackend.go | 103 +++++++++++++++++++++-------------- 3 files changed, 144 insertions(+), 84 deletions(-) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 22c1772a..b8d2a94b 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "text/tabwriter" "time" @@ -490,7 +491,46 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { ch := make(chan JobContainer) + go func() { + defer close(ch) + + numWorkers := 4 + jobPaths := make(chan string, numWorkers*2) + var wg sync.WaitGroup + + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for jobPath := range jobPaths { + job, err := loadJobMeta(filepath.Join(jobPath, "meta.json")) + if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { + cclog.Errorf("in %s: %s", jobPath, err.Error()) + continue + } + + if loadMetricData { + isCompressed := true + filename := filepath.Join(jobPath, "data.json.gz") + + if !util.CheckFileExists(filename) { + filename = filepath.Join(jobPath, "data.json") + isCompressed = false + } + + data, err := loadJobData(filename, isCompressed) + if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { + cclog.Errorf("in %s: %s", jobPath, err.Error()) + } + ch <- JobContainer{Meta: job, Data: &data} + } else { + ch <- JobContainer{Meta: job, Data: nil} + } + } + }() + } + clustersDir, err := os.ReadDir(fsa.path) if err != nil { cclog.Fatalf("Reading clusters failed @ cluster dirs: %s", err.Error()) @@ -507,7 +547,6 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { for _, lvl1Dir := range lvl1Dirs { if !lvl1Dir.IsDir() { - // Could be the cluster.json file continue } @@ -525,35 +564,17 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { for _, startTimeDir := range startTimeDirs { if startTimeDir.IsDir() { - job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json")) - if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { - cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) - } - - if loadMetricData { - isCompressed := true - filename := filepath.Join(dirpath, startTimeDir.Name(), "data.json.gz") - - if !util.CheckFileExists(filename) { - filename = filepath.Join(dirpath, startTimeDir.Name(), "data.json") - isCompressed = false - } - - data, err := loadJobData(filename, isCompressed) - if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { - cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) - } - ch <- JobContainer{Meta: job, Data: &data} - } else { - ch <- JobContainer{Meta: job, Data: nil} - } + jobPaths <- filepath.Join(dirpath, startTimeDir.Name()) } } } } } - close(ch) + + close(jobPaths) + wg.Wait() }() + return ch } diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index eacdde76..c874a320 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -17,6 +17,7 @@ import ( "os" "strconv" "strings" + "sync" "text/tabwriter" "time" @@ -813,29 +814,18 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { ctx := context.Background() defer close(ch) - for _, cluster := range s3a.clusters { - prefix := cluster + "/" + numWorkers := 4 + metaKeys := make(chan string, numWorkers*2) + var wg sync.WaitGroup - paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s3a.bucket), - Prefix: aws.String(prefix), - }) - - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error()) - } - - for _, obj := range page.Contents { - if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") { - continue - } - - // Load job metadata + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for metaKey := range metaKeys { result, err := s3a.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s3a.bucket), - Key: obj.Key, + Key: aws.String(metaKey), }) if err != nil { cclog.Errorf("S3Archive Iter() > GetObject meta error: %v", err) @@ -867,8 +857,34 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { ch <- JobContainer{Meta: job, Data: nil} } } + }() + } + + for _, cluster := range s3a.clusters { + prefix := cluster + "/" + + paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s3a.bucket), + Prefix: aws.String(prefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error()) + } + + for _, obj := range page.Contents { + if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") { + continue + } + metaKeys <- *obj.Key + } } } + + close(metaKeys) + wg.Wait() }() return ch diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 589beea4..6fa188ba 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -16,6 +16,7 @@ import ( "os" "slices" "strconv" + "sync" "text/tabwriter" "time" @@ -547,62 +548,84 @@ func (sa *SqliteArchive) CompressLast(starttime int64) int64 { return last } +type sqliteJobRow struct { + metaBlob []byte + dataBlob []byte + compressed bool +} + func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { ch := make(chan JobContainer) go func() { defer close(ch) - rows, err := sa.db.Query("SELECT job_id, cluster, start_time, meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") + rows, err := sa.db.Query("SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") if err != nil { cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) } defer rows.Close() - for rows.Next() { - var jobID int64 - var cluster string - var startTime int64 - var metaBlob []byte - var dataBlob []byte - var compressed bool + numWorkers := 4 + jobRows := make(chan sqliteJobRow, numWorkers*2) + var wg sync.WaitGroup - if err := rows.Scan(&jobID, &cluster, &startTime, &metaBlob, &dataBlob, &compressed); err != nil { + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for row := range jobRows { + job, err := DecodeJobMeta(bytes.NewReader(row.metaBlob)) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err) + continue + } + + if loadMetricData && row.dataBlob != nil { + var reader io.Reader = bytes.NewReader(row.dataBlob) + if row.compressed { + gzipReader, err := gzip.NewReader(reader) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + continue + } + decompressed, err := io.ReadAll(gzipReader) + gzipReader.Close() + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decompress error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + continue + } + reader = bytes.NewReader(decompressed) + } + + key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime) + jobData, err := DecodeJobData(reader, key) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + } else { + ch <- JobContainer{Meta: job, Data: &jobData} + } + } else { + ch <- JobContainer{Meta: job, Data: nil} + } + } + }() + } + + for rows.Next() { + var row sqliteJobRow + if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) continue } - - job, err := DecodeJobMeta(bytes.NewReader(metaBlob)) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err) - continue - } - - if loadMetricData && dataBlob != nil { - var reader io.Reader = bytes.NewReader(dataBlob) - if compressed { - gzipReader, err := gzip.NewReader(reader) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err) - ch <- JobContainer{Meta: job, Data: nil} - continue - } - defer gzipReader.Close() - reader = gzipReader - } - - key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime) - jobData, err := DecodeJobData(reader, key) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err) - ch <- JobContainer{Meta: job, Data: nil} - } else { - ch <- JobContainer{Meta: job, Data: &jobData} - } - } else { - ch <- JobContainer{Meta: job, Data: nil} - } + jobRows <- row } + + close(jobRows) + wg.Wait() }() return ch From 72b2560ecfcd6eb1bafe46a01840547f0c508275 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:11:26 +0100 Subject: [PATCH 013/341] Add progress bar for import function --- tools/archive-manager/main.go | 158 +++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 31 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 30aa9088..ae0ae9e6 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "strings" "sync" "sync/atomic" "time" @@ -33,78 +34,172 @@ func parseDate(in string) int64 { return 0 } +// countJobs counts the total number of jobs in the source archive. +func countJobs(srcBackend archive.ArchiveBackend) int { + count := 0 + for range srcBackend.Iter(false) { + count++ + } + return count +} + +// formatDuration formats a duration as a human-readable string. +func formatDuration(d time.Duration) string { + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } else if d < time.Hour { + return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) + } + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) +} + +// progressMeter displays import progress to the terminal. +type progressMeter struct { + total int + processed int32 + imported int32 + skipped int32 + failed int32 + startTime time.Time + done chan struct{} +} + +func newProgressMeter(total int) *progressMeter { + return &progressMeter{ + total: total, + startTime: time.Now(), + done: make(chan struct{}), + } +} + +func (p *progressMeter) start() { + go func() { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + p.render() + case <-p.done: + p.render() + fmt.Println() + return + } + } + }() +} + +func (p *progressMeter) render() { + processed := atomic.LoadInt32(&p.processed) + imported := atomic.LoadInt32(&p.imported) + skipped := atomic.LoadInt32(&p.skipped) + failed := atomic.LoadInt32(&p.failed) + + elapsed := time.Since(p.startTime) + percent := float64(processed) / float64(p.total) * 100 + if p.total == 0 { + percent = 0 + } + + var eta string + var throughput float64 + if processed > 0 { + throughput = float64(processed) / elapsed.Seconds() + remaining := float64(p.total-int(processed)) / throughput + eta = formatDuration(time.Duration(remaining) * time.Second) + } else { + eta = "calculating..." + } + + barWidth := 30 + filled := int(float64(barWidth) * float64(processed) / float64(p.total)) + if p.total == 0 { + filled = 0 + } + + var bar strings.Builder + for i := range barWidth { + if i < filled { + bar.WriteString("█") + } else { + bar.WriteString("░") + } + } + + fmt.Printf("\r[%s] %5.1f%% | %d/%d | %.1f jobs/s | ETA: %s | ✓%d ○%d ✗%d ", + bar.String(), percent, processed, p.total, throughput, eta, imported, skipped, failed) +} + +func (p *progressMeter) stop() { + close(p.done) +} + // importArchive imports all jobs from a source archive backend to a destination archive backend. // It uses parallel processing with a worker pool to improve performance. // Returns the number of successfully imported jobs, failed jobs, and any error encountered. func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, error) { cclog.Info("Starting parallel archive import...") - // Use atomic counters for thread-safe updates - var imported int32 - var failed int32 - var skipped int32 + cclog.Info("Counting jobs in source archive (this may take a long time) ...") + totalJobs := countJobs(srcBackend) + cclog.Infof("Found %d jobs to process", totalJobs) + + progress := newProgressMeter(totalJobs) - // Number of parallel workers numWorkers := 4 cclog.Infof("Using %d parallel workers", numWorkers) - // Create channels for job distribution jobs := make(chan archive.JobContainer, numWorkers*2) - // WaitGroup to track worker completion var wg sync.WaitGroup - // Start worker goroutines + progress.start() + for i := range numWorkers { wg.Add(1) go func(workerID int) { defer wg.Done() for job := range jobs { - // Validate job metadata if job.Meta == nil { cclog.Warn("Skipping job with nil metadata") - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Validate job data if job.Data == nil { cclog.Warnf("Job %d from cluster %s has no metric data, skipping", job.Meta.JobID, job.Meta.Cluster) - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Check if job already exists in destination if dstBackend.Exists(job.Meta) { cclog.Debugf("Job %d (cluster: %s, start: %d) already exists in destination, skipping", job.Meta.JobID, job.Meta.Cluster, job.Meta.StartTime) - atomic.AddInt32(&skipped, 1) + atomic.AddInt32(&progress.skipped, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Import job to destination if err := dstBackend.ImportJob(job.Meta, job.Data); err != nil { cclog.Errorf("Failed to import job %d from cluster %s: %s", job.Meta.JobID, job.Meta.Cluster, err.Error()) - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Successfully imported - newCount := atomic.AddInt32(&imported, 1) - if newCount%100 == 0 { - cclog.Infof("Progress: %d jobs imported, %d skipped, %d failed", - newCount, atomic.LoadInt32(&skipped), atomic.LoadInt32(&failed)) - } + atomic.AddInt32(&progress.imported, 1) + atomic.AddInt32(&progress.processed, 1) } }(i) } - // Feed jobs to workers go func() { - // Import cluster configs first clusters := srcBackend.GetClusters() for _, clusterName := range clusters { clusterCfg, err := srcBackend.LoadClusterCfg(clusterName) @@ -126,15 +221,16 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err close(jobs) }() - // Wait for all workers to complete wg.Wait() + progress.stop() - finalImported := int(atomic.LoadInt32(&imported)) - finalFailed := int(atomic.LoadInt32(&failed)) - finalSkipped := int(atomic.LoadInt32(&skipped)) + finalImported := int(atomic.LoadInt32(&progress.imported)) + finalFailed := int(atomic.LoadInt32(&progress.failed)) + finalSkipped := int(atomic.LoadInt32(&progress.skipped)) - cclog.Infof("Import completed: %d jobs imported, %d skipped, %d failed", - finalImported, finalSkipped, finalFailed) + elapsed := time.Since(progress.startTime) + cclog.Infof("Import completed in %s: %d jobs imported, %d skipped, %d failed", + formatDuration(elapsed), finalImported, finalSkipped, finalFailed) if finalFailed > 0 { return finalImported, finalFailed, fmt.Errorf("%d jobs failed to import", finalFailed) @@ -150,7 +246,7 @@ func main() { flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path. Default is ./var/job-archive") flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "info", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagRemoveCluster, "remove-cluster", "", "Remove cluster from archive and database") flag.StringVar(&flagRemoveBefore, "remove-before", "", "Remove all jobs with start time before date (Format: 2006-Jan-04)") From 14f1192ccbb04fe66019366726d05c0925306d3c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:35:33 +0100 Subject: [PATCH 014/341] Introduce central nats client --- cmd/cc-backend/main.go | 8 ++ cmd/cc-backend/server.go | 6 + pkg/nats/client.go | 246 +++++++++++++++++++++++++++++++++++++++ pkg/nats/config.go | 63 ++++++++++ 4 files changed, 323 insertions(+) create mode 100644 pkg/nats/client.go create mode 100644 pkg/nats/config.go diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index d89109e3..c3e33872 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -30,6 +30,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/tagger" "github.com/ClusterCockpit/cc-backend/internal/taskmanager" "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" @@ -267,6 +268,13 @@ func generateJWT(authHandle *auth.Authentication, username string) error { } func initSubsystems() error { + // Initialize nats client + natsConfig := ccconf.GetPackageConfig("nats") + if err := nats.Init(natsConfig); err != nil { + return fmt.Errorf("initializing nats client: %w", err) + } + nats.Connect() + // Initialize job archive archiveCfg := ccconf.GetPackageConfig("archive") if archiveCfg == nil { diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 975d38a1..2c5ce8bc 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -31,6 +31,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/generated" "github.com/ClusterCockpit/cc-backend/internal/memorystore" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" + "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/runtimeEnv" @@ -363,6 +364,11 @@ func (s *Server) Shutdown(ctx context.Context) { shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + nc := nats.GetClient() + if nc != nil { + nc.Close() + } + // First shut down the server gracefully (waiting for all ongoing requests) if err := s.server.Shutdown(shutdownCtx); err != nil { cclog.Errorf("Server shutdown error: %v", err) diff --git a/pkg/nats/client.go b/pkg/nats/client.go new file mode 100644 index 00000000..e61d060b --- /dev/null +++ b/pkg/nats/client.go @@ -0,0 +1,246 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package nats provides a generic NATS messaging client for publish/subscribe communication. +// +// The package wraps the nats.go library with connection management, automatic reconnection +// handling, and subscription tracking. It supports multiple authentication methods including +// username/password and credential files. +// +// # Configuration +// +// Configure the client via JSON in the application config: +// +// { +// "nats": { +// "address": "nats://localhost:4222", +// "username": "user", +// "password": "secret" +// } +// } +// +// Or using a credentials file: +// +// { +// "nats": { +// "address": "nats://localhost:4222", +// "creds-file-path": "/path/to/creds.json" +// } +// } +// +// # Usage +// +// The package provides a singleton client initialized once and retrieved globally: +// +// nats.Init(rawConfig) +// nats.Connect() +// +// client := nats.GetClient() +// client.Subscribe("events", func(subject string, data []byte) { +// fmt.Printf("Received: %s\n", data) +// }) +// +// client.Publish("events", []byte("hello")) +// +// # Thread Safety +// +// All Client methods are safe for concurrent use. +package nats + +import ( + "context" + "fmt" + "sync" + + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/nats-io/nats.go" +) + +var ( + clientOnce sync.Once + clientInstance *Client +) + +// Client wraps a NATS connection with subscription management. +type Client struct { + conn *nats.Conn + subscriptions []*nats.Subscription + mu sync.Mutex +} + +// MessageHandler is a callback function for processing received messages. +type MessageHandler func(subject string, data []byte) + +// Connect initializes the singleton NATS client using the global Keys config. +func Connect() { + clientOnce.Do(func() { + if Keys.Address == "" { + cclog.Warn("NATS: no address configured, skipping connection") + return + } + + client, err := NewClient(nil) + if err != nil { + cclog.Errorf("NATS connection failed: %v", err) + return + } + + clientInstance = client + }) +} + +// GetClient returns the singleton NATS client instance. +func GetClient() *Client { + if clientInstance == nil { + cclog.Warn("NATS client not initialized") + } + return clientInstance +} + +// NewClient creates a new NATS client. If cfg is nil, uses the global Keys config. +func NewClient(cfg *NatsConfig) (*Client, error) { + if cfg == nil { + cfg = &Keys + } + + if cfg.Address == "" { + return nil, fmt.Errorf("NATS address is required") + } + + var opts []nats.Option + + if cfg.Username != "" && cfg.Password != "" { + opts = append(opts, nats.UserInfo(cfg.Username, cfg.Password)) + } + + if cfg.CredsFilePath != "" { + opts = append(opts, nats.UserCredentials(cfg.CredsFilePath)) + } + + opts = append(opts, nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + if err != nil { + cclog.Warnf("NATS disconnected: %v", err) + } + })) + + opts = append(opts, nats.ReconnectHandler(func(nc *nats.Conn) { + cclog.Infof("NATS reconnected to %s", nc.ConnectedUrl()) + })) + + opts = append(opts, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + cclog.Errorf("NATS error: %v", err) + })) + + nc, err := nats.Connect(cfg.Address, opts...) + if err != nil { + return nil, fmt.Errorf("NATS connect failed: %w", err) + } + + cclog.Infof("NATS connected to %s", cfg.Address) + + return &Client{ + conn: nc, + subscriptions: make([]*nats.Subscription, 0), + }, nil +} + +// Subscribe registers a handler for messages on the given subject. +func (c *Client) Subscribe(subject string, handler MessageHandler) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.Subscribe(subject, func(msg *nats.Msg) { + handler(msg.Subject, msg.Data) + }) + if err != nil { + return fmt.Errorf("NATS subscribe to '%s' failed: %w", subject, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS subscribed to '%s'", subject) + return nil +} + +// SubscribeQueue registers a handler with queue group for load-balanced message processing. +func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) { + handler(msg.Subject, msg.Data) + }) + if err != nil { + return fmt.Errorf("NATS queue subscribe to '%s' (queue: %s) failed: %w", subject, queue, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS queue subscribed to '%s' (queue: %s)", subject, queue) + return nil +} + +// SubscribeChan subscribes to a subject and delivers messages to the provided channel. +func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.ChanSubscribe(subject, ch) + if err != nil { + return fmt.Errorf("NATS chan subscribe to '%s' failed: %w", subject, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS chan subscribed to '%s'", subject) + return nil +} + +// Publish sends data to the specified subject. +func (c *Client) Publish(subject string, data []byte) error { + if err := c.conn.Publish(subject, data); err != nil { + return fmt.Errorf("NATS publish to '%s' failed: %w", subject, err) + } + return nil +} + +// Request sends a request and waits for a response with the given context timeout. +func (c *Client) Request(subject string, data []byte, timeout context.Context) ([]byte, error) { + msg, err := c.conn.RequestWithContext(timeout, subject, data) + if err != nil { + return nil, fmt.Errorf("NATS request to '%s' failed: %w", subject, err) + } + return msg.Data, nil +} + +// Flush flushes the connection buffer to ensure all published messages are sent. +func (c *Client) Flush() error { + return c.conn.Flush() +} + +// Close unsubscribes all subscriptions and closes the NATS connection. +func (c *Client) Close() { + c.mu.Lock() + defer c.mu.Unlock() + + for _, sub := range c.subscriptions { + if err := sub.Unsubscribe(); err != nil { + cclog.Warnf("NATS unsubscribe failed: %v", err) + } + } + c.subscriptions = nil + + if c.conn != nil { + c.conn.Close() + cclog.Info("NATS connection closed") + } +} + +// IsConnected returns true if the client has an active connection. +func (c *Client) IsConnected() bool { + return c.conn != nil && c.conn.IsConnected() +} + +// Connection returns the underlying NATS connection for advanced usage. +func (c *Client) Connection() *nats.Conn { + return c.conn +} diff --git a/pkg/nats/config.go b/pkg/nats/config.go new file mode 100644 index 00000000..32a0bbda --- /dev/null +++ b/pkg/nats/config.go @@ -0,0 +1,63 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nats + +import ( + "bytes" + "encoding/json" + + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" +) + +// NatsConfig holds the configuration for connecting to a NATS server. +type NatsConfig struct { + Address string `json:"address"` // NATS server address (e.g., "nats://localhost:4222") + Username string `json:"username"` // Username for authentication (optional) + Password string `json:"password"` // Password for authentication (optional) + CredsFilePath string `json:"creds-file-path"` // Path to credentials file (optional) +} + +// Keys holds the global NATS configuration loaded via Init. +var Keys NatsConfig + +const ConfigSchema = `{ + "type": "object", + "description": "Configuration for NATS messaging client.", + "properties": { + "address": { + "description": "Address of the NATS server (e.g., 'nats://localhost:4222').", + "type": "string" + }, + "username": { + "description": "Username for NATS authentication (optional).", + "type": "string" + }, + "password": { + "description": "Password for NATS authentication (optional).", + "type": "string" + }, + "creds-file-path": { + "description": "Path to NATS credentials file for authentication (optional).", + "type": "string" + } + }, + "required": ["address"] +}` + +// Init initializes the global Keys configuration from JSON. +func Init(rawConfig json.RawMessage) error { + var err error + + if rawConfig != nil { + dec := json.NewDecoder(bytes.NewReader(rawConfig)) + dec.DisallowUnknownFields() + if err = dec.Decode(&Keys); err != nil { + cclog.Errorf("Error while initializing nats client: %s", err.Error()) + } + } + + return err +} From 5e2cbd75fae36d949180c0d71105def3a0482ea7 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:45:48 +0100 Subject: [PATCH 015/341] Review and refactor --- internal/memorystore/api.go | 16 +++- internal/memorystore/archive.go | 51 ++++++----- internal/memorystore/avroCheckpoint.go | 19 ++-- internal/memorystore/avroHelper.go | 6 +- internal/memorystore/avroStruct.go | 9 +- internal/memorystore/buffer.go | 14 +-- internal/memorystore/checkpoint.go | 48 +++------- internal/memorystore/config.go | 26 +++--- internal/memorystore/level.go | 2 +- internal/memorystore/lineprotocol.go | 119 +++---------------------- internal/memorystore/memorystore.go | 40 +++------ internal/memorystore/stats.go | 2 +- 12 files changed, 108 insertions(+), 244 deletions(-) diff --git a/internal/memorystore/api.go b/internal/memorystore/api.go index 1f7a531f..b96dc1fd 100644 --- a/internal/memorystore/api.go +++ b/internal/memorystore/api.go @@ -6,12 +6,18 @@ package memorystore import ( + "errors" "math" "github.com/ClusterCockpit/cc-lib/schema" "github.com/ClusterCockpit/cc-lib/util" ) +var ( + ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'") + ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty") +) + type APIMetricData struct { Error *string `json:"error,omitempty"` Data schema.FloatArray `json:"data,omitempty"` @@ -109,10 +115,14 @@ func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metr } func FetchData(req APIQueryRequest) (*APIQueryResponse, error) { - req.WithData = true - req.WithData = true - req.WithData = true + if req.From > req.To { + return nil, ErrInvalidTimeRange + } + if req.Cluster == "" && req.ForAllNodes != nil { + return nil, ErrEmptyCluster + } + req.WithData = true ms := GetMemoryStore() response := APIQueryResponse{ diff --git a/internal/memorystore/archive.go b/internal/memorystore/archive.go index 56065aaf..5019ee7a 100644 --- a/internal/memorystore/archive.go +++ b/internal/memorystore/archive.go @@ -32,17 +32,14 @@ func Archiving(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(d) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: t := time.Now().Add(-d) cclog.Infof("[METRICSTORE]> start archiving checkpoints (older than %s)...", t.Format(time.RFC3339)) n, err := ArchiveCheckpoints(Keys.Checkpoints.RootDir, @@ -165,25 +162,33 @@ func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead n := 0 for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - r, err := os.Open(filename) + // Use closure to ensure file is closed immediately after use, + // avoiding file descriptor leak from defer in loop + err := func() error { + filename := filepath.Join(dir, checkpoint) + r, err := os.Open(filename) + if err != nil { + return err + } + defer r.Close() + + w, err := zw.Create(checkpoint) + if err != nil { + return err + } + + if _, err = io.Copy(w, r); err != nil { + return err + } + + if err = os.Remove(filename); err != nil { + return err + } + return nil + }() if err != nil { return n, err } - defer r.Close() - - w, err := zw.Create(checkpoint) - if err != nil { - return n, err - } - - if _, err = io.Copy(w, r); err != nil { - return n, err - } - - if err = os.Remove(filename); err != nil { - return n, err - } n += 1 } diff --git a/internal/memorystore/avroCheckpoint.go b/internal/memorystore/avroCheckpoint.go index 4d361514..42e5f623 100644 --- a/internal/memorystore/avroCheckpoint.go +++ b/internal/memorystore/avroCheckpoint.go @@ -24,9 +24,8 @@ import ( "github.com/linkedin/goavro/v2" ) -var NumAvroWorkers int = 4 +var NumAvroWorkers int = DefaultAvroWorkers var startUp bool = true -var ErrNoNewData error = errors.New("no data in the pool") func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { levels := make([]*AvroLevel, 0) @@ -464,19 +463,15 @@ func generateRecord(data map[string]schema.Float) map[string]any { } func correctKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, ":", "___") - key = strings.ReplaceAll(key, ".", "__") - + key = strings.ReplaceAll(key, "_", "_0x5F_") + key = strings.ReplaceAll(key, ":", "_0x3A_") + key = strings.ReplaceAll(key, ".", "_0x2E_") return key } func ReplaceKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, "___", ":") - key = strings.ReplaceAll(key, "__", ".") - + key = strings.ReplaceAll(key, "_0x2E_", ".") + key = strings.ReplaceAll(key, "_0x3A_", ":") + key = strings.ReplaceAll(key, "_0x5F_", "_") return key } diff --git a/internal/memorystore/avroHelper.go b/internal/memorystore/avroHelper.go index 64e57064..a6f6c9bf 100644 --- a/internal/memorystore/avroHelper.go +++ b/internal/memorystore/avroHelper.go @@ -42,7 +42,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { metricName := "" for _, selectorName := range val.Selector { - metricName += selectorName + Delimiter + metricName += selectorName + SelectorDelimiter } metricName += val.MetricName @@ -54,7 +54,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { var selector []string selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10)) - if !testEq(oldSelector, selector) { + if !stringSlicesEqual(oldSelector, selector) { // Get the Avro level for the metric avroLevel = avroStore.root.findAvroLevelOrCreate(selector) @@ -71,7 +71,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { }() } -func testEq(a, b []string) bool { +func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { return false } diff --git a/internal/memorystore/avroStruct.go b/internal/memorystore/avroStruct.go index cc8005c7..bde9e02b 100644 --- a/internal/memorystore/avroStruct.go +++ b/internal/memorystore/avroStruct.go @@ -13,12 +13,11 @@ import ( var ( LineProtocolMessages = make(chan *AvroStruct) - Delimiter = "ZZZZZ" + // SelectorDelimiter separates hierarchical selector components in metric names for Avro encoding + SelectorDelimiter = "_SEL_" ) -// CheckpointBufferMinutes should always be in minutes. -// Its controls the amount of data to hold for given amount of time. -var CheckpointBufferMinutes = 3 +var CheckpointBufferMinutes = DefaultCheckpointBufferMin type AvroStruct struct { MetricName string @@ -73,7 +72,7 @@ func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel { } } - // The level does not exist, take write lock for unqiue access: + // The level does not exist, take write lock for unique access: l.lock.Lock() // While this thread waited for the write lock, another thread // could have created the child node. diff --git a/internal/memorystore/buffer.go b/internal/memorystore/buffer.go index cd2fd8fd..55be2ada 100644 --- a/internal/memorystore/buffer.go +++ b/internal/memorystore/buffer.go @@ -12,15 +12,12 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) -// Default buffer capacity. -// `buffer.data` will only ever grow up to it's capacity and a new link +// BufferCap is the default buffer capacity. +// buffer.data will only ever grow up to its capacity and a new link // in the buffer chain will be created if needed so that no copying // of data or reallocation needs to happen on writes. -const ( - BufferCap int = 512 -) +const BufferCap int = DefaultBufferCapacity -// So that we can reuse allocations var bufferPool sync.Pool = sync.Pool{ New: func() any { return &buffer{ @@ -75,7 +72,6 @@ func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) { newbuf := newBuffer(ts, b.frequency) newbuf.prev = b b.next = newbuf - b.close() b = newbuf idx = 0 } @@ -103,8 +99,6 @@ func (b *buffer) firstWrite() int64 { return b.start + (b.frequency / 2) } -func (b *buffer) close() {} - // Return all known values from `from` to `to`. Gaps of information are represented as NaN. // Simple linear interpolation is done between the two neighboring cells if possible. // If values at the start or end are missing, instead of NaN values, the second and thrid @@ -139,8 +133,6 @@ func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int6 data[i] += schema.NaN } else if t < b.start { data[i] += schema.NaN - // } else if b.data[idx].IsNaN() { - // data[i] += interpolate(idx, b.data) } else { data[i] += b.data[idx] } diff --git a/internal/memorystore/checkpoint.go b/internal/memorystore/checkpoint.go index e19cbf76..c676977c 100644 --- a/internal/memorystore/checkpoint.go +++ b/internal/memorystore/checkpoint.go @@ -28,15 +28,10 @@ import ( "github.com/linkedin/goavro/v2" ) -// File operation constants const ( - // CheckpointFilePerms defines default permissions for checkpoint files CheckpointFilePerms = 0o644 - // CheckpointDirPerms defines default permissions for checkpoint directories - CheckpointDirPerms = 0o755 - // GCTriggerInterval determines how often GC is forced during checkpoint loading - // GC is triggered every GCTriggerInterval*NumWorkers loaded hosts - GCTriggerInterval = 100 + CheckpointDirPerms = 0o755 + GCTriggerInterval = DefaultGCTriggerInterval ) // Whenever changed, update MarshalJSON as well! @@ -71,17 +66,14 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(d) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", lastCheckpoint.Format(time.RFC3339)) now := time.Now() n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, @@ -98,33 +90,23 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { } else { go func() { defer wg.Done() - d, _ := time.ParseDuration("1m") select { case <-ctx.Done(): return case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute): - // This is the first tick untill we collect the data for given minutes. GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) - } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(DefaultAvroCheckpointInterval) + defer ticker.Stop() for { select { case <-ctx.Done(): return - case <-ticks: - // Regular ticks of 1 minute to write data. + case <-ticker.C: GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) } } }() @@ -329,7 +311,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) ( lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics)) nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension) if err != nil { - cclog.Fatalf("[METRICSTORE]> error while loading checkpoints: %s", err.Error()) + cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error()) atomic.AddInt32(&errs, 1) } atomic.AddInt32(&n, int32(nn)) @@ -506,8 +488,8 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error { for key, floatArray := range metricsData { metricName := ReplaceKey(key) - if strings.Contains(metricName, Delimiter) { - subString := strings.Split(metricName, Delimiter) + if strings.Contains(metricName, SelectorDelimiter) { + subString := strings.Split(metricName, SelectorDelimiter) lvl := l @@ -557,12 +539,10 @@ func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schem next: nil, archived: true, } - b.close() minfo, ok := m.Metrics[metricName] if !ok { return nil - // return errors.New("Unkown metric: " + name) } prev := l.metrics[minfo.offset] @@ -616,17 +596,15 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error { b := &buffer{ frequency: metric.Frequency, start: metric.Start, - data: metric.Data[0:n:n], // Space is wasted here :( + data: metric.Data[0:n:n], prev: nil, next: nil, archived: true, } - b.close() minfo, ok := m.Metrics[name] if !ok { continue - // return errors.New("Unkown metric: " + name) } prev := l.metrics[minfo.offset] diff --git a/internal/memorystore/config.go b/internal/memorystore/config.go index 8196ed69..fbd62341 100644 --- a/internal/memorystore/config.go +++ b/internal/memorystore/config.go @@ -7,6 +7,16 @@ package memorystore import ( "fmt" + "time" +) + +const ( + DefaultMaxWorkers = 10 + DefaultBufferCapacity = 512 + DefaultGCTriggerInterval = 100 + DefaultAvroWorkers = 4 + DefaultCheckpointBufferMin = 3 + DefaultAvroCheckpointInterval = time.Minute ) var InternalCCMSFlag bool = false @@ -14,7 +24,7 @@ var InternalCCMSFlag bool = false type MetricStoreConfig struct { // Number of concurrent workers for checkpoint and archive operations. // If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10) - NumWorkers int `json:"num-workers"` + NumWorkers int `json:"num-workers"` Checkpoints struct { FileFormat string `json:"file-format"` Interval string `json:"interval"` @@ -31,20 +41,6 @@ type MetricStoreConfig struct { RootDir string `json:"directory"` DeleteInstead bool `json:"delete-instead"` } `json:"archive"` - Nats []*NatsConfig `json:"nats"` -} - -type NatsConfig struct { - // Address of the nats server - Address string `json:"address"` - - // Username/Password, optional - Username string `json:"username"` - Password string `json:"password"` - - // Creds file path - Credsfilepath string `json:"creds-file-path"` - Subscriptions []struct { // Channel name SubscribeTo string `json:"subscribe-to"` diff --git a/internal/memorystore/level.go b/internal/memorystore/level.go index aaa12103..f3b3d3f5 100644 --- a/internal/memorystore/level.go +++ b/internal/memorystore/level.go @@ -46,7 +46,7 @@ func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level { } } - // The level does not exist, take write lock for unqiue access: + // The level does not exist, take write lock for unique access: l.lock.Lock() // While this thread waited for the write lock, another thread // could have created the child node. diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index 2bbd7eeb..87d3b9e9 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -11,113 +11,31 @@ import ( "sync" "time" + "github.com/ClusterCockpit/cc-backend/pkg/nats" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" "github.com/influxdata/line-protocol/v2/lineprotocol" - "github.com/nats-io/nats.go" ) -// Each connection is handled in it's own goroutine. This is a blocking function. -// func ReceiveRaw(ctx context.Context, -// listener net.Listener, -// handleLine func(*lineprotocol.Decoder, string) error, -// ) error { -// var wg sync.WaitGroup - -// wg.Add(1) -// go func() { -// defer wg.Done() -// <-ctx.Done() -// if err := listener.Close(); err != nil { -// log.Printf("listener.Close(): %s", err.Error()) -// } -// }() - -// for { -// conn, err := listener.Accept() -// if err != nil { -// if errors.Is(err, net.ErrClosed) { -// break -// } - -// log.Printf("listener.Accept(): %s", err.Error()) -// } - -// wg.Add(2) -// go func() { -// defer wg.Done() -// defer conn.Close() - -// dec := lineprotocol.NewDecoder(conn) -// connctx, cancel := context.WithCancel(context.Background()) -// defer cancel() -// go func() { -// defer wg.Done() -// select { -// case <-connctx.Done(): -// conn.Close() -// case <-ctx.Done(): -// conn.Close() -// } -// }() - -// if err := handleLine(dec, "default"); err != nil { -// if errors.Is(err, net.ErrClosed) { -// return -// } - -// log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error()) -// errmsg := make([]byte, 128) -// errmsg = append(errmsg, `error: `...) -// errmsg = append(errmsg, err.Error()...) -// errmsg = append(errmsg, '\n') -// conn.Write(errmsg) -// } -// }() -// } - -// wg.Wait() -// return nil -// } - -// ReceiveNats connects to a nats server and subscribes to "updates". This is a -// blocking function. handleLine will be called for each line recieved via -// nats. Send `true` through the done channel for gracefull termination. -func ReceiveNats(conf *(NatsConfig), - ms *MemoryStore, +func ReceiveNats(ms *MemoryStore, workers int, ctx context.Context, ) error { - var opts []nats.Option - if conf.Username != "" && conf.Password != "" { - opts = append(opts, nats.UserInfo(conf.Username, conf.Password)) - } - - if conf.Credsfilepath != "" { - opts = append(opts, nats.UserCredentials(conf.Credsfilepath)) - } - - nc, err := nats.Connect(conf.Address, opts...) - if err != nil { - return err - } - defer nc.Close() + nc := nats.GetClient() var wg sync.WaitGroup - var subs []*nats.Subscription - msgs := make(chan *nats.Msg, workers*2) + msgs := make(chan []byte, workers*2) - for _, sc := range conf.Subscriptions { + for _, sc := range Keys.Subscriptions { clusterTag := sc.ClusterTag - var sub *nats.Subscription if workers > 1 { wg.Add(workers) for range workers { go func() { for m := range msgs { - dec := lineprotocol.NewDecoderWithBytes(m.Data) + dec := lineprotocol.NewDecoderWithBytes(m) if err := DecodeLine(dec, ms, clusterTag); err != nil { cclog.Errorf("error: %s", err.Error()) } @@ -127,37 +45,24 @@ func ReceiveNats(conf *(NatsConfig), }() } - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - msgs <- m + nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) { + msgs <- data }) } else { - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - dec := lineprotocol.NewDecoderWithBytes(m.Data) + nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) { + dec := lineprotocol.NewDecoderWithBytes(data) if err := DecodeLine(dec, ms, clusterTag); err != nil { cclog.Errorf("error: %s", err.Error()) } }) } - - if err != nil { - return err - } - cclog.Infof("NATS subscription to '%s' on '%s' established", sc.SubscribeTo, conf.Address) - subs = append(subs, sub) + cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo) } <-ctx.Done() - for _, sub := range subs { - err = sub.Unsubscribe() - if err != nil { - cclog.Errorf("NATS unsubscribe failed: %s", err.Error()) - } - } close(msgs) wg.Wait() - nc.Close() - cclog.Print("NATS connection closed") return nil } @@ -266,8 +171,6 @@ func DecodeLine(dec *lineprotocol.Decoder, case "stype-id": subTypeBuf = append(subTypeBuf, val...) default: - // Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need) - // return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val)) } } diff --git a/internal/memorystore/memorystore.go b/internal/memorystore/memorystore.go index 3e372f34..259a86ed 100644 --- a/internal/memorystore/memorystore.go +++ b/internal/memorystore/memorystore.go @@ -44,8 +44,6 @@ var ( shutdownFunc context.CancelFunc ) - - type Metric struct { Name string Value schema.Float @@ -71,8 +69,7 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { // Set NumWorkers from config or use default if Keys.NumWorkers <= 0 { - maxWorkers := 10 - Keys.NumWorkers = min(runtime.NumCPU()/2+1, maxWorkers) + Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers) } cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers) @@ -144,20 +141,9 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { // Store the shutdown function for later use by Shutdown() shutdownFunc = shutdown - if Keys.Nats != nil { - for _, natsConf := range Keys.Nats { - // TODO: When multiple nats configs share a URL, do a single connect. - wg.Add(1) - nc := natsConf - go func() { - // err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx) - err := ReceiveNats(nc, ms, 1, ctx) - if err != nil { - cclog.Fatal(err) - } - wg.Done() - }() - } + err = ReceiveNats(ms, 1, ctx) + if err != nil { + cclog.Fatal(err) } } @@ -244,18 +230,18 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - d := d / 2 - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + tickInterval := d / 2 + if tickInterval <= 0 { + return + } + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: t := time.Now().Add(-d) cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) freed, err := ms.Free(nil, t.Unix()) @@ -332,7 +318,7 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso minfo, ok := m.Metrics[metric] if !ok { - return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: " + metric) + return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric) } n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1) diff --git a/internal/memorystore/stats.go b/internal/memorystore/stats.go index 91b1f2cc..b2cb539a 100644 --- a/internal/memorystore/stats.go +++ b/internal/memorystore/stats.go @@ -77,7 +77,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6 minfo, ok := m.Metrics[metric] if !ok { - return nil, 0, 0, errors.New("unkown metric: " + metric) + return nil, 0, 0, errors.New("unknown metric: " + metric) } n, samples := 0, 0 From 102109388b8ae68c216c28702190abd8c23e5304 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 16 Dec 2025 13:54:17 +0100 Subject: [PATCH 016/341] link to public dashboard in admin options, add return button do public dashboard --- web/frontend/src/Config.root.svelte | 6 ++-- web/frontend/src/DashPublic.root.svelte | 6 ++++ web/frontend/src/config.entrypoint.js | 1 + web/frontend/src/config/AdminSettings.svelte | 6 ++-- web/frontend/src/config/admin/Options.svelte | 32 ++++++++++++++++++-- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/web/frontend/src/Config.root.svelte b/web/frontend/src/Config.root.svelte index e8a2045b..171b2a08 100644 --- a/web/frontend/src/Config.root.svelte +++ b/web/frontend/src/Config.root.svelte @@ -6,7 +6,8 @@ - `isSupport Bool!`: Is currently logged in user support authority - `isApi Bool!`: Is currently logged in user api authority - `username String!`: Empty string if auth. is disabled, otherwise the username as string - - `ncontent String!`: The currently displayed message on the homescreen + - `ncontent String!`: The currently displayed message on the homescreen + - `clusters [String]`: The available clusternames --> @@ -30,7 +32,7 @@ Admin Options - + {/if} diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index dac3f9aa..676e4969 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -30,6 +30,7 @@ Table, Progress, Icon, + Button } from "@sveltestrap/sveltestrap"; import Roofline from "./generic/plots/Roofline.svelte"; import Pie, { colors } from "./generic/plots/Pie.svelte"; @@ -353,6 +354,11 @@ }} /> + + + {#if $statusQuery.fetching || $statesTimed.fetching} diff --git a/web/frontend/src/config.entrypoint.js b/web/frontend/src/config.entrypoint.js index f9d8e45a..5d11eedd 100644 --- a/web/frontend/src/config.entrypoint.js +++ b/web/frontend/src/config.entrypoint.js @@ -10,6 +10,7 @@ mount(Config, { isApi: isApi, username: username, ncontent: ncontent, + clusters: hClusters.map((c) => c.name) // Defined in Header Template }, context: new Map([ ['cc-config', clusterCockpitConfig], diff --git a/web/frontend/src/config/AdminSettings.svelte b/web/frontend/src/config/AdminSettings.svelte index 1a426b8c..79072e30 100644 --- a/web/frontend/src/config/AdminSettings.svelte +++ b/web/frontend/src/config/AdminSettings.svelte @@ -3,6 +3,7 @@ Properties: - `ncontent String`: The homepage notice content + - `clusters [String]`: The available clusternames --> -{#if data && collectData[0].length > 0} +{#if data && collectData.length > 0}
diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte index c6abf068..47a841a3 100644 --- a/web/frontend/src/status/DashInternal.svelte +++ b/web/frontend/src/status/DashInternal.svelte @@ -487,45 +487,51 @@ - - -
-

- Top Projects: Jobs -

- tp['totalJobs'], - )} - entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} - /> -
- - -
- - - - - - {#each $topJobsQuery.data.jobsStatistics as tp, i} - - - - + {#if topJobsQuery?.data?.jobsStatistics?.length > 0} + + +
+

+ Top Projects: Jobs +

+ tp['totalJobs'], + )} + entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} + /> +
+ + +
ProjectJobs
- {scrambleNames ? scramble(tp.id) : tp.id} - - {tp['totalJobs']}
+ + + + - {/each} -
ProjectJobs
- - + {#each $topJobsQuery.data.jobsStatistics as tp, i} + + + + {scrambleNames ? scramble(tp.id) : tp.id} + + + {tp['totalJobs']} + + {/each} + + + + {:else} + Cannot render job status: No state data returned for Pie Chart + {/if} From 43e5fd113108d2262b4ba44be86135dd6a08180f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 05:44:49 +0100 Subject: [PATCH 018/341] Add NATS API backend --- internal/api/nats.go | 232 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 internal/api/nats.go diff --git a/internal/api/nats.go b/internal/api/nats.go new file mode 100644 index 00000000..e02e424f --- /dev/null +++ b/internal/api/nats.go @@ -0,0 +1,232 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package api + +import ( + "bytes" + "database/sql" + "encoding/json" + "sync" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/archiver" + "github.com/ClusterCockpit/cc-backend/internal/importer" + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/nats" + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/ClusterCockpit/cc-lib/schema" +) + +// NATS subject constants for Job and Node APIs. +const ( + SubjectJobStart = "cc.job.start" + SubjectJobStop = "cc.job.stop" + SubjectNodeState = "cc.node.state" +) + +// NatsAPI provides NATS subscription-based handlers for Job and Node operations. +// It mirrors the functionality of the REST API but uses NATS messaging. +type NatsAPI struct { + JobRepository *repository.JobRepository + // RepositoryMutex protects job creation operations from race conditions + // when checking for duplicate jobs during startJob calls. + RepositoryMutex sync.Mutex +} + +// NewNatsAPI creates a new NatsAPI instance with default dependencies. +func NewNatsAPI() *NatsAPI { + return &NatsAPI{ + JobRepository: repository.GetJobRepository(), + } +} + +// StartSubscriptions registers all NATS subscriptions for Job and Node APIs. +// Returns an error if the NATS client is not available or subscription fails. +func (api *NatsAPI) StartSubscriptions() error { + client := nats.GetClient() + if client == nil { + cclog.Warn("NATS client not available, skipping API subscriptions") + return nil + } + + if err := client.Subscribe(SubjectJobStart, api.handleStartJob); err != nil { + return err + } + + if err := client.Subscribe(SubjectJobStop, api.handleStopJob); err != nil { + return err + } + + if err := client.Subscribe(SubjectNodeState, api.handleNodeState); err != nil { + return err + } + + cclog.Info("NATS API subscriptions started") + return nil +} + +// handleStartJob processes job start messages received via NATS. +// Expected JSON payload follows the schema.Job structure. +func (api *NatsAPI) handleStartJob(subject string, data []byte) { + req := schema.Job{ + Shared: "none", + MonitoringStatus: schema.MonitoringStatusRunningOrArchiving, + } + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + cclog.Debugf("NATS %s: %s", subject, req.GoString()) + req.State = schema.JobStateRunning + + if err := importer.SanityChecks(&req); err != nil { + cclog.Errorf("NATS %s: sanity check failed: %v", subject, err) + return + } + + var unlockOnce sync.Once + api.RepositoryMutex.Lock() + defer unlockOnce.Do(api.RepositoryMutex.Unlock) + + jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil) + if err != nil && err != sql.ErrNoRows { + cclog.Errorf("NATS %s: checking for duplicate failed: %v", subject, err) + return + } + if err == nil { + for _, job := range jobs { + if (req.StartTime - job.StartTime) < secondsPerDay { + cclog.Errorf("NATS %s: job with jobId %d, cluster %s already exists (dbid: %d)", + subject, req.JobID, req.Cluster, job.ID) + return + } + } + } + + id, err := api.JobRepository.Start(&req) + if err != nil { + cclog.Errorf("NATS %s: insert into database failed: %v", subject, err) + return + } + unlockOnce.Do(api.RepositoryMutex.Unlock) + + for _, tag := range req.Tags { + if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil { + cclog.Errorf("NATS %s: adding tag to new job %d failed: %v", subject, id, err) + return + } + } + + cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", + id, req.Cluster, req.JobID, req.User, req.StartTime) +} + +// handleStopJob processes job stop messages received via NATS. +// Expected JSON payload follows the StopJobAPIRequest structure. +func (api *NatsAPI) handleStopJob(subject string, data []byte) { + var req StopJobAPIRequest + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + if req.JobID == nil { + cclog.Errorf("NATS %s: the field 'jobId' is required", subject) + return + } + + job, err := api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime) + if err != nil { + cachedJob, cachedErr := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime) + if cachedErr != nil { + cclog.Errorf("NATS %s: finding job failed: %v (cached lookup also failed: %v)", + subject, err, cachedErr) + return + } + job = cachedJob + } + + if job.State != schema.JobStateRunning { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: job has already been stopped (state is: %s)", + subject, job.JobID, job.ID, job.Cluster, job.State) + return + } + + if job.StartTime > req.StopTime { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d", + subject, job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime) + return + } + + if req.State != "" && !req.State.Valid() { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: invalid job state: %#v", + subject, job.JobID, job.ID, job.Cluster, req.State) + return + } else if req.State == "" { + req.State = schema.JobStateCompleted + } + + job.Duration = int32(req.StopTime - job.StartTime) + job.State = req.State + api.JobRepository.Mutex.Lock() + defer api.JobRepository.Mutex.Unlock() + + if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: marking job as '%s' failed: %v", + subject, job.JobID, job.ID, job.Cluster, job.State, err) + return + } + } + + cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s", + job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) + + if job.MonitoringStatus == schema.MonitoringStatusDisabled { + return + } + + archiver.TriggerArchiving(job) +} + +// handleNodeState processes node state update messages received via NATS. +// Expected JSON payload follows the UpdateNodeStatesRequest structure. +func (api *NatsAPI) handleNodeState(subject string, data []byte) { + var req UpdateNodeStatesRequest + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + repo := repository.GetNodeRepository() + + for _, node := range req.Nodes { + state := determineState(node.States) + nodeState := schema.NodeStateDB{ + TimeStamp: time.Now().Unix(), + NodeState: state, + CpusAllocated: node.CpusAllocated, + MemoryAllocated: node.MemoryAllocated, + GpusAllocated: node.GpusAllocated, + HealthState: schema.MonitoringStateFull, + JobsRunning: node.JobsRunning, + } + + repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState) + } + + cclog.Debugf("NATS %s: updated %d node states for cluster %s", subject, len(req.Nodes), req.Cluster) +} From d30c6ef3bf61183d3abd849cd91ee0d76c9cd1b6 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 06:08:09 +0100 Subject: [PATCH 019/341] Make NATS API subjects configurable --- internal/api/nats.go | 33 ++++++++++++++++----------------- internal/config/config.go | 8 ++++++++ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/internal/api/nats.go b/internal/api/nats.go index e02e424f..1bfe9051 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/archiver" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/nats" @@ -20,13 +21,6 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) -// NATS subject constants for Job and Node APIs. -const ( - SubjectJobStart = "cc.job.start" - SubjectJobStop = "cc.job.stop" - SubjectNodeState = "cc.node.state" -) - // NatsAPI provides NATS subscription-based handlers for Job and Node operations. // It mirrors the functionality of the REST API but uses NATS messaging. type NatsAPI struct { @@ -52,19 +46,24 @@ func (api *NatsAPI) StartSubscriptions() error { return nil } - if err := client.Subscribe(SubjectJobStart, api.handleStartJob); err != nil { - return err - } + if config.Keys.APISubjects != nil { - if err := client.Subscribe(SubjectJobStop, api.handleStopJob); err != nil { - return err - } + s := config.Keys.APISubjects - if err := client.Subscribe(SubjectNodeState, api.handleNodeState); err != nil { - return err - } + if err := client.Subscribe(s.SubjectJobStart, api.handleStartJob); err != nil { + return err + } - cclog.Info("NATS API subscriptions started") + if err := client.Subscribe(s.SubjectJobStop, api.handleStopJob); err != nil { + return err + } + + if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil { + return err + } + + cclog.Info("NATS API subscriptions started") + } return nil } diff --git a/internal/config/config.go b/internal/config/config.go index 69a44440..25ca27eb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,8 @@ type ProgramConfig struct { // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" APIAllowedIPs []string `json:"apiAllowedIPs"` + APISubjects *NATSConfig `json:"apiSubjects"` + // Drop root permissions once .env was read and the port was taken. User string `json:"user"` Group string `json:"group"` @@ -87,6 +89,12 @@ type ResampleConfig struct { Trigger int `json:"trigger"` } +type NATSConfig struct { + SubjectJobStart string `json:"subjectJobStart"` + SubjectJobStop string `json:"subjectJobStop"` + SubjectNodeState string `json:"subjectNodeState"` +} + type IntRange struct { From int `json:"from"` To int `json:"to"` From 88dc5036b3d8b64deed2c2d7161708ccef4599c0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 06:32:53 +0100 Subject: [PATCH 020/341] Make import function interuptible and replace countJobs with external call to fd --- tools/archive-manager/main.go | 110 ++++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index ae0ae9e6..940c92d4 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -5,13 +5,18 @@ package main import ( + "context" "encoding/json" "flag" "fmt" "os" + "os/exec" + "os/signal" + "strconv" "strings" "sync" "sync/atomic" + "syscall" "time" "github.com/ClusterCockpit/cc-backend/internal/config" @@ -34,13 +39,56 @@ func parseDate(in string) int64 { return 0 } -// countJobs counts the total number of jobs in the source archive. -func countJobs(srcBackend archive.ArchiveBackend) int { - count := 0 - for range srcBackend.Iter(false) { - count++ +// countJobs counts the total number of jobs in the source archive using external fd command. +// It requires the fd binary to be available in PATH. +// The srcConfig parameter should be the JSON configuration string containing the archive path. +func countJobs(srcConfig string) (int, error) { + fdPath, err := exec.LookPath("fd") + if err != nil { + return 0, fmt.Errorf("fd binary not found in PATH: %w", err) } - return count + + var config struct { + Kind string `json:"kind"` + Path string `json:"path"` + } + if err := json.Unmarshal([]byte(srcConfig), &config); err != nil { + return 0, fmt.Errorf("failed to parse source config: %w", err) + } + + if config.Path == "" { + return 0, fmt.Errorf("no path found in source config") + } + + fdCmd := exec.Command(fdPath, "meta.json", config.Path) + wcCmd := exec.Command("wc", "-l") + + pipe, err := fdCmd.StdoutPipe() + if err != nil { + return 0, fmt.Errorf("failed to create pipe: %w", err) + } + wcCmd.Stdin = pipe + + if err := fdCmd.Start(); err != nil { + return 0, fmt.Errorf("failed to start fd command: %w", err) + } + + output, err := wcCmd.Output() + if err != nil { + return 0, fmt.Errorf("failed to run wc command: %w", err) + } + + if err := fdCmd.Wait(); err != nil { + return 0, fmt.Errorf("fd command failed: %w", err) + } + + countStr := strings.TrimSpace(string(output)) + count, err := strconv.Atoi(countStr) + if err != nil { + return 0, fmt.Errorf("failed to parse count from wc output '%s': %w", countStr, err) + } + + return count, nil } // formatDuration formats a duration as a human-readable string. @@ -137,12 +185,34 @@ func (p *progressMeter) stop() { // importArchive imports all jobs from a source archive backend to a destination archive backend. // It uses parallel processing with a worker pool to improve performance. +// The import can be interrupted by CTRL-C (SIGINT) and will terminate gracefully. // Returns the number of successfully imported jobs, failed jobs, and any error encountered. -func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, error) { +func importArchive(srcBackend, dstBackend archive.ArchiveBackend, srcConfig string) (int, int, error) { cclog.Info("Starting parallel archive import...") + cclog.Info("Press CTRL-C to interrupt (will finish current jobs before exiting)") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + var interrupted atomic.Bool + + go func() { + <-sigChan + cclog.Warn("Interrupt received, stopping import (finishing current jobs)...") + interrupted.Store(true) + cancel() + // Stop listening for further signals to allow force quit with second CTRL-C + signal.Stop(sigChan) + }() cclog.Info("Counting jobs in source archive (this may take a long time) ...") - totalJobs := countJobs(srcBackend) + totalJobs, err := countJobs(srcConfig) + if err != nil { + return 0, 0, fmt.Errorf("failed to count jobs: %w", err) + } cclog.Infof("Found %d jobs to process", totalJobs) progress := newProgressMeter(totalJobs) @@ -200,8 +270,14 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err } go func() { + defer close(jobs) + clusters := srcBackend.GetClusters() for _, clusterName := range clusters { + if ctx.Err() != nil { + return + } + clusterCfg, err := srcBackend.LoadClusterCfg(clusterName) if err != nil { cclog.Errorf("Failed to load cluster config for %s: %v", clusterName, err) @@ -216,9 +292,14 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err } for job := range srcBackend.Iter(true) { - jobs <- job + select { + case <-ctx.Done(): + // Drain remaining items from iterator to avoid resource leak + // but don't process them + return + case jobs <- job: + } } - close(jobs) }() wg.Wait() @@ -229,6 +310,13 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err finalSkipped := int(atomic.LoadInt32(&progress.skipped)) elapsed := time.Since(progress.startTime) + + if interrupted.Load() { + cclog.Warnf("Import interrupted after %s: %d jobs imported, %d skipped, %d failed", + formatDuration(elapsed), finalImported, finalSkipped, finalFailed) + return finalImported, finalFailed, fmt.Errorf("import interrupted by user") + } + cclog.Infof("Import completed in %s: %d jobs imported, %d skipped, %d failed", formatDuration(elapsed), finalImported, finalSkipped, finalFailed) @@ -284,7 +372,7 @@ func main() { cclog.Info("Destination backend initialized successfully") // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, flagSrcConfig) if err != nil { cclog.Errorf("Import completed with errors: %s", err.Error()) if failed > 0 { From 4ecc050c4cb17353c66833ab3c7e16e39d46e2e6 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 07:03:01 +0100 Subject: [PATCH 021/341] Fix deadlock if NATS is not configured --- cmd/cc-backend/main.go | 2 +- internal/memorystore/lineprotocol.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index c3e33872..f9b198df 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -271,7 +271,7 @@ func initSubsystems() error { // Initialize nats client natsConfig := ccconf.GetPackageConfig("nats") if err := nats.Init(natsConfig); err != nil { - return fmt.Errorf("initializing nats client: %w", err) + cclog.Warnf("initializing (optional) nats client: %s", err.Error()) } nats.Connect() diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index 87d3b9e9..aebdbdca 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -23,6 +23,11 @@ func ReceiveNats(ms *MemoryStore, ) error { nc := nats.GetClient() + if nc == nil { + cclog.Warn("NATS client not initialized") + return nil + } + var wg sync.WaitGroup msgs := make(chan []byte, workers*2) From 0a5e15509627940bf360e816541e63c19d8c5f0e Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 07:03:10 +0100 Subject: [PATCH 022/341] Remove debug setting --- internal/taskmanager/retentionService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/taskmanager/retentionService.go b/internal/taskmanager/retentionService.go index 2bd221b4..acd07307 100644 --- a/internal/taskmanager/retentionService.go +++ b/internal/taskmanager/retentionService.go @@ -16,7 +16,7 @@ import ( func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { cclog.Info("Register retention delete service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(14, 30, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) From f4b00e9de173abfded748b683d9cd53d76088c13 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:38:00 +0100 Subject: [PATCH 023/341] Use Info instead of warn loglevel for database file missing msg --- internal/repository/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 58ab3e69..dec93a94 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -118,7 +118,7 @@ func MigrateDB(backend string, db string) error { v, dirty, err := m.Version() if err != nil { if err == migrate.ErrNilVersion { - cclog.Warn("Legacy database without version or missing database file!") + cclog.Info("Legacy database without version or missing database file!") } else { return err } From d1a78c13a4f7b99dea3e44259fa5a46549460f2c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:38:14 +0100 Subject: [PATCH 024/341] Make loglevel info default for demo --- startDemo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/startDemo.sh b/startDemo.sh index 10ba7f0c..b494e814 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -4,7 +4,7 @@ if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend -server -dev + ./cc-backend -server -dev -loglevel info else make ./cc-backend --init @@ -15,5 +15,5 @@ else rm ./job-archive-demo.tar ./cc-backend -dev -init-db -add-user demo:admin,api:demo - ./cc-backend -server -dev + ./cc-backend -server -dev -loglevel info fi From 79a2ca8ae8cb3c401c4ef38b7807d4b6d1871c2b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:44:37 +0100 Subject: [PATCH 025/341] Adapt unit test to new API --- tools/archive-manager/import_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/archive-manager/import_test.go b/tools/archive-manager/import_test.go index 02288285..b1032118 100644 --- a/tools/archive-manager/import_test.go +++ b/tools/archive-manager/import_test.go @@ -48,7 +48,7 @@ func TestImportFileToSqlite(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -111,13 +111,13 @@ func TestImportFileToFile(t *testing.T) { } // Create destination archive directory - if err := os.MkdirAll(dstArchive, 0755); err != nil { + if err := os.MkdirAll(dstArchive, 0o755); err != nil { t.Fatalf("Failed to create destination directory: %s", err.Error()) } // Write version file versionFile := filepath.Join(dstArchive, "version.txt") - if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil { + if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil { t.Fatalf("Failed to write version file: %s", err.Error()) } @@ -136,7 +136,7 @@ func TestImportFileToFile(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -183,7 +183,7 @@ func TestImportDataIntegrity(t *testing.T) { } // Perform import - _, _, err = importArchive(srcBackend, dstBackend) + _, _, err = importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -253,13 +253,13 @@ func TestImportEmptyArchive(t *testing.T) { dstDb := filepath.Join(tmpdir, "dst-archive.db") // Create empty source archive - if err := os.MkdirAll(srcArchive, 0755); err != nil { + if err := os.MkdirAll(srcArchive, 0o755); err != nil { t.Fatalf("Failed to create source directory: %s", err.Error()) } // Write version file versionFile := filepath.Join(srcArchive, "version.txt") - if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil { + if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil { t.Fatalf("Failed to write version file: %s", err.Error()) } @@ -277,7 +277,7 @@ func TestImportEmptyArchive(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import from empty archive should not fail: %s", err.Error()) } @@ -321,13 +321,13 @@ func TestImportDuplicateJobs(t *testing.T) { } // First import - imported1, _, err := importArchive(srcBackend, dstBackend) + imported1, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Fatalf("First import failed: %s", err.Error()) } // Second import (should skip all jobs) - imported2, _, err := importArchive(srcBackend, dstBackend) + imported2, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Second import failed: %s", err.Error()) } @@ -366,7 +366,7 @@ func TestImportToEmptyFileDestination(t *testing.T) { util.CopyDir(testDataPath, srcArchive) // Setup empty destination directory - os.MkdirAll(dstArchive, 0755) + os.MkdirAll(dstArchive, 0o755) // NOTE: NOT writing version.txt here! // Initialize source @@ -384,7 +384,7 @@ func TestImportToEmptyFileDestination(t *testing.T) { } // Perform import - imported, _, err := importArchive(srcBackend, dstBackend) + imported, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %v", err) } From b8fdfc30c06766651a39774fbc9685fb64b5c71f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 10:12:49 +0100 Subject: [PATCH 026/341] Fix performance bugs in sqlite archive backend --- internal/importer/initDB.go | 14 ++++---- pkg/archive/sqliteBackend.go | 52 ++++++++++++++++++++++------ pkg/archive/sqliteBackend_test.go | 57 ++++++++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index a4789576..12f49010 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -111,18 +111,22 @@ func InitDB() error { continue } - id, err := r.TransactionAddNamed(t, + id, jobErr := r.TransactionAddNamed(t, repository.NamedJobInsert, jobMeta) - if err != nil { - cclog.Errorf("repository initDB(): %v", err) + if jobErr != nil { + cclog.Errorf("repository initDB(): %v", jobErr) errorOccured++ continue } + // Job successfully inserted, increment counter + i += 1 + for _, tag := range jobMeta.Tags { tagstr := tag.Name + ":" + tag.Type tagID, ok := tags[tagstr] if !ok { + var err error tagID, err = r.TransactionAdd(t, addTagQuery, tag.Name, tag.Type) @@ -138,10 +142,6 @@ func InitDB() error { setTagQuery, id, tagID) } - - if err == nil { - i += 1 - } } if errorOccured > 0 { diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 6fa188ba..0b7a22d2 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -61,6 +61,7 @@ CREATE TABLE IF NOT EXISTS jobs ( CREATE INDEX IF NOT EXISTS idx_jobs_cluster ON jobs(cluster); CREATE INDEX IF NOT EXISTS idx_jobs_start_time ON jobs(start_time); +CREATE INDEX IF NOT EXISTS idx_jobs_order ON jobs(cluster, start_time); CREATE INDEX IF NOT EXISTS idx_jobs_lookup ON jobs(cluster, job_id, start_time); CREATE TABLE IF NOT EXISTS clusters ( @@ -560,11 +561,15 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { go func() { defer close(ch) - rows, err := sa.db.Query("SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") - if err != nil { - cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) + const chunkSize = 1000 + offset := 0 + + var query string + if loadMetricData { + query = "SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?" + } else { + query = "SELECT meta_json FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?" } - defer rows.Close() numWorkers := 4 jobRows := make(chan sqliteJobRow, numWorkers*2) @@ -615,13 +620,40 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { }() } - for rows.Next() { - var row sqliteJobRow - if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { - cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) - continue + for { + rows, err := sa.db.Query(query, chunkSize, offset) + if err != nil { + cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) } - jobRows <- row + + rowCount := 0 + for rows.Next() { + var row sqliteJobRow + + if loadMetricData { + if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { + cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) + continue + } + } else { + if err := rows.Scan(&row.metaBlob); err != nil { + cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) + continue + } + row.dataBlob = nil + row.compressed = false + } + + jobRows <- row + rowCount++ + } + rows.Close() + + if rowCount < chunkSize { + break + } + + offset += chunkSize } close(jobRows) diff --git a/pkg/archive/sqliteBackend_test.go b/pkg/archive/sqliteBackend_test.go index 285055fc..b72b8f6c 100644 --- a/pkg/archive/sqliteBackend_test.go +++ b/pkg/archive/sqliteBackend_test.go @@ -294,7 +294,7 @@ func TestSqliteCompress(t *testing.T) { // Compress should not panic even with missing data sa.Compress([]*schema.Job{job}) - + t.Log("Compression method verified") } @@ -311,3 +311,58 @@ func TestSqliteConfigParsing(t *testing.T) { t.Errorf("expected dbPath '/tmp/test.db', got '%s'", cfg.DBPath) } } + +func TestSqliteIterChunking(t *testing.T) { + tmpfile := t.TempDir() + "/test.db" + defer os.Remove(tmpfile) + + var sa SqliteArchive + _, err := sa.Init(json.RawMessage(`{"dbPath":"` + tmpfile + `"}`)) + if err != nil { + t.Fatalf("init failed: %v", err) + } + defer sa.db.Close() + + const totalJobs = 2500 + for i := 1; i <= totalJobs; i++ { + job := &schema.Job{ + JobID: int64(i), + Cluster: "test", + StartTime: int64(i * 1000), + NumNodes: 1, + Resources: []*schema.Resource{{Hostname: "node001"}}, + } + if err := sa.StoreJobMeta(job); err != nil { + t.Fatalf("store failed: %v", err) + } + } + + t.Run("IterWithoutData", func(t *testing.T) { + count := 0 + for container := range sa.Iter(false) { + if container.Meta == nil { + t.Error("expected non-nil meta") + } + if container.Data != nil { + t.Error("expected nil data when loadMetricData is false") + } + count++ + } + if count != totalJobs { + t.Errorf("expected %d jobs, got %d", totalJobs, count) + } + }) + + t.Run("IterWithData", func(t *testing.T) { + count := 0 + for container := range sa.Iter(true) { + if container.Meta == nil { + t.Error("expected non-nil meta") + } + count++ + } + if count != totalJobs { + t.Errorf("expected %d jobs, got %d", totalJobs, count) + } + }) +} From d2f2d7895426a4a50ccd507f3716542d220c2a0b Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Wed, 17 Dec 2025 15:58:42 +0100 Subject: [PATCH 027/341] Changing JWT output to stdout and change to help text --- .gitignore | 2 +- cmd/cc-backend/cli.go | 2 +- cmd/cc-backend/main.go | 2 +- configs/config.json | 8 ++++++-- startDemo.sh | 33 +++++++++++++++++++++++++-------- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index e03d8071..db9f922b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ /var/checkpoints* migrateTimestamps.pl -test_ccms_write_api.sh +test_ccms_write_api* /web/frontend/public/build /web/frontend/node_modules diff --git a/cmd/cc-backend/cli.go b/cmd/cc-backend/cli.go index af32b643..9ee56cb2 100644 --- a/cmd/cc-backend/cli.go +++ b/cmd/cc-backend/cli.go @@ -33,6 +33,6 @@ func cliInit() { flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: ") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info , warn (default), err, crit]`") flag.Parse() } diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index f9b198df..104182f2 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -263,7 +263,7 @@ func generateJWT(authHandle *auth.Authentication, username string) error { return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err) } - cclog.Infof("JWT: Successfully generated JWT for user '%s': %s", user.Username, jwt) + fmt.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) return nil } diff --git a/configs/config.json b/configs/config.json index 5bffc969..88a9e930 100644 --- a/configs/config.json +++ b/configs/config.json @@ -9,8 +9,12 @@ "apiAllowedIPs": ["*"], "short-running-jobs-duration": 300, "resampling": { - "trigger": 30, - "resolutions": [600, 300, 120, 60] + "minimumPoints": 600, + "trigger": 180, + "resolutions": [ + 240, + 60 + ] } }, "cron": { diff --git a/startDemo.sh b/startDemo.sh index b494e814..904caa02 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -1,19 +1,36 @@ #!/bin/sh -# rm -rf var +rm -rf var if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend -server -dev -loglevel info + ./cc-backend -server -dev else make - ./cc-backend --init + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar + tar xf job-archive-dev.tar + rm ./job-archive-dev.tar + + cp ./configs/env-template.txt .env cp ./configs/config-demo.json config.json - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar - tar xf job-archive-demo.tar - rm ./job-archive-demo.tar + echo 3 > /home/adityauj/cc-backend/var/job-archive/version.txt + + ./cc-backend --loglevel info -migrate-db + ./cc-backend --loglevel info -dev -init-db -add-user demo:admin,api:demo + + # Generate JWT and extract only the token value + JWT=$(./cc-backend -jwt demo | grep -oP "(?<=JWT: Successfully generated JWT for user 'demo': ).*") + + # Replace the existing JWT in test_ccms_write_api.sh with the new one + if [ -n "$JWT" ]; then + sed -i "1s|^JWT=.*|JWT=\"$JWT\"|" test_ccms_write_api.sh + echo "✅ Updated JWT in test_ccms_write_api.sh" + else + echo "❌ Failed to generate JWT for demo user" + exit 1 + fi + + ./cc-backend -server -dev - ./cc-backend -dev -init-db -add-user demo:admin,api:demo - ./cc-backend -server -dev -loglevel info fi From 32e535384708caeae0ac886293360515ff8eb9d0 Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Wed, 17 Dec 2025 18:14:36 +0100 Subject: [PATCH 028/341] Fix to NATS deadlock and revert demo script --- cmd/cc-backend/main.go | 2 +- configs/config-demo.json | 17 ++++++++++++++++- internal/memorystore/lineprotocol.go | 1 - startDemo.sh | 22 ++++------------------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 104182f2..6239d36c 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -263,7 +263,7 @@ func generateJWT(authHandle *auth.Authentication, username string) error { return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err) } - fmt.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) + cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) return nil } diff --git a/configs/config-demo.json b/configs/config-demo.json index 70ca2a02..58366fb5 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -29,6 +29,11 @@ "max-age": "2000h" } }, + "nats": { + "address": "nats://0.0.0.0:4222", + "username": "root", + "password": "root" + }, "clusters": [ { "name": "fritz", @@ -86,6 +91,16 @@ "interval": "1h", "directory": "./var/archive" }, - "retention-in-memory": "48h" + "retention-in-memory": "48h", + "subscriptions": [ + { + "subscribe-to": "hpc-nats", + "cluster-tag": "fritz" + }, + { + "subscribe-to": "hpc-nats", + "cluster-tag": "alex" + } + ] } } \ No newline at end of file diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index aebdbdca..6404361f 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -64,7 +64,6 @@ func ReceiveNats(ms *MemoryStore, cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo) } - <-ctx.Done() close(msgs) wg.Wait() diff --git a/startDemo.sh b/startDemo.sh index 904caa02..e709db27 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -1,6 +1,6 @@ #!/bin/sh -rm -rf var +# rm -rf var if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' @@ -14,23 +14,9 @@ else cp ./configs/env-template.txt .env cp ./configs/config-demo.json config.json - echo 3 > /home/adityauj/cc-backend/var/job-archive/version.txt - - ./cc-backend --loglevel info -migrate-db - ./cc-backend --loglevel info -dev -init-db -add-user demo:admin,api:demo - - # Generate JWT and extract only the token value - JWT=$(./cc-backend -jwt demo | grep -oP "(?<=JWT: Successfully generated JWT for user 'demo': ).*") - - # Replace the existing JWT in test_ccms_write_api.sh with the new one - if [ -n "$JWT" ]; then - sed -i "1s|^JWT=.*|JWT=\"$JWT\"|" test_ccms_write_api.sh - echo "✅ Updated JWT in test_ccms_write_api.sh" - else - echo "❌ Failed to generate JWT for demo user" - exit 1 - fi + ./cc-backend -migrate-db + ./cc-backend -dev -init-db -add-user demo:admin,api:demo ./cc-backend -server -dev -fi +fi \ No newline at end of file From 19c8e9beb15f1fc3ff6044eedc12f9e96d36ad78 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 18 Dec 2025 10:44:58 +0100 Subject: [PATCH 029/341] move extensive NodeMetricsList handling to node repo func --- internal/graph/schema.resolvers.go | 129 ++------------------------- internal/repository/node.go | 136 +++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 123 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 02046443..9747479a 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -806,140 +806,24 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub return nil, errors.New("you need to be administrator or support staff for this query") } + nodeRepo := repository.GetNodeRepository() + nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page) + if nerr != nil { + return nil, errors.New("Could not retrieve node list required for resolving NodeMetricsList") + } + if metrics == nil { for _, mc := range archive.GetCluster(cluster).MetricConfig { metrics = append(metrics, mc.Name) } } - // Build Filters - queryFilters := make([]*model.NodeFilter, 0) - if cluster != "" { - queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}}) - } - if subCluster != "" { - queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}}) - } - if nodeFilter != "" && stateFilter != "notindb" { - queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}}) - } - if stateFilter != "all" && stateFilter != "notindb" { - var queryState schema.SchedulerState = schema.SchedulerState(stateFilter) - queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState}) - } - // if healthFilter != "all" { - // filters = append(filters, &model.NodeFilter{HealthState: &healthFilter}) - // } - - // Special Case: Disable Paging for missing nodes filter, save IPP for later - var backupItems int - if stateFilter == "notindb" { - backupItems = page.ItemsPerPage - page.ItemsPerPage = -1 - } - - // Query Nodes From DB - nodeRepo := repository.GetNodeRepository() - rawNodes, serr := nodeRepo.QueryNodes(ctx, queryFilters, page, nil) // Order not Used - if serr != nil { - cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)") - return nil, serr - } - - // Intermediate Node Result Info - nodes := make([]string, 0) - stateMap := make(map[string]string) - for _, node := range rawNodes { - nodes = append(nodes, node.Hostname) - stateMap[node.Hostname] = string(node.NodeState) - } - - // Setup Vars - var countNodes int - var cerr error - var hasNextPage bool - - // Special Case: Find Nodes not in DB node table but in metricStore only - if stateFilter == "notindb" { - // Reapply Original Paging - page.ItemsPerPage = backupItems - // Get Nodes From Topology - var topoNodes []string - if subCluster != "" { - scNodes := archive.NodeLists[cluster][subCluster] - topoNodes = scNodes.PrintList() - } else { - subClusterNodeLists := archive.NodeLists[cluster] - for _, nodeList := range subClusterNodeLists { - topoNodes = append(topoNodes, nodeList.PrintList()...) - } - } - // Compare to all nodes from cluster/subcluster in DB - var missingNodes []string - for _, scanNode := range topoNodes { - if !slices.Contains(nodes, scanNode) { - missingNodes = append(missingNodes, scanNode) - } - } - // Filter nodes by name - if nodeFilter != "" { - filteredNodesByName := []string{} - for _, missingNode := range missingNodes { - if strings.Contains(missingNode, nodeFilter) { - filteredNodesByName = append(filteredNodesByName, missingNode) - } - } - missingNodes = filteredNodesByName - } - // Sort Missing Nodes Alphanumerically - slices.Sort(missingNodes) - // Total Missing - countNodes = len(missingNodes) - // Apply paging - if countNodes > page.ItemsPerPage { - start := (page.Page - 1) * page.ItemsPerPage - end := start + page.ItemsPerPage - if end > countNodes { - end = countNodes - hasNextPage = false - } else { - hasNextPage = true - } - nodes = missingNodes[start:end] - } else { - nodes = missingNodes - } - - } else { - // DB Nodes: Count and Find Next Page - countNodes, cerr = nodeRepo.CountNodes(ctx, queryFilters) - if cerr != nil { - cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)") - return nil, cerr - } - - // Example Page 4 @ 10 IpP : Does item 41 exist? - // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists. - nextPage := &model.PageRequest{ - ItemsPerPage: 1, - Page: ((page.Page * page.ItemsPerPage) + 1), - } - nextNodes, err := nodeRepo.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used - if err != nil { - cclog.Warn("Error while querying next nodes") - return nil, err - } - hasNextPage = len(nextNodes) == 1 - } - - // Load Metric Data For Specified Nodes Only data, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx) if err != nil { cclog.Warn("error while loading node data (Resolver.NodeMetricsList") return nil, err } - // Build Result nodeMetricsList := make([]*model.NodeMetrics, 0, len(data)) for hostname, metrics := range data { host := &model.NodeMetrics{ @@ -965,7 +849,6 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub nodeMetricsList = append(nodeMetricsList, host) } - // Final Return nodeMetricsListResult := &model.NodesResultList{ Items: nodeMetricsList, TotalNodes: &countNodes, diff --git a/internal/repository/node.go b/internal/repository/node.go index 9a1f3530..3b597eda 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -10,6 +10,8 @@ import ( "database/sql" "encoding/json" "fmt" + "slices" + "strings" "sync" "time" @@ -551,6 +553,140 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model. return timedStates, nil } +func (r *NodeRepository) GetNodesForList( + ctx context.Context, + cluster string, + subCluster string, + stateFilter string, + nodeFilter string, + page *model.PageRequest, +) ([]string, map[string]string, int, bool, error) { + + // Init Return Vars + nodes := make([]string, 0) + stateMap := make(map[string]string) + countNodes := 0 + hasNextPage := false + + // Build Filters + queryFilters := make([]*model.NodeFilter, 0) + if cluster != "" { + queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}}) + } + if subCluster != "" { + queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}}) + } + if nodeFilter != "" && stateFilter != "notindb" { + queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}}) + } + if stateFilter != "all" && stateFilter != "notindb" { + var queryState schema.SchedulerState = schema.SchedulerState(stateFilter) + queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState}) + } + // if healthFilter != "all" { + // filters = append(filters, &model.NodeFilter{HealthState: &healthFilter}) + // } + + // Special Case: Disable Paging for missing nodes filter, save IPP for later + var backupItems int + if stateFilter == "notindb" { + backupItems = page.ItemsPerPage + page.ItemsPerPage = -1 + } + + // Query Nodes From DB + rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used + if serr != nil { + cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)") + return nil, nil, 0, false, serr + } + + // Intermediate Node Result Info + for _, node := range rawNodes { + if node == nil { + continue + } + nodes = append(nodes, node.Hostname) + stateMap[node.Hostname] = string(node.NodeState) + } + + // Special Case: Find Nodes not in DB node table but in metricStore only + if stateFilter == "notindb" { + // Reapply Original Paging + page.ItemsPerPage = backupItems + // Get Nodes From Topology + var topoNodes []string + if subCluster != "" { + scNodes := archive.NodeLists[cluster][subCluster] + topoNodes = scNodes.PrintList() + } else { + subClusterNodeLists := archive.NodeLists[cluster] + for _, nodeList := range subClusterNodeLists { + topoNodes = append(topoNodes, nodeList.PrintList()...) + } + } + // Compare to all nodes from cluster/subcluster in DB + var missingNodes []string + for _, scanNode := range topoNodes { + if !slices.Contains(nodes, scanNode) { + missingNodes = append(missingNodes, scanNode) + } + } + // Filter nodes by name + if nodeFilter != "" { + filteredNodesByName := []string{} + for _, missingNode := range missingNodes { + if strings.Contains(missingNode, nodeFilter) { + filteredNodesByName = append(filteredNodesByName, missingNode) + } + } + missingNodes = filteredNodesByName + } + // Sort Missing Nodes Alphanumerically + slices.Sort(missingNodes) + // Total Missing + countNodes = len(missingNodes) + // Apply paging + if countNodes > page.ItemsPerPage { + start := (page.Page - 1) * page.ItemsPerPage + end := start + page.ItemsPerPage + if end > countNodes { + end = countNodes + hasNextPage = false + } else { + hasNextPage = true + } + nodes = missingNodes[start:end] + } else { + nodes = missingNodes + } + + } else { + // DB Nodes: Count and Find Next Page + var cerr error + countNodes, cerr = r.CountNodes(ctx, queryFilters) + if cerr != nil { + cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)") + return nil, nil, 0, false, cerr + } + + // Example Page 4 @ 10 IpP : Does item 41 exist? + // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists. + nextPage := &model.PageRequest{ + ItemsPerPage: 1, + Page: ((page.Page * page.ItemsPerPage) + 1), + } + nextNodes, err := r.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used + if err != nil { + cclog.Warn("Error while querying next nodes") + return nil, nil, 0, false, err + } + hasNextPage = len(nextNodes) == 1 + } + + return nodes, stateMap, countNodes, hasNextPage, nil +} + func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { user := GetUserFromContext(ctx) return AccessCheckWithUser(user, query) From e707fd08936ac683936e45af0155091e05ee7ced Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 18 Dec 2025 11:26:05 +0100 Subject: [PATCH 030/341] Provide fallback in archive manager in case fd is not available --- tools/archive-manager/main.go | 70 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 940c92d4..4972fe96 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -9,9 +9,11 @@ import ( "encoding/json" "flag" "fmt" + "io/fs" "os" "os/exec" "os/signal" + "path/filepath" "strconv" "strings" "sync" @@ -39,28 +41,47 @@ func parseDate(in string) int64 { return 0 } -// countJobs counts the total number of jobs in the source archive using external fd command. -// It requires the fd binary to be available in PATH. -// The srcConfig parameter should be the JSON configuration string containing the archive path. -func countJobs(srcConfig string) (int, error) { - fdPath, err := exec.LookPath("fd") - if err != nil { - return 0, fmt.Errorf("fd binary not found in PATH: %w", err) - } - +// parseArchivePath extracts the path from the source config JSON. +func parseArchivePath(srcConfig string) (string, error) { var config struct { Kind string `json:"kind"` Path string `json:"path"` } if err := json.Unmarshal([]byte(srcConfig), &config); err != nil { - return 0, fmt.Errorf("failed to parse source config: %w", err) + return "", fmt.Errorf("failed to parse source config: %w", err) } if config.Path == "" { - return 0, fmt.Errorf("no path found in source config") + return "", fmt.Errorf("no path found in source config") } - fdCmd := exec.Command(fdPath, "meta.json", config.Path) + return config.Path, nil +} + +// countJobsNative counts jobs using native Go filepath.WalkDir. +// This is used as a fallback when fd/fdfind is not available. +func countJobsNative(archivePath string) (int, error) { + count := 0 + err := filepath.WalkDir(archivePath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // Skip directories we can't access + } + if !d.IsDir() && d.Name() == "meta.json" { + count++ + } + return nil + }) + + if err != nil { + return 0, fmt.Errorf("failed to walk directory: %w", err) + } + + return count, nil +} + +// countJobsWithFd counts jobs using the external fd command. +func countJobsWithFd(fdPath, archivePath string) (int, error) { + fdCmd := exec.Command(fdPath, "meta.json", archivePath) wcCmd := exec.Command("wc", "-l") pipe, err := fdCmd.StdoutPipe() @@ -91,6 +112,31 @@ func countJobs(srcConfig string) (int, error) { return count, nil } +// countJobs counts the total number of jobs in the source archive. +// It tries to use external fd/fdfind command for speed, falling back to +// native Go filepath.WalkDir if neither is available. +// The srcConfig parameter should be the JSON configuration string containing the archive path. +func countJobs(srcConfig string) (int, error) { + archivePath, err := parseArchivePath(srcConfig) + if err != nil { + return 0, err + } + + // Try fd first (common name) + if fdPath, err := exec.LookPath("fd"); err == nil { + return countJobsWithFd(fdPath, archivePath) + } + + // Try fdfind (Debian/Ubuntu package name) + if fdPath, err := exec.LookPath("fdfind"); err == nil { + return countJobsWithFd(fdPath, archivePath) + } + + // Fall back to native Go implementation + cclog.Debug("fd/fdfind not found, using native Go file walker") + return countJobsNative(archivePath) +} + // formatDuration formats a duration as a human-readable string. func formatDuration(d time.Duration) string { if d < time.Minute { From 43bdb56072554ed32fdddfa783b9c184826574fa Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 18 Dec 2025 15:04:03 +0100 Subject: [PATCH 031/341] add fallback case if metric has no name in nodeListRow --- web/frontend/src/systems/nodelist/NodeListRow.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index d2c71ff4..bc93a323 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -148,13 +148,19 @@ hoststate={nodeData?.state? nodeData.state: 'notindb'}/> {/if} - {#each refinedData as metricData (metricData.data.name)} + {#each refinedData as metricData, i (metricData?.data?.name || i)} {#key metricData} {#if metricData?.disabled} Metric disabled for subcluster {metricData.data.name}:{nodeData.subCluster}{metricData?.data?.name ? metricData.data.name : `Metric Index ${i}`}:{nodeData.subCluster} + {:else if !metricData?.data?.name} + Metric without name for subcluster {`Metric Index ${i}`}:{nodeData.subCluster} {:else if !!metricData.data?.metric.statisticsSeries} From 6e74fa294aa3e3d61aa07a891d5ccf0dabbaf989 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 18 Dec 2025 15:47:30 +0100 Subject: [PATCH 032/341] Add role-based visibility for metrics Fixes #387 --- go.mod | 2 +- go.sum | 4 +- internal/graph/schema.resolvers.go | 45 +++++++------ pkg/archive/clusterConfig.go | 100 +++++++++++++++-------------- pkg/nats/client.go | 2 +- 5 files changed, 81 insertions(+), 72 deletions(-) diff --git a/go.mod b/go.mod index 75e62f1e..df8e1fb9 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ tool ( require ( github.com/99designs/gqlgen v0.17.84 - github.com/ClusterCockpit/cc-lib v1.0.0 + github.com/ClusterCockpit/cc-lib v1.0.2 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.0 github.com/aws/aws-sdk-go-v2/config v1.31.20 diff --git a/go.sum b/go.sum index e8630b7c..711c5551 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/ClusterCockpit/cc-lib v1.0.0 h1:/8DFRomt4BpVWKWrsEZ/ru4K8x76QTVnEgdwHc5eSps= -github.com/ClusterCockpit/cc-lib v1.0.0/go.mod h1:UGdOvXEnjFqlnPSxtvtFwO6BtXYW6NnXFoud9FtN93k= +github.com/ClusterCockpit/cc-lib v1.0.2 h1:ZWn3oZkXgxrr3zSigBdlOOfayZ4Om4xL20DhmritPPg= +github.com/ClusterCockpit/cc-lib v1.0.2/go.mod h1:UGdOvXEnjFqlnPSxtvtFwO6BtXYW6NnXFoud9FtN93k= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 9747479a..cd4af057 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -88,14 +88,14 @@ func (r *jobResolver) EnergyFootprint(ctx context.Context, obj *schema.Job) ([]* res := []*model.EnergyFootprintValue{} for name, value := range rawEnergyFootprint { // Suboptimal: Nearly hardcoded metric name expectations - matchCpu := regexp.MustCompile(`cpu|Cpu|CPU`) + matchCPU := regexp.MustCompile(`cpu|Cpu|CPU`) matchAcc := regexp.MustCompile(`acc|Acc|ACC`) matchMem := regexp.MustCompile(`mem|Mem|MEM`) matchCore := regexp.MustCompile(`core|Core|CORE`) hwType := "" switch test := name; { // NOtice ';' for var declaration - case matchCpu.MatchString(test): + case matchCPU.MatchString(test): hwType = "CPU" case matchAcc.MatchString(test): hwType = "Accelerator" @@ -175,9 +175,9 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds } tags := []*schema.Tag{} - for _, tagId := range tagIds { + for _, tagID := range tagIds { // Get ID - tid, err := strconv.ParseInt(tagId, 10, 64) + tid, err := strconv.ParseInt(tagID, 10, 64) if err != nil { cclog.Warn("Error while parsing tag id") return nil, err @@ -222,9 +222,9 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta } tags := []*schema.Tag{} - for _, tagId := range tagIds { + for _, tagID := range tagIds { // Get ID - tid, err := strconv.ParseInt(tagId, 10, 64) + tid, err := strconv.ParseInt(tagID, 10, 64) if err != nil { cclog.Warn("Error while parsing tag id") return nil, err @@ -265,9 +265,9 @@ func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []strin } tags := []int{} - for _, tagId := range tagIds { + for _, tagID := range tagIds { // Get ID - tid, err := strconv.ParseInt(tagId, 10, 64) + tid, err := strconv.ParseInt(tagID, 10, 64) if err != nil { cclog.Warn("Error while parsing tag id for removal") return nil, err @@ -317,7 +317,7 @@ func (r *nodeResolver) SchedulerState(ctx context.Context, obj *schema.Node) (sc if obj.NodeState != "" { return obj.NodeState, nil } else { - return "", fmt.Errorf("No SchedulerState (NodeState) on Object") + return "", fmt.Errorf("no SchedulerState (NodeState) on Object") } } @@ -343,6 +343,14 @@ func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) { // GlobalMetrics is the resolver for the globalMetrics field. func (r *queryResolver) GlobalMetrics(ctx context.Context) ([]*schema.GlobalMetricListItem, error) { + user := repository.GetUserFromContext(ctx) + + if user != nil { + if user.HasRole(schema.RoleUser) || user.HasRole(schema.RoleManager) { + return archive.GlobalUserMetricList, nil + } + } + return archive.GlobalMetricList, nil } @@ -373,12 +381,12 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]* // Node is the resolver for the node field. func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, error) { repo := repository.GetNodeRepository() - numericId, err := strconv.ParseInt(id, 10, 64) + numericID, err := strconv.ParseInt(id, 10, 64) if err != nil { cclog.Warn("Error while parsing job id") return nil, err } - return repo.GetNodeByID(numericId, false) + return repo.GetNodeByID(numericID, false) } // Nodes is the resolver for the nodes field. @@ -405,8 +413,7 @@ func (r *queryResolver) NodeStates(ctx context.Context, filter []*model.NodeFilt return nil, herr } - allCounts := make([]*model.NodeStates, 0) - allCounts = append(stateCounts, healthCounts...) + allCounts := append(stateCounts, healthCounts...) return allCounts, nil } @@ -433,18 +440,18 @@ func (r *queryResolver) NodeStatesTimed(ctx context.Context, filter []*model.Nod return healthCounts, nil } - return nil, errors.New("Unknown Node State Query Type") + return nil, errors.New("unknown Node State Query Type") } // Job is the resolver for the job field. func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) { - numericId, err := strconv.ParseInt(id, 10, 64) + numericID, err := strconv.ParseInt(id, 10, 64) if err != nil { cclog.Warn("Error while parsing job id") return nil, err } - job, err := r.Repo.FindByID(ctx, numericId) + job, err := r.Repo.FindByID(ctx, numericID) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -809,7 +816,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub nodeRepo := repository.GetNodeRepository() nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page) if nerr != nil { - return nil, errors.New("Could not retrieve node list required for resolving NodeMetricsList") + return nil, errors.New("could not retrieve node list required for resolving NodeMetricsList") } if metrics == nil { @@ -898,9 +905,7 @@ func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metr collectorUnit[metric] = scopedMetric.Unit // Collect Initial Data for _, ser := range scopedMetric.Series { - for _, val := range ser.Data { - collectorData[metric] = append(collectorData[metric], val) - } + collectorData[metric] = append(collectorData[metric], ser.Data...) } } } else { diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 13890c94..696601b7 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -6,7 +6,6 @@ package archive import ( - "errors" "fmt" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" @@ -14,13 +13,16 @@ import ( ) var ( - Clusters []*schema.Cluster - GlobalMetricList []*schema.GlobalMetricListItem - NodeLists map[string]map[string]NodeList + Clusters []*schema.Cluster + GlobalMetricList []*schema.GlobalMetricListItem + GlobalUserMetricList []*schema.GlobalMetricListItem + NodeLists map[string]map[string]NodeList ) func initClusterConfig() error { Clusters = []*schema.Cluster{} + GlobalMetricList = []*schema.GlobalMetricListItem{} + GlobalUserMetricList = []*schema.GlobalMetricListItem{} NodeLists = map[string]map[string]NodeList{} metricLookup := make(map[string]schema.GlobalMetricListItem) @@ -29,38 +31,41 @@ func initClusterConfig() error { cluster, err := ar.LoadClusterCfg(c) if err != nil { cclog.Warnf("Error while loading cluster config for cluster '%v'", c) - return err + return fmt.Errorf("failed to load cluster config for '%s': %w", c, err) } - if len(cluster.Name) == 0 || - len(cluster.MetricConfig) == 0 || - len(cluster.SubClusters) == 0 { - return errors.New("cluster.name, cluster.metricConfig and cluster.SubClusters should not be empty") + if len(cluster.Name) == 0 { + return fmt.Errorf("cluster name is empty in config for '%s'", c) + } + if len(cluster.MetricConfig) == 0 { + return fmt.Errorf("cluster '%s' has no metric configurations", cluster.Name) + } + if len(cluster.SubClusters) == 0 { + return fmt.Errorf("cluster '%s' has no subclusters defined", cluster.Name) } for _, mc := range cluster.MetricConfig { if len(mc.Name) == 0 { - return errors.New("cluster.metricConfig.name should not be empty") + return fmt.Errorf("cluster '%s' has a metric config with empty name", cluster.Name) } if mc.Timestep < 1 { - return errors.New("cluster.metricConfig.timestep should not be smaller than one") + return fmt.Errorf("metric '%s' in cluster '%s' has invalid timestep %d (must be >= 1)", mc.Name, cluster.Name, mc.Timestep) } - // For backwards compability... + // For backwards compatibility... if mc.Scope == "" { mc.Scope = schema.MetricScopeNode } if !mc.Scope.Valid() { - return errors.New("cluster.metricConfig.scope must be a valid scope ('node', 'scocket', ...)") + return fmt.Errorf("metric '%s' in cluster '%s' has invalid scope '%s' (must be 'node', 'socket', 'core', etc.)", mc.Name, cluster.Name, mc.Scope) } - ml, ok := metricLookup[mc.Name] - if !ok { + if _, ok := metricLookup[mc.Name]; !ok { metricLookup[mc.Name] = schema.GlobalMetricListItem{ - Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint, + Name: mc.Name, Scope: mc.Scope, Restrict: mc.Restrict, Unit: mc.Unit, Footprint: mc.Footprint, } - ml = metricLookup[mc.Name] } + availability := schema.ClusterSupport{Cluster: cluster.Name} scLookup := make(map[string]*schema.SubClusterConfig) @@ -90,39 +95,35 @@ func initClusterConfig() error { } if cfg, ok := scLookup[sc.Name]; ok { - if !cfg.Remove { - availability.SubClusters = append(availability.SubClusters, sc.Name) - newMetric.Peak = cfg.Peak - newMetric.Normal = cfg.Normal - newMetric.Caution = cfg.Caution - newMetric.Alert = cfg.Alert - newMetric.Footprint = cfg.Footprint - newMetric.Energy = cfg.Energy - newMetric.LowerIsBetter = cfg.LowerIsBetter - sc.MetricConfig = append(sc.MetricConfig, *newMetric) + if cfg.Remove { + continue + } + newMetric.Peak = cfg.Peak + newMetric.Normal = cfg.Normal + newMetric.Caution = cfg.Caution + newMetric.Alert = cfg.Alert + newMetric.Footprint = cfg.Footprint + newMetric.Energy = cfg.Energy + newMetric.LowerIsBetter = cfg.LowerIsBetter + } - if newMetric.Footprint != "" { - sc.Footprint = append(sc.Footprint, newMetric.Name) - ml.Footprint = newMetric.Footprint - } - if newMetric.Energy != "" { - sc.EnergyFootprint = append(sc.EnergyFootprint, newMetric.Name) - } - } - } else { - availability.SubClusters = append(availability.SubClusters, sc.Name) - sc.MetricConfig = append(sc.MetricConfig, *newMetric) + availability.SubClusters = append(availability.SubClusters, sc.Name) + sc.MetricConfig = append(sc.MetricConfig, *newMetric) - if newMetric.Footprint != "" { - sc.Footprint = append(sc.Footprint, newMetric.Name) - } - if newMetric.Energy != "" { - sc.EnergyFootprint = append(sc.EnergyFootprint, newMetric.Name) - } + if newMetric.Footprint != "" { + sc.Footprint = append(sc.Footprint, newMetric.Name) + item := metricLookup[mc.Name] + item.Footprint = newMetric.Footprint + metricLookup[mc.Name] = item + } + if newMetric.Energy != "" { + sc.EnergyFootprint = append(sc.EnergyFootprint, newMetric.Name) } } - ml.Availability = append(metricLookup[mc.Name].Availability, availability) - metricLookup[mc.Name] = ml + + item := metricLookup[mc.Name] + item.Availability = append(item.Availability, availability) + metricLookup[mc.Name] = item } Clusters = append(Clusters, cluster) @@ -141,8 +142,11 @@ func initClusterConfig() error { } } - for _, ml := range metricLookup { - GlobalMetricList = append(GlobalMetricList, &ml) + for _, metric := range metricLookup { + GlobalMetricList = append(GlobalMetricList, &metric) + if !metric.Restrict { + GlobalUserMetricList = append(GlobalUserMetricList, &metric) + } } return nil diff --git a/pkg/nats/client.go b/pkg/nats/client.go index e61d060b..822a7b26 100644 --- a/pkg/nats/client.go +++ b/pkg/nats/client.go @@ -83,7 +83,7 @@ func Connect() { client, err := NewClient(nil) if err != nil { - cclog.Errorf("NATS connection failed: %v", err) + cclog.Warnf("NATS connection failed: %v", err) return } From d446c135468056f11df75d272964b375dc6bfa7a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 18 Dec 2025 15:47:51 +0100 Subject: [PATCH 033/341] Restore startDemo script --- startDemo.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/startDemo.sh b/startDemo.sh index e709db27..108c95f5 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -1,22 +1,18 @@ #!/bin/sh -# rm -rf var - if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend -server -dev + ./cc-backend -server -dev -loglevel info else make - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar - tar xf job-archive-dev.tar - rm ./job-archive-dev.tar - - cp ./configs/env-template.txt .env + ./cc-backend --init cp ./configs/config-demo.json config.json - ./cc-backend -migrate-db + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar + tar xf job-archive-demo.tar + rm ./job-archive-demo.tar + ./cc-backend -dev -init-db -add-user demo:admin,api:demo + ./cc-backend -server -dev -loglevel info +fi - ./cc-backend -server -dev - -fi \ No newline at end of file From 436afa4a61116035e069801650551e8bd84c57f9 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 18 Dec 2025 15:55:30 +0100 Subject: [PATCH 034/341] fix tag count by including type in grouping --- internal/repository/tags.go | 9 +++++---- internal/routerConfig/routes.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 5ca13382..8a076e8a 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -224,10 +224,10 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts } // Query and Count Jobs with attached Tags - q := sq.Select("t.tag_name, t.id, count(jt.tag_id)"). + q := sq.Select("t.tag_type, t.tag_name, t.id, count(jt.tag_id)"). From("tag t"). LeftJoin("jobtag jt ON t.id = jt.tag_id"). - GroupBy("t.tag_name") + GroupBy("t.tag_type, t.tag_name") // Build scope list for filtering var scopeBuilder strings.Builder @@ -260,14 +260,15 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts counts = make(map[string]int) for rows.Next() { + var tagType string var tagName string var tagId int var count int - if err = rows.Scan(&tagName, &tagId, &count); err != nil { + if err = rows.Scan(&tagType, &tagName, &tagId, &count); err != nil { return nil, nil, err } // Use tagId as second Map-Key component to differentiate tags with identical names - counts[fmt.Sprint(tagName, tagId)] = count + counts[fmt.Sprint(tagType, tagName, tagId)] = count } err = rows.Err() diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index c2126cd0..4466034d 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -205,13 +205,13 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { "id": tag.ID, "name": tag.Name, "scope": tag.Scope, - "count": counts[fmt.Sprint(tag.Name, tag.ID)], + "count": counts[fmt.Sprint(tag.Type, tag.Name, tag.ID)], } tagMap[tag.Type] = append(tagMap[tag.Type], tagItem) } } else if userAuthlevel < 4 && userAuthlevel >= 2 { // User+ : Show global and admin scope only if at least 1 tag used, private scope regardless of count for _, tag := range tags { - tagCount := counts[fmt.Sprint(tag.Name, tag.ID)] + tagCount := counts[fmt.Sprint(tag.Type, tag.Name, tag.ID)] if ((tag.Scope == "global" || tag.Scope == "admin") && tagCount >= 1) || (tag.Scope != "global" && tag.Scope != "admin") { tagItem := map[string]interface{}{ "id": tag.ID, From c58b01a602543d893c0ffdecb34461ce4964a725 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 19 Dec 2025 14:42:02 +0100 Subject: [PATCH 035/341] fix wrong render condition order in nodeList --- web/frontend/src/systems/NodeList.svelte | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index c01ef237..e904076e 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -246,16 +246,7 @@ {$nodesQuery.error.message} - {:else} - {#each nodes as nodeData (nodeData.host)} - - {:else} - - No nodes found - - {/each} - {/if} - {#if $nodesQuery.fetching || !$nodesQuery.data} + {:else if $nodesQuery.fetching || !$nodesQuery.data}
@@ -272,6 +263,14 @@
+ {:else} + {#each nodes as nodeData (nodeData.host)} + + {:else} + + No nodes found + + {/each} {/if} From 7a0975b94d8af215139c2202c55f2dbfa0a2153a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 19 Dec 2025 15:10:15 +0100 Subject: [PATCH 036/341] final fix render race condition if metrics change in nodeList --- web/frontend/src/Systems.root.svelte | 2 +- web/frontend/src/systems/NodeList.svelte | 24 ++++++++++--------- .../src/systems/nodelist/NodeListRow.svelte | 18 ++++++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/web/frontend/src/Systems.root.svelte b/web/frontend/src/Systems.root.svelte index b27cefa9..d81a64cd 100644 --- a/web/frontend/src/Systems.root.svelte +++ b/web/frontend/src/Systems.root.svelte @@ -269,7 +269,7 @@ {:else} - + {/if} {/if} diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index e904076e..fa758c18 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -5,7 +5,7 @@ - `cluster String`: The nodes' cluster - `subCluster String`: The nodes' subCluster [Default: ""] - `ccconfig Object?`: The ClusterCockpit Config Context [Default: null] - - `selectedMetrics [String]`: The array of selected metrics [Default []] + - `pendingSelectedMetrics [String]`: The array of selected metrics [Default []] - `selectedResolution Number?`: The selected data resolution [Default: 0] - `hostnameFilter String?`: The active hostnamefilter [Default: ""] - `hoststateFilter String?`: The active hoststatefilter [Default: ""] @@ -27,7 +27,7 @@ cluster, subCluster = "", ccconfig = null, - selectedMetrics = [], + pendingSelectedMetrics = [], selectedResolution = 0, hostnameFilter = "", hoststateFilter = "", @@ -94,6 +94,7 @@ /* State Init */ let nodes = $state([]); + let selectedMetrics = $state(pendingSelectedMetrics); let page = $state(1); let itemsPerPage = $state(usePaging ? (ccconfig?.nodeList_nodesPerPage || 10) : 10); let headerPaddingTop = $state(0); @@ -110,7 +111,7 @@ stateFilter: hoststateFilter, nodeFilter: hostnameFilter, scopes: ["core", "socket", "accelerator"], - metrics: selectedMetrics, + metrics: pendingSelectedMetrics, from: from.toISOString(), to: to.toISOString(), paging: paging, @@ -140,15 +141,17 @@ $effect(() => { if ($nodesQuery?.data) { untrack(() => { - handleNodes($nodesQuery?.data?.nodeMetricsList); + nodes = handleNodes($nodesQuery?.data?.nodeMetricsList); + matchedNodes = $nodesQuery?.data?.totalNodes || 0; }); + selectedMetrics = [...pendingSelectedMetrics]; // Trigger Rerender in NodeListRow Only After Data is Fetched }; }); $effect(() => { // Triggers (Except Paging) from, to - selectedMetrics, selectedResolution + pendingSelectedMetrics, selectedResolution hostnameFilter, hoststateFilter // Continous Scroll: Paging if parameters change: Existing entries will not match new selections // Nodes Array Reset in HandleNodes func @@ -162,17 +165,16 @@ if (data) { if (usePaging) { // console.log('New Paging', $state.snapshot(paging)) - nodes = [...data.items].sort((a, b) => a.host.localeCompare(b.host)); + return [...data.items].sort((a, b) => a.host.localeCompare(b.host)); } else { if ($state.snapshot(page) == 1) { // console.log('Page 1 Reset', [...data.items]) - nodes = [...data.items].sort((a, b) => a.host.localeCompare(b.host)); + return [...data.items].sort((a, b) => a.host.localeCompare(b.host)); } else { // console.log('Add Nodes', $state.snapshot(nodes), [...data.items]) - nodes = nodes.concat([...data.items]) + return nodes.concat([...data.items]) } } - matchedNodes = data.totalNodes; }; }; @@ -228,7 +230,7 @@ {/if} - {#each selectedMetrics as metric (metric)} + {#each pendingSelectedMetrics as metric (metric)} {:else if $nodesQuery.fetching || !$nodesQuery.data} - +
{#if !usePaging}

diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index bc93a323..e7e095ea 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -128,6 +128,24 @@ } return pendingExtendedLegendData; } + + /* Inspect */ + // $inspect(selectedMetrics).with((type, selectedMetrics) => { + // console.log(type, 'selectedMetrics', selectedMetrics) + // }); + + // $inspect(nodeData).with((type, nodeData) => { + // console.log(type, 'nodeData', nodeData) + // }); + + // $inspect(refinedData).with((type, refinedData) => { + // console.log(type, 'refinedData', refinedData) + // }); + + // $inspect(dataHealth).with((type, dataHealth) => { + // console.log(type, 'dataHealth', dataHealth) + // }); + From 91b90d033e8703acffe5740872310a2f094fdcd2 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 19 Dec 2025 15:27:35 +0100 Subject: [PATCH 037/341] fix metric select drag and drop --- .../src/generic/select/MetricSelection.svelte | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/web/frontend/src/generic/select/MetricSelection.svelte b/web/frontend/src/generic/select/MetricSelection.svelte index 8bcaefcb..eeab56d7 100644 --- a/web/frontend/src/generic/select/MetricSelection.svelte +++ b/web/frontend/src/generic/select/MetricSelection.svelte @@ -107,13 +107,18 @@ } } + function columnsDragOver(event) { + event.preventDefault(); + event.dataTransfer.dropEffect = 'move'; + } + function columnsDragStart(event, i) { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.dropEffect = "move"; event.dataTransfer.setData("text/plain", i); } - function columnsDrag(event, target) { + function columnsDrop(event, target) { event.dataTransfer.dropEffect = "move"; const start = Number.parseInt(event.dataTransfer.getData("text/plain")); @@ -182,19 +187,18 @@ {/if} {#each listedMetrics as metric, index (metric)}

  • { - event.preventDefault() - return false + columnsDragOver(event) }} ondragstart={(event) => { columnsDragStart(event, index) }} ondrop={(event) => { event.preventDefault() - columnsDrag(event, index) + columnsDrop(event, index) }} ondragenter={() => (columnHovering = index)} > @@ -237,4 +241,10 @@ color: #fff; cursor: grabbing; } + + li.prevent-select { + -webkit-user-select: none; /* Safari */ + -ms-user-select: none; /* IE 10 and IE 11 */ + user-select: none; /* Standard syntax */ +} From af7d208c21c9dfbd539d2baa9c6bd6d4ea95d85c Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 19 Dec 2025 16:16:57 +0100 Subject: [PATCH 038/341] remove unused class --- web/frontend/src/generic/select/MetricSelection.svelte | 6 ------ 1 file changed, 6 deletions(-) diff --git a/web/frontend/src/generic/select/MetricSelection.svelte b/web/frontend/src/generic/select/MetricSelection.svelte index eeab56d7..67bbbd01 100644 --- a/web/frontend/src/generic/select/MetricSelection.svelte +++ b/web/frontend/src/generic/select/MetricSelection.svelte @@ -241,10 +241,4 @@ color: #fff; cursor: grabbing; } - - li.prevent-select { - -webkit-user-select: none; /* Safari */ - -ms-user-select: none; /* IE 10 and IE 11 */ - user-select: none; /* Standard syntax */ -} From 7acc89e42d6b23f0250f387f93dc72223cce1655 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 19 Dec 2025 17:52:21 +0100 Subject: [PATCH 039/341] move public dash close button --- web/frontend/src/DashPublic.root.svelte | 27 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 36e6703e..25e2683c 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -338,7 +338,7 @@ - + - - - {#if $statusQuery.fetching || $statesTimed.fetching} @@ -368,6 +363,13 @@ {:else if $statusQuery.error || $statesTimed.error} + + + + + {#if $statusQuery.error} @@ -385,8 +387,17 @@ - -

    Cluster {presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}

    + + + +

    Cluster {presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}

    + + + + +

    CPU(s)

    {[...clusterInfo?.processorTypes].join(', ')}

    From fdee4f89386aa8c712effe19467cf755d97807e8 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 20 Dec 2025 09:21:58 +0100 Subject: [PATCH 040/341] Integrate NATS API. Only start either REST start/stop API or NATS start/stop API --- cmd/cc-backend/server.go | 26 +++++++++++++++++--------- internal/api/rest.go | 7 +++++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 2c5ce8bc..4ed79622 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -49,9 +49,10 @@ const ( // Server encapsulates the HTTP server state and dependencies type Server struct { - router *mux.Router - server *http.Server - apiHandle *api.RestAPI + router *mux.Router + server *http.Server + restAPIHandle *api.RestAPI + natsAPIHandle *api.NatsAPI } func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) { @@ -104,7 +105,7 @@ func (s *Server) init() error { authHandle := auth.GetAuthInstance() - s.apiHandle = api.New() + s.restAPIHandle = api.New() info := map[string]any{} info["hasOpenIDConnect"] = false @@ -240,13 +241,20 @@ func (s *Server) init() error { // Mount all /monitoring/... and /api/... routes. routerConfig.SetupRoutes(secured, buildInfo) - s.apiHandle.MountAPIRoutes(securedapi) - s.apiHandle.MountUserAPIRoutes(userapi) - s.apiHandle.MountConfigAPIRoutes(configapi) - s.apiHandle.MountFrontendAPIRoutes(frontendapi) + s.restAPIHandle.MountAPIRoutes(securedapi) + s.restAPIHandle.MountUserAPIRoutes(userapi) + s.restAPIHandle.MountConfigAPIRoutes(configapi) + s.restAPIHandle.MountFrontendAPIRoutes(frontendapi) + + if config.Keys.APISubjects != nil { + s.natsAPIHandle = api.NewNatsAPI() + if err := s.natsAPIHandle.StartSubscriptions(); err != nil { + return fmt.Errorf("starting NATS subscriptions: %w", err) + } + } if memorystore.InternalCCMSFlag { - s.apiHandle.MountMetricStoreAPIRoutes(metricstoreapi) + s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi) } if config.Keys.EmbedStaticFiles { diff --git a/internal/api/rest.go b/internal/api/rest.go index 8232b64e..ebcf31ed 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -79,8 +79,11 @@ func (api *RestAPI) MountAPIRoutes(r *mux.Router) { // Slurm node state r.HandleFunc("/nodestate/", api.updateNodeStates).Methods(http.MethodPost, http.MethodPut) // Job Handler - r.HandleFunc("/jobs/start_job/", api.startJob).Methods(http.MethodPost, http.MethodPut) - r.HandleFunc("/jobs/stop_job/", api.stopJobByRequest).Methods(http.MethodPost, http.MethodPut) + if config.Keys.APISubjects == nil { + cclog.Info("Enabling REST start/stop job API") + r.HandleFunc("/jobs/start_job/", api.startJob).Methods(http.MethodPost, http.MethodPut) + r.HandleFunc("/jobs/stop_job/", api.stopJobByRequest).Methods(http.MethodPost, http.MethodPut) + } r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) r.HandleFunc("/jobs/{id}", api.getJobByID).Methods(http.MethodPost) r.HandleFunc("/jobs/{id}", api.getCompleteJobByID).Methods(http.MethodGet) From e56532e5c8b8b52f9e1089552cc949fb6844db43 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 20 Dec 2025 09:35:54 +0100 Subject: [PATCH 041/341] Add example json API payloads --- configs/startJobPayload.json | 22 ++++++++++++++++++++++ configs/stopJobPayload.json | 7 +++++++ 2 files changed, 29 insertions(+) create mode 100644 configs/startJobPayload.json create mode 100644 configs/stopJobPayload.json diff --git a/configs/startJobPayload.json b/configs/startJobPayload.json new file mode 100644 index 00000000..9517876f --- /dev/null +++ b/configs/startJobPayload.json @@ -0,0 +1,22 @@ +{ + "cluster": "fritz", + "jobId": 123000, + "jobState": "running", + "numAcc": 0, + "numHwthreads": 72, + "numNodes": 1, + "partition": "main", + "requestedMemory": 128000, + "resources": [{ "hostname": "f0726" }], + "startTime": 1649723812, + "subCluster": "main", + "submitTime": 1649723812, + "user": "k106eb10", + "project": "k106eb", + "walltime": 86400, + "metaData": { + "slurmInfo": "JobId=398759\nJobName=myJob\nUserId=dummyUser\nGroupId=dummyGroup\nAccount=dummyAccount\nQOS=normal Requeue=False Restarts=0 BatchFlag=True\nTimeLimit=1439'\nSubmitTime=2023-02-09T14:10:18\nPartition=singlenode\nNodeList=xx\nNumNodes=xx NumCPUs=72 NumTasks=72 CPUs/Task=1\nNTasksPerNode:Socket:Core=0:None:None\nTRES_req=cpu=72,mem=250000M,node=1,billing=72\nTRES_alloc=cpu=72,node=1,billing=72\nCommand=myCmd\nWorkDir=myDir\nStdErr=\nStdOut=\n", + "jobScript": "#!/bin/bash -l\n#SBATCH --job-name=dummy_job\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/dummy/\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\n\n#This is a dummy job script\n./mybinary\n", + "jobName": "ams_pipeline" + } +} diff --git a/configs/stopJobPayload.json b/configs/stopJobPayload.json new file mode 100644 index 00000000..baf76f95 --- /dev/null +++ b/configs/stopJobPayload.json @@ -0,0 +1,7 @@ +{ + "cluster": "fritz", + "jobId": 123000, + "jobState": "completed", + "startTime": 1649723812, + "stopTime": 1649763839 +} From 3cfcd301281c14af88f60e10790c4d52e44c213b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 20 Dec 2025 10:17:54 +0100 Subject: [PATCH 042/341] Add CLAUDE.md documentation for Claude Code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provides architecture overview, build commands, and development workflows to help future Claude Code instances work productively in this codebase. Includes guidance on GraphQL/REST API patterns, database migrations, and the repository/metric data architecture. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- CLAUDE.md | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..2bb08c98 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,198 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +ClusterCockpit is a job-specific performance monitoring framework for HPC clusters. This is a Golang backend that provides REST and GraphQL APIs, serves a Svelte-based frontend, and manages job archives and metric data from various time-series databases. + +## Build and Development Commands + +### Building + +```bash +# Build everything (frontend + backend) +make + +# Build only the frontend +make frontend + +# Build only the backend (requires frontend to be built first) +go build -ldflags='-s -X main.date=$(date +"%Y-%m-%d:T%H:%M:%S") -X main.version=1.4.4 -X main.commit=$(git rev-parse --short HEAD)' ./cmd/cc-backend +``` + +### Testing + +```bash +# Run all tests +make test + +# Run tests with verbose output +go test -v ./... + +# Run tests for a specific package +go test ./internal/repository +``` + +### Code Generation + +```bash +# Regenerate GraphQL schema and resolvers (after modifying api/*.graphqls) +make graphql + +# Regenerate Swagger/OpenAPI docs (after modifying API comments) +make swagger +``` + +### Frontend Development + +```bash +cd web/frontend + +# Install dependencies +npm install + +# Build for production +npm run build + +# Development mode with watch +npm run dev +``` + +### Running + +```bash +# Initialize database and create admin user +./cc-backend -init-db -add-user demo:admin:demo + +# Start server in development mode (enables GraphQL Playground and Swagger UI) +./cc-backend -server -dev -loglevel info + +# Start demo with sample data +./startDemo.sh +``` + +## Architecture + +### Backend Structure + +The backend follows a layered architecture with clear separation of concerns: + +- **cmd/cc-backend**: Entry point, orchestrates initialization of all subsystems +- **internal/repository**: Data access layer using repository pattern + - Abstracts database operations (SQLite/MySQL) + - Implements LRU caching for performance + - Provides repositories for Job, User, Node, and Tag entities + - Transaction support for batch operations +- **internal/api**: REST API endpoints (Swagger/OpenAPI documented) +- **internal/graph**: GraphQL API (uses gqlgen) + - Schema in `api/*.graphqls` + - Generated code in `internal/graph/generated/` + - Resolvers in `internal/graph/schema.resolvers.go` +- **internal/auth**: Authentication layer + - Supports local accounts, LDAP, OIDC, and JWT tokens + - Implements rate limiting for login attempts +- **internal/metricdata**: Metric data repository abstraction + - Pluggable backends: cc-metric-store, Prometheus, InfluxDB + - Each cluster can have a different metric data backend +- **internal/archiver**: Job archiving to file-based archive +- **pkg/archive**: Job archive backend implementations + - File system backend (default) + - S3 backend + - SQLite backend (experimental) +- **pkg/nats**: NATS integration for metric ingestion + +### Frontend Structure + +- **web/frontend**: Svelte 5 application + - Uses Rollup for building + - Components organized by feature (analysis, job, user, etc.) + - GraphQL client using @urql/svelte + - Bootstrap 5 + SvelteStrap for UI + - uPlot for time-series visualization +- **web/templates**: Server-side Go templates + +### Key Concepts + +**Job Archive**: Completed jobs are stored in a file-based archive following the [ClusterCockpit job-archive specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). Each job has a `meta.json` file with metadata and metric data files. + +**Metric Data Repositories**: Time-series metric data is stored separately from job metadata. The system supports multiple backends (cc-metric-store is recommended). Configuration is per-cluster in `config.json`. + +**Authentication Flow**: +1. Multiple authenticators can be configured (local, LDAP, OIDC, JWT) +2. Each authenticator's `CanLogin` method is called to determine if it should handle the request +3. The first authenticator that returns true performs the actual `Login` +4. JWT tokens are used for API authentication + +**Database Migrations**: SQL migrations in `internal/repository/migrations/` are applied automatically on startup. Version tracking in `version` table. + +**Scopes**: Metrics can be collected at different scopes: +- Node scope (always available) +- Core scope (for jobs with ≤8 nodes) +- Accelerator scope (for GPU/accelerator metrics) + +## Configuration + +- **config.json**: Main configuration (clusters, metric repositories, archive settings) +- **.env**: Environment variables (secrets like JWT keys) + - Copy from `configs/env-template.txt` + - NEVER commit this file +- **cluster.json**: Cluster topology and metric definitions (loaded from archive or config) + +## Database + +- Default: SQLite 3 (`./var/job.db`) +- Optional: MySQL/MariaDB +- Connection managed by `internal/repository` +- Schema version in `internal/repository/migration.go` + +## Code Generation + +**GraphQL** (gqlgen): +- Schema: `api/*.graphqls` +- Config: `gqlgen.yml` +- Generated code: `internal/graph/generated/` +- Custom resolvers: `internal/graph/schema.resolvers.go` +- Run `make graphql` after schema changes + +**Swagger/OpenAPI**: +- Annotations in `internal/api/*.go` +- Generated docs: `api/docs.go`, `api/swagger.yaml` +- Run `make swagger` after API changes + +## Testing Conventions + +- Test files use `_test.go` suffix +- Test data in `testdata/` subdirectories +- Repository tests use in-memory SQLite +- API tests use httptest + +## Common Workflows + +### Adding a new GraphQL field +1. Edit schema in `api/*.graphqls` +2. Run `make graphql` +3. Implement resolver in `internal/graph/schema.resolvers.go` + +### Adding a new REST endpoint +1. Add handler in `internal/api/*.go` +2. Add route in `internal/api/rest.go` +3. Add Swagger annotations +4. Run `make swagger` + +### Adding a new metric data backend +1. Implement `MetricDataRepository` interface in `internal/metricdata/` +2. Register in `metricdata.Init()` switch statement +3. Update config.json schema documentation + +### Modifying database schema +1. Create new migration in `internal/repository/migrations/` +2. Increment `repository.Version` +3. Test with fresh database and existing database + +## Dependencies + +- Go 1.24.0+ (check go.mod for exact version) +- Node.js (for frontend builds) +- SQLite 3 or MySQL/MariaDB +- Optional: NATS server for metric ingestion From b35172e2f7bc56fad47a12ef36398ddba376d6db Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 20 Dec 2025 11:13:02 +0100 Subject: [PATCH 043/341] Add context information for CLAUDE coding agent --- CLAUDE.md | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2bb08c98..379b4dbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,10 +1,14 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides guidance to Claude Code (claude.ai/code) when working with +code in this repository. ## Project Overview -ClusterCockpit is a job-specific performance monitoring framework for HPC clusters. This is a Golang backend that provides REST and GraphQL APIs, serves a Svelte-based frontend, and manages job archives and metric data from various time-series databases. +ClusterCockpit is a job-specific performance monitoring framework for HPC +clusters. This is a Golang backend that provides REST and GraphQL APIs, serves a +Svelte-based frontend, and manages job archives and metric data from various +time-series databases. ## Build and Development Commands @@ -80,7 +84,7 @@ The backend follows a layered architecture with clear separation of concerns: - **cmd/cc-backend**: Entry point, orchestrates initialization of all subsystems - **internal/repository**: Data access layer using repository pattern - - Abstracts database operations (SQLite/MySQL) + - Abstracts database operations (SQLite3 only) - Implements LRU caching for performance - Provides repositories for Job, User, Node, and Tag entities - Transaction support for batch operations @@ -114,19 +118,27 @@ The backend follows a layered architecture with clear separation of concerns: ### Key Concepts -**Job Archive**: Completed jobs are stored in a file-based archive following the [ClusterCockpit job-archive specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). Each job has a `meta.json` file with metadata and metric data files. +**Job Archive**: Completed jobs are stored in a file-based archive following the +[ClusterCockpit job-archive +specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). +Each job has a `meta.json` file with metadata and metric data files. -**Metric Data Repositories**: Time-series metric data is stored separately from job metadata. The system supports multiple backends (cc-metric-store is recommended). Configuration is per-cluster in `config.json`. +**Metric Data Repositories**: Time-series metric data is stored separately from +job metadata. The system supports multiple backends (cc-metric-store is +recommended). Configuration is per-cluster in `config.json`. **Authentication Flow**: + 1. Multiple authenticators can be configured (local, LDAP, OIDC, JWT) 2. Each authenticator's `CanLogin` method is called to determine if it should handle the request 3. The first authenticator that returns true performs the actual `Login` 4. JWT tokens are used for API authentication -**Database Migrations**: SQL migrations in `internal/repository/migrations/` are applied automatically on startup. Version tracking in `version` table. +**Database Migrations**: SQL migrations in `internal/repository/migrations/` are +applied automatically on startup. Version tracking in `version` table. **Scopes**: Metrics can be collected at different scopes: + - Node scope (always available) - Core scope (for jobs with ≤8 nodes) - Accelerator scope (for GPU/accelerator metrics) @@ -142,13 +154,13 @@ The backend follows a layered architecture with clear separation of concerns: ## Database - Default: SQLite 3 (`./var/job.db`) -- Optional: MySQL/MariaDB - Connection managed by `internal/repository` - Schema version in `internal/repository/migration.go` ## Code Generation **GraphQL** (gqlgen): + - Schema: `api/*.graphqls` - Config: `gqlgen.yml` - Generated code: `internal/graph/generated/` @@ -156,6 +168,7 @@ The backend follows a layered architecture with clear separation of concerns: - Run `make graphql` after schema changes **Swagger/OpenAPI**: + - Annotations in `internal/api/*.go` - Generated docs: `api/docs.go`, `api/swagger.yaml` - Run `make swagger` after API changes @@ -170,22 +183,26 @@ The backend follows a layered architecture with clear separation of concerns: ## Common Workflows ### Adding a new GraphQL field + 1. Edit schema in `api/*.graphqls` 2. Run `make graphql` 3. Implement resolver in `internal/graph/schema.resolvers.go` ### Adding a new REST endpoint + 1. Add handler in `internal/api/*.go` 2. Add route in `internal/api/rest.go` 3. Add Swagger annotations 4. Run `make swagger` ### Adding a new metric data backend + 1. Implement `MetricDataRepository` interface in `internal/metricdata/` 2. Register in `metricdata.Init()` switch statement 3. Update config.json schema documentation ### Modifying database schema + 1. Create new migration in `internal/repository/migrations/` 2. Increment `repository.Version` 3. Test with fresh database and existing database @@ -194,5 +211,5 @@ The backend follows a layered architecture with clear separation of concerns: - Go 1.24.0+ (check go.mod for exact version) - Node.js (for frontend builds) -- SQLite 3 or MySQL/MariaDB +- SQLite 3 (only supported database) - Optional: NATS server for metric ingestion From 1cd4a57bd3206e1f3115c1cbc58fcad5cbfb87a5 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 20 Dec 2025 11:13:41 +0100 Subject: [PATCH 044/341] Remove support for mysql/mariadb --- README.md | 11 +- cmd/cc-backend/init.go | 4 +- cmd/cc-backend/main.go | 19 ++- configs/config-mariadb.json | 64 --------- go.mod | 2 - go.sum | 49 +------ init/clustercockpit.service | 2 +- internal/api/api_test.go | 2 +- internal/config/config.go | 4 +- internal/config/schema.go | 2 +- internal/importer/importer_test.go | 2 +- internal/repository/dbConnection.go | 56 ++++---- internal/repository/job.go | 52 ++------ internal/repository/migration.go | 89 ++++--------- .../migrations/mysql/01_init-schema.down.sql | 5 - .../migrations/mysql/01_init-schema.up.sql | 66 ---------- .../migrations/mysql/02_add-index.down.sql | 8 -- .../migrations/mysql/02_add-index.up.sql | 8 -- .../mysql/03_add-userprojects.down.sql | 1 - .../mysql/03_add-userprojects.up.sql | 1 - .../mysql/04_alter-table-job.down.sql | 5 - .../mysql/04_alter-table-job.up.sql | 5 - .../migrations/mysql/05_extend-tags.down.sql | 2 - .../migrations/mysql/05_extend-tags.up.sql | 2 - .../mysql/06_change-config.down.sql | 1 - .../migrations/mysql/06_change-config.up.sql | 1 - .../migrations/mysql/07_fix-tag-id.down.sql | 3 - .../migrations/mysql/07_fix-tag-id.up.sql | 3 - .../mysql/08_add-footprint.down.sql | 83 ------------ .../migrations/mysql/08_add-footprint.up.sql | 123 ------------------ internal/repository/node_test.go | 2 +- internal/repository/repository_test.go | 2 +- internal/repository/stats.go | 49 +++---- internal/repository/userConfig_test.go | 2 +- internal/tagger/detectApp_test.go | 2 +- 35 files changed, 104 insertions(+), 628 deletions(-) delete mode 100644 configs/config-mariadb.json delete mode 100644 internal/repository/migrations/mysql/01_init-schema.down.sql delete mode 100644 internal/repository/migrations/mysql/01_init-schema.up.sql delete mode 100644 internal/repository/migrations/mysql/02_add-index.down.sql delete mode 100644 internal/repository/migrations/mysql/02_add-index.up.sql delete mode 100644 internal/repository/migrations/mysql/03_add-userprojects.down.sql delete mode 100644 internal/repository/migrations/mysql/03_add-userprojects.up.sql delete mode 100644 internal/repository/migrations/mysql/04_alter-table-job.down.sql delete mode 100644 internal/repository/migrations/mysql/04_alter-table-job.up.sql delete mode 100644 internal/repository/migrations/mysql/05_extend-tags.down.sql delete mode 100644 internal/repository/migrations/mysql/05_extend-tags.up.sql delete mode 100644 internal/repository/migrations/mysql/06_change-config.down.sql delete mode 100644 internal/repository/migrations/mysql/06_change-config.up.sql delete mode 100644 internal/repository/migrations/mysql/07_fix-tag-id.down.sql delete mode 100644 internal/repository/migrations/mysql/07_fix-tag-id.up.sql delete mode 100644 internal/repository/migrations/mysql/08_add-footprint.down.sql delete mode 100644 internal/repository/migrations/mysql/08_add-footprint.up.sql diff --git a/README.md b/README.md index 0799bd92..a0352d17 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,11 @@ is also served by the backend using [Svelte](https://svelte.dev/) components. Layout and styling are based on [Bootstrap 5](https://getbootstrap.com/) using [Bootstrap Icons](https://icons.getbootstrap.com/). -The backend uses [SQLite 3](https://sqlite.org/) as a relational SQL database by -default. Optionally it can use a MySQL/MariaDB database server. While there are -metric data backends for the InfluxDB and Prometheus time series databases, the -only tested and supported setup is to use cc-metric-store as the metric data -backend. Documentation on how to integrate ClusterCockpit with other time series -databases will be added in the future. +The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database. +While there are metric data backends for the InfluxDB and Prometheus time series +databases, the only tested and supported setup is to use cc-metric-store as the +metric data backend. Documentation on how to integrate ClusterCockpit with other +time series databases will be added in the future. Completed batch jobs are stored in a file-based job archive according to [this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index ee60b12c..151eee9e 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -105,9 +105,9 @@ func initEnv() { cclog.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error()) } - err := repository.MigrateDB("sqlite3", "./var/job.db") + err := repository.MigrateDB("./var/job.db") if err != nil { - cclog.Abortf("Could not initialize default sqlite3 database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error()) + cclog.Abortf("Could not initialize default SQLite database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error()) } if err := os.Mkdir("var/job-archive", 0o777); err != nil { cclog.Abortf("Could not create default ./var/job-archive folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error()) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 6239d36c..9464ccf4 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -40,7 +40,6 @@ import ( "github.com/google/gops/agent" "github.com/joho/godotenv" - _ "github.com/go-sql-driver/mysql" _ "github.com/mattn/go-sqlite3" ) @@ -120,30 +119,30 @@ func initDatabase() error { func handleDatabaseCommands() error { if flagMigrateDB { - err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB) + err := repository.MigrateDB(config.Keys.DB) if err != nil { return fmt.Errorf("migrating database to version %d: %w", repository.Version, err) } - cclog.Exitf("MigrateDB Success: Migrated '%s' database at location '%s' to version %d.\n", - config.Keys.DBDriver, config.Keys.DB, repository.Version) + cclog.Exitf("MigrateDB Success: Migrated SQLite database at '%s' to version %d.\n", + config.Keys.DB, repository.Version) } if flagRevertDB { - err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB) + err := repository.RevertDB(config.Keys.DB) if err != nil { return fmt.Errorf("reverting database to version %d: %w", repository.Version-1, err) } - cclog.Exitf("RevertDB Success: Reverted '%s' database at location '%s' to version %d.\n", - config.Keys.DBDriver, config.Keys.DB, repository.Version-1) + cclog.Exitf("RevertDB Success: Reverted SQLite database at '%s' to version %d.\n", + config.Keys.DB, repository.Version-1) } if flagForceDB { - err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB) + err := repository.ForceDB(config.Keys.DB) if err != nil { return fmt.Errorf("forcing database to version %d: %w", repository.Version, err) } - cclog.Exitf("ForceDB Success: Forced '%s' database at location '%s' to version %d.\n", - config.Keys.DBDriver, config.Keys.DB, repository.Version) + cclog.Exitf("ForceDB Success: Forced SQLite database at '%s' to version %d.\n", + config.Keys.DB, repository.Version) } return nil diff --git a/configs/config-mariadb.json b/configs/config-mariadb.json deleted file mode 100644 index 38bb8a93..00000000 --- a/configs/config-mariadb.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "addr": "127.0.0.1:8080", - "short-running-jobs-duration": 300, - "archive": { - "kind": "file", - "path": "./var/job-archive" - }, - "jwts": { - "max-age": "2000h" - }, - "db-driver": "mysql", - "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit", - "enable-resampling": { - "trigger": 30, - "resolutions": [600, 300, 120, 60] - }, - "emission-constant": 317, - "clusters": [ - { - "name": "fritz", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - }, - { - "name": "alex", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - } - ] -} diff --git a/go.mod b/go.mod index df8e1fb9..eb061de7 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/expr-lang/expr v1.17.6 github.com/go-co-op/gocron/v2 v2.18.2 github.com/go-ldap/ldap/v3 v3.4.12 - github.com/go-sql-driver/mysql v1.9.3 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/gops v0.3.28 @@ -48,7 +47,6 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect diff --git a/go.sum b/go.sum index 711c5551..fd4980da 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/gqlgen v0.17.84 h1:iVMdiStgUVx/BFkMb0J5GAXlqfqtQ7bqMCYK6v52kQ0= github.com/99designs/gqlgen v0.17.84/go.mod h1:qjoUqzTeiejdo+bwUg8unqSpeYG42XrcrQboGIezmFA= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/ClusterCockpit/cc-lib v1.0.2 h1:ZWn3oZkXgxrr3zSigBdlOOfayZ4Om4xL20DhmritPPg= @@ -12,8 +10,6 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw= github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= @@ -70,10 +66,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow= github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= @@ -85,16 +77,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= -github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec= github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -113,10 +95,6 @@ github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZR github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.22.3 h1:dKMwfV4fmt6Ah90zloTbUKWMD+0he+12XYAsPotrkn8= github.com/go-openapi/jsonpointer v0.22.3/go.mod h1:0lBbqeRsQ5lIanv3LHZBrmRGHLHcQoOXQnf88fHlGWo= github.com/go-openapi/jsonreference v0.21.3 h1:96Dn+MRPa0nYAR8DR1E03SblB5FJvh7W6krPI0Z7qMc= @@ -145,15 +123,12 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8U github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= @@ -241,17 +216,11 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= @@ -265,13 +234,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -323,16 +286,6 @@ github.com/vektah/gqlparser/v2 v2.5.31/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6O github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= diff --git a/init/clustercockpit.service b/init/clustercockpit.service index 0a9448de..b4ed8bfa 100644 --- a/init/clustercockpit.service +++ b/init/clustercockpit.service @@ -3,7 +3,7 @@ Description=ClusterCockpit Web Server Documentation=https://github.com/ClusterCockpit/cc-backend Wants=network-online.target After=network-online.target -After=mariadb.service mysql.service +# Database is file-based SQLite - no service dependency required [Service] WorkingDirectory=/opt/monitoring/cc-backend diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 70b0f0aa..d311767c 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -141,7 +141,7 @@ func setup(t *testing.T) *api.RestAPI { } dbfilepath := filepath.Join(tmpdir, "test.db") - err := repository.MigrateDB("sqlite3", dbfilepath) + err := repository.MigrateDB(dbfilepath) if err != nil { t.Fatal(err) } diff --git a/internal/config/config.go b/internal/config/config.go index 25ca27eb..b7b8ed06 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,10 +37,10 @@ type ProgramConfig struct { EmbedStaticFiles bool `json:"embed-static-files"` StaticFiles string `json:"static-files"` - // 'sqlite3' or 'mysql' (mysql will work for mariadb as well) + // Database driver - only 'sqlite3' is supported DBDriver string `json:"db-driver"` - // For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!). + // Path to SQLite database file DB string `json:"db"` // Keep all metric data in the metric data repositories, diff --git a/internal/config/schema.go b/internal/config/schema.go index ed1f42d8..b171f96a 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -41,7 +41,7 @@ var configSchema = ` "type": "string" }, "db": { - "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).", + "description": "Path to SQLite database file (e.g., './var/job.db')", "type": "string" }, "disable-archive": { diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 2aa007da..470f7603 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -107,7 +107,7 @@ func setup(t *testing.T) *repository.JobRepository { } dbfilepath := filepath.Join(tmpdir, "test.db") - err := repository.MigrateDB("sqlite3", dbfilepath) + err := repository.MigrateDB(dbfilepath) if err != nil { t.Fatal(err) } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 1c14c956..0f7536b7 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -55,6 +55,10 @@ func Connect(driver string, db string) { var err error var dbHandle *sqlx.DB + if driver != "sqlite3" { + cclog.Abortf("Unsupported database driver '%s'. Only 'sqlite3' is supported.\n", driver) + } + dbConnOnce.Do(func() { opts := DatabaseOptions{ URL: db, @@ -64,39 +68,31 @@ func Connect(driver string, db string) { ConnectionMaxIdleTime: repoConfig.ConnectionMaxIdleTime, } - switch driver { - case "sqlite3": - // TODO: Have separate DB handles for Writes and Reads - // Optimize SQLite connection: https://kerkour.com/sqlite-for-servers - connectionURLParams := make(url.Values) - connectionURLParams.Add("_txlock", "immediate") - connectionURLParams.Add("_journal_mode", "WAL") - connectionURLParams.Add("_busy_timeout", "5000") - connectionURLParams.Add("_synchronous", "NORMAL") - connectionURLParams.Add("_cache_size", "1000000000") - connectionURLParams.Add("_foreign_keys", "true") - opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode()) + // TODO: Have separate DB handles for Writes and Reads + // Optimize SQLite connection: https://kerkour.com/sqlite-for-servers + connectionURLParams := make(url.Values) + connectionURLParams.Add("_txlock", "immediate") + connectionURLParams.Add("_journal_mode", "WAL") + connectionURLParams.Add("_busy_timeout", "5000") + connectionURLParams.Add("_synchronous", "NORMAL") + connectionURLParams.Add("_cache_size", "1000000000") + connectionURLParams.Add("_foreign_keys", "true") + opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode()) - if cclog.Loglevel() == "debug" { - sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{})) - dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL) - } else { - dbHandle, err = sqlx.Open("sqlite3", opts.URL) - } - - err = setupSqlite(dbHandle.DB) - if err != nil { - cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error()) - } - case "mysql": - opts.URL += "?multiStatements=true" - dbHandle, err = sqlx.Open("mysql", opts.URL) - default: - cclog.Abortf("DB Connection: Unsupported database driver '%s'.\n", driver) + if cclog.Loglevel() == "debug" { + sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{})) + dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL) + } else { + dbHandle, err = sqlx.Open("sqlite3", opts.URL) } if err != nil { - cclog.Abortf("DB Connection: Could not connect to '%s' database with sqlx.Open().\nError: %s\n", driver, err.Error()) + cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error()) + } + + err = setupSqlite(dbHandle.DB) + if err != nil { + cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error()) } dbHandle.SetMaxOpenConns(opts.MaxOpenConnections) @@ -105,7 +101,7 @@ func Connect(driver string, db string) { dbHandle.SetConnMaxIdleTime(opts.ConnectionMaxIdleTime) dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver} - err = checkDBVersion(driver, dbHandle.DB) + err = checkDBVersion(dbHandle.DB) if err != nil { cclog.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error()) } diff --git a/internal/repository/job.go b/internal/repository/job.go index f23a14cf..47959379 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -14,8 +14,6 @@ // Initialize the database connection before using any repository: // // repository.Connect("sqlite3", "./var/job.db") -// // or for MySQL: -// repository.Connect("mysql", "user:password@tcp(localhost:3306)/dbname") // // # Configuration // @@ -158,52 +156,22 @@ func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) { } func (r *JobRepository) Optimize() error { - var err error - - switch r.driver { - case "sqlite3": - if _, err = r.DB.Exec(`VACUUM`); err != nil { - return err - } - case "mysql": - cclog.Info("Optimize currently not supported for mysql driver") + if _, err := r.DB.Exec(`VACUUM`); err != nil { + return err } - return nil } func (r *JobRepository) Flush() error { - var err error - - switch r.driver { - case "sqlite3": - if _, err = r.DB.Exec(`DELETE FROM jobtag`); err != nil { - return err - } - if _, err = r.DB.Exec(`DELETE FROM tag`); err != nil { - return err - } - if _, err = r.DB.Exec(`DELETE FROM job`); err != nil { - return err - } - case "mysql": - if _, err = r.DB.Exec(`SET FOREIGN_KEY_CHECKS = 0`); err != nil { - return err - } - if _, err = r.DB.Exec(`TRUNCATE TABLE jobtag`); err != nil { - return err - } - if _, err = r.DB.Exec(`TRUNCATE TABLE tag`); err != nil { - return err - } - if _, err = r.DB.Exec(`TRUNCATE TABLE job`); err != nil { - return err - } - if _, err = r.DB.Exec(`SET FOREIGN_KEY_CHECKS = 1`); err != nil { - return err - } + if _, err := r.DB.Exec(`DELETE FROM jobtag`); err != nil { + return err + } + if _, err := r.DB.Exec(`DELETE FROM tag`); err != nil { + return err + } + if _, err := r.DB.Exec(`DELETE FROM job`); err != nil { + return err } - return nil } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index dec93a94..43e913cc 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -12,7 +12,6 @@ import ( cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database/mysql" "github.com/golang-migrate/migrate/v4/database/sqlite3" "github.com/golang-migrate/migrate/v4/source/iofs" ) @@ -22,40 +21,19 @@ const Version uint = 10 //go:embed migrations/* var migrationFiles embed.FS -func checkDBVersion(backend string, db *sql.DB) error { - var m *migrate.Migrate +func checkDBVersion(db *sql.DB) error { + driver, err := sqlite3.WithInstance(db, &sqlite3.Config{}) + if err != nil { + return err + } + d, err := iofs.New(migrationFiles, "migrations/sqlite3") + if err != nil { + return err + } - switch backend { - case "sqlite3": - driver, err := sqlite3.WithInstance(db, &sqlite3.Config{}) - if err != nil { - return err - } - d, err := iofs.New(migrationFiles, "migrations/sqlite3") - if err != nil { - return err - } - - m, err = migrate.NewWithInstance("iofs", d, "sqlite3", driver) - if err != nil { - return err - } - case "mysql": - driver, err := mysql.WithInstance(db, &mysql.Config{}) - if err != nil { - return err - } - d, err := iofs.New(migrationFiles, "migrations/mysql") - if err != nil { - return err - } - - m, err = migrate.NewWithInstance("iofs", d, "mysql", driver) - if err != nil { - return err - } - default: - cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend) + m, err := migrate.NewWithInstance("iofs", d, "sqlite3", driver) + if err != nil { + return err } v, dirty, err := m.Version() @@ -80,37 +58,22 @@ func checkDBVersion(backend string, db *sql.DB) error { return nil } -func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err error) { - switch backend { - case "sqlite3": - d, err := iofs.New(migrationFiles, "migrations/sqlite3") - if err != nil { - cclog.Fatal(err) - } +func getMigrateInstance(db string) (m *migrate.Migrate, err error) { + d, err := iofs.New(migrationFiles, "migrations/sqlite3") + if err != nil { + return nil, err + } - m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db)) - if err != nil { - return m, err - } - case "mysql": - d, err := iofs.New(migrationFiles, "migrations/mysql") - if err != nil { - return m, err - } - - m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("mysql://%s?multiStatements=true", db)) - if err != nil { - return m, err - } - default: - cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend) + m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db)) + if err != nil { + return nil, err } return m, nil } -func MigrateDB(backend string, db string) error { - m, err := getMigrateInstance(backend, db) +func MigrateDB(db string) error { + m, err := getMigrateInstance(db) if err != nil { return err } @@ -144,8 +107,8 @@ func MigrateDB(backend string, db string) error { return nil } -func RevertDB(backend string, db string) error { - m, err := getMigrateInstance(backend, db) +func RevertDB(db string) error { + m, err := getMigrateInstance(db) if err != nil { return err } @@ -162,8 +125,8 @@ func RevertDB(backend string, db string) error { return nil } -func ForceDB(backend string, db string) error { - m, err := getMigrateInstance(backend, db) +func ForceDB(db string) error { + m, err := getMigrateInstance(db) if err != nil { return err } diff --git a/internal/repository/migrations/mysql/01_init-schema.down.sql b/internal/repository/migrations/mysql/01_init-schema.down.sql deleted file mode 100644 index 68da6469..00000000 --- a/internal/repository/migrations/mysql/01_init-schema.down.sql +++ /dev/null @@ -1,5 +0,0 @@ -DROP TABLE IF EXISTS job; -DROP TABLE IF EXISTS tags; -DROP TABLE IF EXISTS jobtag; -DROP TABLE IF EXISTS configuration; -DROP TABLE IF EXISTS user; diff --git a/internal/repository/migrations/mysql/01_init-schema.up.sql b/internal/repository/migrations/mysql/01_init-schema.up.sql deleted file mode 100644 index 3a6930cd..00000000 --- a/internal/repository/migrations/mysql/01_init-schema.up.sql +++ /dev/null @@ -1,66 +0,0 @@ -CREATE TABLE IF NOT EXISTS job ( - id INTEGER AUTO_INCREMENT PRIMARY KEY , - job_id BIGINT NOT NULL, - cluster VARCHAR(255) NOT NULL, - subcluster VARCHAR(255) NOT NULL, - start_time BIGINT NOT NULL, -- Unix timestamp - - user VARCHAR(255) NOT NULL, - project VARCHAR(255) NOT NULL, - `partition` VARCHAR(255) NOT NULL, - array_job_id BIGINT NOT NULL, - duration INT NOT NULL DEFAULT 0, - walltime INT NOT NULL DEFAULT 0, - job_state VARCHAR(255) NOT NULL - CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', - 'stopped', 'timeout', 'preempted', 'out_of_memory')), - meta_data TEXT, -- JSON - resources TEXT NOT NULL, -- JSON - - num_nodes INT NOT NULL, - num_hwthreads INT NOT NULL, - num_acc INT NOT NULL, - smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )), - exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)), - monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)), - - mem_used_max REAL NOT NULL DEFAULT 0.0, - flops_any_avg REAL NOT NULL DEFAULT 0.0, - mem_bw_avg REAL NOT NULL DEFAULT 0.0, - load_avg REAL NOT NULL DEFAULT 0.0, - net_bw_avg REAL NOT NULL DEFAULT 0.0, - net_data_vol_total REAL NOT NULL DEFAULT 0.0, - file_bw_avg REAL NOT NULL DEFAULT 0.0, - file_data_vol_total REAL NOT NULL DEFAULT 0.0, - UNIQUE (job_id, cluster, start_time) - ); - -CREATE TABLE IF NOT EXISTS tag ( - id INTEGER PRIMARY KEY, - tag_type VARCHAR(255) NOT NULL, - tag_name VARCHAR(255) NOT NULL, - UNIQUE (tag_type, tag_name)); - -CREATE TABLE IF NOT EXISTS jobtag ( - job_id INTEGER, - tag_id INTEGER, - PRIMARY KEY (job_id, tag_id), - FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE CASCADE, - FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE); - -CREATE TABLE IF NOT EXISTS user ( - username varchar(255) PRIMARY KEY NOT NULL, - password varchar(255) DEFAULT NULL, - ldap tinyint NOT NULL DEFAULT 0, /* col called "ldap" for historic reasons, fills the "AuthSource" */ - name varchar(255) DEFAULT NULL, - roles varchar(255) NOT NULL DEFAULT "[]", - email varchar(255) DEFAULT NULL); - -CREATE TABLE IF NOT EXISTS configuration ( - username varchar(255), - confkey varchar(255), - value varchar(255), - PRIMARY KEY (username, confkey), - FOREIGN KEY (username) REFERENCES user (username) ON DELETE CASCADE ON UPDATE NO ACTION); - - diff --git a/internal/repository/migrations/mysql/02_add-index.down.sql b/internal/repository/migrations/mysql/02_add-index.down.sql deleted file mode 100644 index 1392c45c..00000000 --- a/internal/repository/migrations/mysql/02_add-index.down.sql +++ /dev/null @@ -1,8 +0,0 @@ -DROP INDEX IF EXISTS job_stats; -DROP INDEX IF EXISTS job_by_user; -DROP INDEX IF EXISTS job_by_starttime; -DROP INDEX IF EXISTS job_by_job_id; -DROP INDEX IF EXISTS job_list; -DROP INDEX IF EXISTS job_list_user; -DROP INDEX IF EXISTS job_list_users; -DROP INDEX IF EXISTS job_list_users_start; diff --git a/internal/repository/migrations/mysql/02_add-index.up.sql b/internal/repository/migrations/mysql/02_add-index.up.sql deleted file mode 100644 index 2524bd93..00000000 --- a/internal/repository/migrations/mysql/02_add-index.up.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user); -CREATE INDEX IF NOT EXISTS job_by_user ON job (user); -CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time); -CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id); -CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state); -CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state); -CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state); -CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state); diff --git a/internal/repository/migrations/mysql/03_add-userprojects.down.sql b/internal/repository/migrations/mysql/03_add-userprojects.down.sql deleted file mode 100644 index bbf1e649..00000000 --- a/internal/repository/migrations/mysql/03_add-userprojects.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user DROP COLUMN projects; diff --git a/internal/repository/migrations/mysql/03_add-userprojects.up.sql b/internal/repository/migrations/mysql/03_add-userprojects.up.sql deleted file mode 100644 index d0f19c21..00000000 --- a/internal/repository/migrations/mysql/03_add-userprojects.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user ADD COLUMN projects varchar(255) NOT NULL DEFAULT "[]"; diff --git a/internal/repository/migrations/mysql/04_alter-table-job.down.sql b/internal/repository/migrations/mysql/04_alter-table-job.down.sql deleted file mode 100644 index ebc74549..00000000 --- a/internal/repository/migrations/mysql/04_alter-table-job.down.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE job - MODIFY `partition` VARCHAR(255) NOT NULL, - MODIFY array_job_id BIGINT NOT NULL, - MODIFY num_hwthreads INT NOT NULL, - MODIFY num_acc INT NOT NULL; diff --git a/internal/repository/migrations/mysql/04_alter-table-job.up.sql b/internal/repository/migrations/mysql/04_alter-table-job.up.sql deleted file mode 100644 index 9fe76208..00000000 --- a/internal/repository/migrations/mysql/04_alter-table-job.up.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE job - MODIFY `partition` VARCHAR(255), - MODIFY array_job_id BIGINT, - MODIFY num_hwthreads INT, - MODIFY num_acc INT; diff --git a/internal/repository/migrations/mysql/05_extend-tags.down.sql b/internal/repository/migrations/mysql/05_extend-tags.down.sql deleted file mode 100644 index 925c9f8f..00000000 --- a/internal/repository/migrations/mysql/05_extend-tags.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE tag DROP COLUMN insert_time; -ALTER TABLE jobtag DROP COLUMN insert_time; diff --git a/internal/repository/migrations/mysql/05_extend-tags.up.sql b/internal/repository/migrations/mysql/05_extend-tags.up.sql deleted file mode 100644 index 4577564a..00000000 --- a/internal/repository/migrations/mysql/05_extend-tags.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE tag ADD COLUMN insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP; -ALTER TABLE jobtag ADD COLUMN insert_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP; diff --git a/internal/repository/migrations/mysql/06_change-config.down.sql b/internal/repository/migrations/mysql/06_change-config.down.sql deleted file mode 100644 index 0651790c..00000000 --- a/internal/repository/migrations/mysql/06_change-config.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE configuration MODIFY value VARCHAR(255); diff --git a/internal/repository/migrations/mysql/06_change-config.up.sql b/internal/repository/migrations/mysql/06_change-config.up.sql deleted file mode 100644 index e35ff195..00000000 --- a/internal/repository/migrations/mysql/06_change-config.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE configuration MODIFY value TEXT; diff --git a/internal/repository/migrations/mysql/07_fix-tag-id.down.sql b/internal/repository/migrations/mysql/07_fix-tag-id.down.sql deleted file mode 100644 index 9f9959ac..00000000 --- a/internal/repository/migrations/mysql/07_fix-tag-id.down.sql +++ /dev/null @@ -1,3 +0,0 @@ -SET FOREIGN_KEY_CHECKS = 0; -ALTER TABLE tag MODIFY id INTEGER; -SET FOREIGN_KEY_CHECKS = 1; diff --git a/internal/repository/migrations/mysql/07_fix-tag-id.up.sql b/internal/repository/migrations/mysql/07_fix-tag-id.up.sql deleted file mode 100644 index 1abc4b35..00000000 --- a/internal/repository/migrations/mysql/07_fix-tag-id.up.sql +++ /dev/null @@ -1,3 +0,0 @@ -SET FOREIGN_KEY_CHECKS = 0; -ALTER TABLE tag MODIFY id INTEGER AUTO_INCREMENT; -SET FOREIGN_KEY_CHECKS = 1; diff --git a/internal/repository/migrations/mysql/08_add-footprint.down.sql b/internal/repository/migrations/mysql/08_add-footprint.down.sql deleted file mode 100644 index 57f2145c..00000000 --- a/internal/repository/migrations/mysql/08_add-footprint.down.sql +++ /dev/null @@ -1,83 +0,0 @@ -ALTER TABLE job DROP energy; -ALTER TABLE job DROP energy_footprint; -ALTER TABLE job ADD COLUMN flops_any_avg; -ALTER TABLE job ADD COLUMN mem_bw_avg; -ALTER TABLE job ADD COLUMN mem_used_max; -ALTER TABLE job ADD COLUMN load_avg; -ALTER TABLE job ADD COLUMN net_bw_avg; -ALTER TABLE job ADD COLUMN net_data_vol_total; -ALTER TABLE job ADD COLUMN file_bw_avg; -ALTER TABLE job ADD COLUMN file_data_vol_total; - -UPDATE job SET flops_any_avg = json_extract(footprint, '$.flops_any_avg'); -UPDATE job SET mem_bw_avg = json_extract(footprint, '$.mem_bw_avg'); -UPDATE job SET mem_used_max = json_extract(footprint, '$.mem_used_max'); -UPDATE job SET load_avg = json_extract(footprint, '$.cpu_load_avg'); -UPDATE job SET net_bw_avg = json_extract(footprint, '$.net_bw_avg'); -UPDATE job SET net_data_vol_total = json_extract(footprint, '$.net_data_vol_total'); -UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg'); -UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total'); - -ALTER TABLE job DROP footprint; --- Do not use reserved keywords anymore -RENAME TABLE hpc_user TO `user`; -ALTER TABLE job RENAME COLUMN hpc_user TO `user`; -ALTER TABLE job RENAME COLUMN cluster_partition TO `partition`; - -DROP INDEX IF EXISTS jobs_cluster; -DROP INDEX IF EXISTS jobs_cluster_user; -DROP INDEX IF EXISTS jobs_cluster_project; -DROP INDEX IF EXISTS jobs_cluster_subcluster; -DROP INDEX IF EXISTS jobs_cluster_starttime; -DROP INDEX IF EXISTS jobs_cluster_duration; -DROP INDEX IF EXISTS jobs_cluster_numnodes; - -DROP INDEX IF EXISTS jobs_cluster_partition; -DROP INDEX IF EXISTS jobs_cluster_partition_starttime; -DROP INDEX IF EXISTS jobs_cluster_partition_duration; -DROP INDEX IF EXISTS jobs_cluster_partition_numnodes; - -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate; -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_user; -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_project; -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_starttime; -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_duration; -DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numnodes; - -DROP INDEX IF EXISTS jobs_cluster_jobstate; -DROP INDEX IF EXISTS jobs_cluster_jobstate_user; -DROP INDEX IF EXISTS jobs_cluster_jobstate_project; - -DROP INDEX IF EXISTS jobs_cluster_jobstate_starttime; -DROP INDEX IF EXISTS jobs_cluster_jobstate_duration; -DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes; - -DROP INDEX IF EXISTS jobs_user; -DROP INDEX IF EXISTS jobs_user_starttime; -DROP INDEX IF EXISTS jobs_user_duration; -DROP INDEX IF EXISTS jobs_user_numnodes; - -DROP INDEX IF EXISTS jobs_project; -DROP INDEX IF EXISTS jobs_project_user; -DROP INDEX IF EXISTS jobs_project_starttime; -DROP INDEX IF EXISTS jobs_project_duration; -DROP INDEX IF EXISTS jobs_project_numnodes; - -DROP INDEX IF EXISTS jobs_jobstate; -DROP INDEX IF EXISTS jobs_jobstate_user; -DROP INDEX IF EXISTS jobs_jobstate_project; -DROP INDEX IF EXISTS jobs_jobstate_starttime; -DROP INDEX IF EXISTS jobs_jobstate_duration; -DROP INDEX IF EXISTS jobs_jobstate_numnodes; - -DROP INDEX IF EXISTS jobs_arrayjobid_starttime; -DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime; - -DROP INDEX IF EXISTS jobs_starttime; -DROP INDEX IF EXISTS jobs_duration; -DROP INDEX IF EXISTS jobs_numnodes; - -DROP INDEX IF EXISTS jobs_duration_starttime; -DROP INDEX IF EXISTS jobs_numnodes_starttime; -DROP INDEX IF EXISTS jobs_numacc_starttime; -DROP INDEX IF EXISTS jobs_energy_starttime; diff --git a/internal/repository/migrations/mysql/08_add-footprint.up.sql b/internal/repository/migrations/mysql/08_add-footprint.up.sql deleted file mode 100644 index 207ccf9e..00000000 --- a/internal/repository/migrations/mysql/08_add-footprint.up.sql +++ /dev/null @@ -1,123 +0,0 @@ -DROP INDEX IF EXISTS job_stats ON job; -DROP INDEX IF EXISTS job_by_user ON job; -DROP INDEX IF EXISTS job_by_starttime ON job; -DROP INDEX IF EXISTS job_by_job_id ON job; -DROP INDEX IF EXISTS job_list ON job; -DROP INDEX IF EXISTS job_list_user ON job; -DROP INDEX IF EXISTS job_list_users ON job; -DROP INDEX IF EXISTS job_list_users_start ON job; - -ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; -ALTER TABLE job ADD COLUMN energy_footprint JSON; - -ALTER TABLE job ADD COLUMN footprint JSON; -ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global'; - --- Do not use reserved keywords anymore -RENAME TABLE `user` TO hpc_user; -ALTER TABLE job RENAME COLUMN `user` TO hpc_user; -ALTER TABLE job RENAME COLUMN `partition` TO cluster_partition; - -ALTER TABLE job MODIFY COLUMN cluster VARCHAR(50); -ALTER TABLE job MODIFY COLUMN hpc_user VARCHAR(50); -ALTER TABLE job MODIFY COLUMN subcluster VARCHAR(50); -ALTER TABLE job MODIFY COLUMN project VARCHAR(50); -ALTER TABLE job MODIFY COLUMN cluster_partition VARCHAR(50); -ALTER TABLE job MODIFY COLUMN job_state VARCHAR(25); - -UPDATE job SET footprint = '{"flops_any_avg": 0.0}'; -UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg); -UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg); -UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max); -UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg); -UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0; -UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0; -UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0; -UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0; - -ALTER TABLE job DROP flops_any_avg; -ALTER TABLE job DROP mem_bw_avg; -ALTER TABLE job DROP mem_used_max; -ALTER TABLE job DROP load_avg; -ALTER TABLE job DROP net_bw_avg; -ALTER TABLE job DROP net_data_vol_total; -ALTER TABLE job DROP file_bw_avg; -ALTER TABLE job DROP file_data_vol_total; - --- Indices for: Single filters, combined filters, sorting, sorting with filters --- Cluster Filter -CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); -CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); -CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); -CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); --- Cluster Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes); - --- Cluster+Partition Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition); --- Cluster+Partition Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); - --- Cluster+Partition+Jobstate Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project); --- Cluster+Partition+Jobstate Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes); - --- Cluster+JobState Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); --- Cluster+JobState Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); - --- User Filter -CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user); --- User Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); -CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); -CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); - --- Project Filter -CREATE INDEX IF NOT EXISTS jobs_project ON job (project); -CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); --- Project Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); -CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration); -CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes); - --- JobState Filter -CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); -CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); -CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); -CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); --- JobState Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration); -CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes); - --- ArrayJob Filter -CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); - --- Sorting without active filters -CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); -CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); -CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes); - --- Single filters with default starttime sorting -CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time); -CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time); -CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time); -CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time); - --- Optimize DB index usage diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index b42e09b8..466f51ee 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -130,7 +130,7 @@ func nodeTestSetup(t *testing.T) { } dbfilepath := filepath.Join(tmpdir, "test.db") - err := MigrateDB("sqlite3", dbfilepath) + err := MigrateDB(dbfilepath) if err != nil { t.Fatal(err) } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 1346e4da..e3dec7fc 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -149,7 +149,7 @@ func setup(tb testing.TB) *JobRepository { tb.Helper() cclog.Init("warn", true) dbfile := "testdata/job.db" - err := MigrateDB("sqlite3", dbfile) + err := MigrateDB(dbfile) noErr(tb, err) Connect("sqlite3", dbfile) return GetJobRepository() diff --git a/internal/repository/stats.go b/internal/repository/stats.go index ba0d09f5..c92f5193 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -73,9 +73,6 @@ func (r *JobRepository) buildStatsQuery( col string, ) sq.SelectBuilder { var query sq.SelectBuilder - castType := r.getCastType() - - // fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as %s) as value`, time.Now().Unix(), castType) if col != "" { // Scan columns: id, name, totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours @@ -84,26 +81,26 @@ func (r *JobRepository) buildStatsQuery( "name", "COUNT(job.id) as totalJobs", "COUNT(DISTINCT job.hpc_user) AS totalUsers", - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s) as totalWalltime`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s) as totalNodes`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s) as totalNodeHours`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as %s) as totalCores`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as %s) as totalCoreHours`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_acc) as %s) as totalAccs`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s) as totalAccHours`, time.Now().Unix(), castType), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as int) as totalWalltime`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_nodes) as int) as totalNodes`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as int) as totalNodeHours`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as int) as totalCores`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as int) as totalCoreHours`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_acc) as int) as totalAccs`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as int) as totalAccHours`, time.Now().Unix()), ).From("job").LeftJoin("hpc_user ON hpc_user.username = job.hpc_user").GroupBy(col) } else { // Scan columns: totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours query = sq.Select( "COUNT(job.id) as totalJobs", "COUNT(DISTINCT job.hpc_user) AS totalUsers", - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s)`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s)`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s)`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as %s)`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as %s)`, time.Now().Unix(), castType), - fmt.Sprintf(`CAST(SUM(job.num_acc) as %s)`, castType), - fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s)`, time.Now().Unix(), castType), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as int)`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_nodes) as int)`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as int)`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as int)`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as int)`, time.Now().Unix()), + fmt.Sprintf(`CAST(SUM(job.num_acc) as int)`), + fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as int)`, time.Now().Unix()), ).From("job") } @@ -114,21 +111,6 @@ func (r *JobRepository) buildStatsQuery( return query } -func (r *JobRepository) getCastType() string { - var castType string - - switch r.driver { - case "sqlite3": - castType = "int" - case "mysql": - castType = "unsigned" - default: - castType = "" - } - - return castType -} - func (r *JobRepository) JobsStatsGrouped( ctx context.Context, filter []*model.JobFilter, @@ -477,10 +459,9 @@ func (r *JobRepository) AddHistograms( targetBinSize = 3600 } - castType := r.getCastType() var err error // Return X-Values always as seconds, will be formatted into minutes and hours in frontend - value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), targetBinSize, castType) + value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize) stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, targetBinSize, &targetBinCount) if err != nil { cclog.Warn("Error while loading job statistics histogram: job duration") diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index 0d6dc374..b6f68430 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -42,7 +42,7 @@ func setupUserTest(t *testing.T) *UserCfgRepo { cclog.Init("info", true) dbfilepath := "testdata/job.db" - err := MigrateDB("sqlite3", dbfilepath) + err := MigrateDB(dbfilepath) if err != nil { t.Fatal(err) } diff --git a/internal/tagger/detectApp_test.go b/internal/tagger/detectApp_test.go index f9fc91d0..7145d04f 100644 --- a/internal/tagger/detectApp_test.go +++ b/internal/tagger/detectApp_test.go @@ -15,7 +15,7 @@ func setup(tb testing.TB) *repository.JobRepository { tb.Helper() cclog.Init("warn", true) dbfile := "../repository/testdata/job.db" - err := repository.MigrateDB("sqlite3", dbfile) + err := repository.MigrateDB(dbfile) noErr(tb, err) repository.Connect("sqlite3", dbfile) return repository.GetJobRepository() From e37591ce6d1386807315f6a9ca594aff81aa0bb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:02:41 +0000 Subject: [PATCH 045/341] Bump rollup from 4.53.3 to 4.54.0 in /web/frontend Bumps [rollup](https://github.com/rollup/rollup) from 4.53.3 to 4.54.0. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/master/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v4.53.3...v4.54.0) --- updated-dependencies: - dependency-name: rollup dependency-version: 4.54.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- web/frontend/package-lock.json | 189 +++++++++++++++++---------------- web/frontend/package.json | 2 +- 2 files changed, 98 insertions(+), 93 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 4c7e4bf5..c9769a59 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -24,7 +24,7 @@ "@rollup/plugin-node-resolve": "^16.0.1", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.53.3", + "rollup": "^4.54.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", "svelte": "^5.44.0" @@ -244,9 +244,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.53.3.tgz", - "integrity": "sha512-mRSi+4cBjrRLoaal2PnqH82Wqyb+d3HsPUN/W+WslCXsZsyHa9ZeQQX/pQsZaVIWDkPcpV6jJ+3KLbTbgnwv8w==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.54.0.tgz", + "integrity": "sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng==", "cpu": [ "arm" ], @@ -258,9 +258,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.53.3.tgz", - "integrity": "sha512-CbDGaMpdE9sh7sCmTrTUyllhrg65t6SwhjlMJsLr+J8YjFuPmCEjbBSx4Z/e4SmDyH3aB5hGaJUP2ltV/vcs4w==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.54.0.tgz", + "integrity": "sha512-Skx39Uv+u7H224Af+bDgNinitlmHyQX1K/atIA32JP3JQw6hVODX5tkbi2zof/E69M1qH2UoN3Xdxgs90mmNYw==", "cpu": [ "arm64" ], @@ -272,9 +272,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.53.3.tgz", - "integrity": "sha512-Nr7SlQeqIBpOV6BHHGZgYBuSdanCXuw09hon14MGOLGmXAFYjx1wNvquVPmpZnl0tLjg25dEdr4IQ6GgyToCUA==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.54.0.tgz", + "integrity": "sha512-k43D4qta/+6Fq+nCDhhv9yP2HdeKeP56QrUUTW7E6PhZP1US6NDqpJj4MY0jBHlJivVJD5P8NxrjuobZBJTCRw==", "cpu": [ "arm64" ], @@ -286,9 +286,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.53.3.tgz", - "integrity": "sha512-DZ8N4CSNfl965CmPktJ8oBnfYr3F8dTTNBQkRlffnUarJ2ohudQD17sZBa097J8xhQ26AwhHJ5mvUyQW8ddTsQ==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.54.0.tgz", + "integrity": "sha512-cOo7biqwkpawslEfox5Vs8/qj83M/aZCSSNIWpVzfU2CYHa2G3P1UN5WF01RdTHSgCkri7XOlTdtk17BezlV3A==", "cpu": [ "x64" ], @@ -300,9 +300,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.53.3.tgz", - "integrity": "sha512-yMTrCrK92aGyi7GuDNtGn2sNW+Gdb4vErx4t3Gv/Tr+1zRb8ax4z8GWVRfr3Jw8zJWvpGHNpss3vVlbF58DZ4w==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.54.0.tgz", + "integrity": "sha512-miSvuFkmvFbgJ1BevMa4CPCFt5MPGw094knM64W9I0giUIMMmRYcGW/JWZDriaw/k1kOBtsWh1z6nIFV1vPNtA==", "cpu": [ "arm64" ], @@ -314,9 +314,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.53.3.tgz", - "integrity": "sha512-lMfF8X7QhdQzseM6XaX0vbno2m3hlyZFhwcndRMw8fbAGUGL3WFMBdK0hbUBIUYcEcMhVLr1SIamDeuLBnXS+Q==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.54.0.tgz", + "integrity": "sha512-KGXIs55+b/ZfZsq9aR026tmr/+7tq6VG6MsnrvF4H8VhwflTIuYh+LFUlIsRdQSgrgmtM3fVATzEAj4hBQlaqQ==", "cpu": [ "x64" ], @@ -328,9 +328,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.53.3.tgz", - "integrity": "sha512-k9oD15soC/Ln6d2Wv/JOFPzZXIAIFLp6B+i14KhxAfnq76ajt0EhYc5YPeX6W1xJkAdItcVT+JhKl1QZh44/qw==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.54.0.tgz", + "integrity": "sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==", "cpu": [ "arm" ], @@ -342,9 +342,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.53.3.tgz", - "integrity": "sha512-vTNlKq+N6CK/8UktsrFuc+/7NlEYVxgaEgRXVUVK258Z5ymho29skzW1sutgYjqNnquGwVUObAaxae8rZ6YMhg==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.54.0.tgz", + "integrity": "sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==", "cpu": [ "arm" ], @@ -356,9 +356,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.53.3.tgz", - "integrity": "sha512-RGrFLWgMhSxRs/EWJMIFM1O5Mzuz3Xy3/mnxJp/5cVhZ2XoCAxJnmNsEyeMJtpK+wu0FJFWz+QF4mjCA7AUQ3w==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.54.0.tgz", + "integrity": "sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==", "cpu": [ "arm64" ], @@ -370,9 +370,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.53.3.tgz", - "integrity": "sha512-kASyvfBEWYPEwe0Qv4nfu6pNkITLTb32p4yTgzFCocHnJLAHs+9LjUu9ONIhvfT/5lv4YS5muBHyuV84epBo/A==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.54.0.tgz", + "integrity": "sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==", "cpu": [ "arm64" ], @@ -384,9 +384,9 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.53.3.tgz", - "integrity": "sha512-JiuKcp2teLJwQ7vkJ95EwESWkNRFJD7TQgYmCnrPtlu50b4XvT5MOmurWNrCj3IFdyjBQ5p9vnrX4JM6I8OE7g==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.54.0.tgz", + "integrity": "sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==", "cpu": [ "loong64" ], @@ -398,9 +398,9 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.53.3.tgz", - "integrity": "sha512-EoGSa8nd6d3T7zLuqdojxC20oBfNT8nexBbB/rkxgKj5T5vhpAQKKnD+h3UkoMuTyXkP5jTjK/ccNRmQrPNDuw==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.54.0.tgz", + "integrity": "sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==", "cpu": [ "ppc64" ], @@ -412,9 +412,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.53.3.tgz", - "integrity": "sha512-4s+Wped2IHXHPnAEbIB0YWBv7SDohqxobiiPA1FIWZpX+w9o2i4LezzH/NkFUl8LRci/8udci6cLq+jJQlh+0g==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.54.0.tgz", + "integrity": "sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==", "cpu": [ "riscv64" ], @@ -426,9 +426,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.53.3.tgz", - "integrity": "sha512-68k2g7+0vs2u9CxDt5ktXTngsxOQkSEV/xBbwlqYcUrAVh6P9EgMZvFsnHy4SEiUl46Xf0IObWVbMvPrr2gw8A==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.54.0.tgz", + "integrity": "sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==", "cpu": [ "riscv64" ], @@ -440,9 +440,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.53.3.tgz", - "integrity": "sha512-VYsFMpULAz87ZW6BVYw3I6sWesGpsP9OPcyKe8ofdg9LHxSbRMd7zrVrr5xi/3kMZtpWL/wC+UIJWJYVX5uTKg==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.54.0.tgz", + "integrity": "sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==", "cpu": [ "s390x" ], @@ -454,9 +454,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.53.3.tgz", - "integrity": "sha512-3EhFi1FU6YL8HTUJZ51imGJWEX//ajQPfqWLI3BQq4TlvHy4X0MOr5q3D2Zof/ka0d5FNdPwZXm3Yyib/UEd+w==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.54.0.tgz", + "integrity": "sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==", "cpu": [ "x64" ], @@ -468,9 +468,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.53.3.tgz", - "integrity": "sha512-eoROhjcc6HbZCJr+tvVT8X4fW3/5g/WkGvvmwz/88sDtSJzO7r/blvoBDgISDiCjDRZmHpwud7h+6Q9JxFwq1Q==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.54.0.tgz", + "integrity": "sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==", "cpu": [ "x64" ], @@ -482,9 +482,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.53.3.tgz", - "integrity": "sha512-OueLAWgrNSPGAdUdIjSWXw+u/02BRTcnfw9PN41D2vq/JSEPnJnVuBgw18VkN8wcd4fjUs+jFHVM4t9+kBSNLw==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.54.0.tgz", + "integrity": "sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==", "cpu": [ "arm64" ], @@ -496,9 +496,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.53.3.tgz", - "integrity": "sha512-GOFuKpsxR/whszbF/bzydebLiXIHSgsEUp6M0JI8dWvi+fFa1TD6YQa4aSZHtpmh2/uAlj/Dy+nmby3TJ3pkTw==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.54.0.tgz", + "integrity": "sha512-c2V0W1bsKIKfbLMBu/WGBz6Yci8nJ/ZJdheE0EwB73N3MvHYKiKGs3mVilX4Gs70eGeDaMqEob25Tw2Gb9Nqyw==", "cpu": [ "arm64" ], @@ -510,9 +510,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.53.3.tgz", - "integrity": "sha512-iah+THLcBJdpfZ1TstDFbKNznlzoxa8fmnFYK4V67HvmuNYkVdAywJSoteUszvBQ9/HqN2+9AZghbajMsFT+oA==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.54.0.tgz", + "integrity": "sha512-woEHgqQqDCkAzrDhvDipnSirm5vxUXtSKDYTVpZG3nUdW/VVB5VdCYA2iReSj/u3yCZzXID4kuKG7OynPnB3WQ==", "cpu": [ "ia32" ], @@ -524,9 +524,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.53.3.tgz", - "integrity": "sha512-J9QDiOIZlZLdcot5NXEepDkstocktoVjkaKUtqzgzpt2yWjGlbYiKyp05rWwk4nypbYUNoFAztEgixoLaSETkg==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.54.0.tgz", + "integrity": "sha512-dzAc53LOuFvHwbCEOS0rPbXp6SIhAf2txMP5p6mGyOXXw5mWY8NGGbPMPrs4P1WItkfApDathBj/NzMLUZ9rtQ==", "cpu": [ "x64" ], @@ -538,9 +538,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.53.3.tgz", - "integrity": "sha512-UhTd8u31dXadv0MopwGgNOBpUVROFKWVQgAg5N1ESyCz8AuBcMqm4AuTjrwgQKGDfoFuz02EuMRHQIw/frmYKQ==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.54.0.tgz", + "integrity": "sha512-hYT5d3YNdSh3mbCU1gwQyPgQd3T2ne0A3KG8KSBdav5TiBg6eInVmV+TeR5uHufiIgSFg0XsOWGW5/RhNcSvPg==", "cpu": [ "x64" ], @@ -621,6 +621,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -821,6 +822,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.12.0.tgz", "integrity": "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -927,6 +929,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -976,11 +979,12 @@ } }, "node_modules/rollup": { - "version": "4.53.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz", - "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==", + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.54.0.tgz", + "integrity": "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -992,28 +996,28 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.53.3", - "@rollup/rollup-android-arm64": "4.53.3", - "@rollup/rollup-darwin-arm64": "4.53.3", - "@rollup/rollup-darwin-x64": "4.53.3", - "@rollup/rollup-freebsd-arm64": "4.53.3", - "@rollup/rollup-freebsd-x64": "4.53.3", - "@rollup/rollup-linux-arm-gnueabihf": "4.53.3", - "@rollup/rollup-linux-arm-musleabihf": "4.53.3", - "@rollup/rollup-linux-arm64-gnu": "4.53.3", - "@rollup/rollup-linux-arm64-musl": "4.53.3", - "@rollup/rollup-linux-loong64-gnu": "4.53.3", - "@rollup/rollup-linux-ppc64-gnu": "4.53.3", - "@rollup/rollup-linux-riscv64-gnu": "4.53.3", - "@rollup/rollup-linux-riscv64-musl": "4.53.3", - "@rollup/rollup-linux-s390x-gnu": "4.53.3", - "@rollup/rollup-linux-x64-gnu": "4.53.3", - "@rollup/rollup-linux-x64-musl": "4.53.3", - "@rollup/rollup-openharmony-arm64": "4.53.3", - "@rollup/rollup-win32-arm64-msvc": "4.53.3", - "@rollup/rollup-win32-ia32-msvc": "4.53.3", - "@rollup/rollup-win32-x64-gnu": "4.53.3", - "@rollup/rollup-win32-x64-msvc": "4.53.3", + "@rollup/rollup-android-arm-eabi": "4.54.0", + "@rollup/rollup-android-arm64": "4.54.0", + "@rollup/rollup-darwin-arm64": "4.54.0", + "@rollup/rollup-darwin-x64": "4.54.0", + "@rollup/rollup-freebsd-arm64": "4.54.0", + "@rollup/rollup-freebsd-x64": "4.54.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.54.0", + "@rollup/rollup-linux-arm-musleabihf": "4.54.0", + "@rollup/rollup-linux-arm64-gnu": "4.54.0", + "@rollup/rollup-linux-arm64-musl": "4.54.0", + "@rollup/rollup-linux-loong64-gnu": "4.54.0", + "@rollup/rollup-linux-ppc64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-musl": "4.54.0", + "@rollup/rollup-linux-s390x-gnu": "4.54.0", + "@rollup/rollup-linux-x64-gnu": "4.54.0", + "@rollup/rollup-linux-x64-musl": "4.54.0", + "@rollup/rollup-openharmony-arm64": "4.54.0", + "@rollup/rollup-win32-arm64-msvc": "4.54.0", + "@rollup/rollup-win32-ia32-msvc": "4.54.0", + "@rollup/rollup-win32-x64-gnu": "4.54.0", + "@rollup/rollup-win32-x64-msvc": "4.54.0", "fsevents": "~2.3.2" } }, @@ -1161,6 +1165,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.44.0.tgz", "integrity": "sha512-R7387No2zEGw4CtYtI2rgsui6BqjFARzoZFGLiLN5OPla0Pq4Ra2WwcP/zBomP3MYalhSNvF1fzDMuU0P0zPJw==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", diff --git a/web/frontend/package.json b/web/frontend/package.json index 3f7434f7..7a759c71 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -11,7 +11,7 @@ "@rollup/plugin-node-resolve": "^16.0.1", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.53.3", + "rollup": "^4.54.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", "svelte": "^5.44.0" From fe78f2f433136a9fb48de5c330a3daac23af4d2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:03:31 +0000 Subject: [PATCH 046/341] Bump github.com/coreos/go-oidc/v3 from 3.16.0 to 3.17.0 Bumps [github.com/coreos/go-oidc/v3](https://github.com/coreos/go-oidc) from 3.16.0 to 3.17.0. - [Release notes](https://github.com/coreos/go-oidc/releases) - [Commits](https://github.com/coreos/go-oidc/compare/v3.16.0...v3.17.0) --- updated-dependencies: - dependency-name: github.com/coreos/go-oidc/v3 dependency-version: 3.17.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df8e1fb9..411734f3 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.31.20 github.com/aws/aws-sdk-go-v2/credentials v1.18.24 github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 - github.com/coreos/go-oidc/v3 v3.16.0 + github.com/coreos/go-oidc/v3 v3.17.0 github.com/expr-lang/expr v1.17.6 github.com/go-co-op/gocron/v2 v2.18.2 github.com/go-ldap/ldap/v3 v3.4.12 diff --git a/go.sum b/go.sum index 711c5551..08540674 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow= -github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= +github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= +github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From 5a8b929448309601100cfb96cc2df55e06b0b402 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:04:43 +0000 Subject: [PATCH 047/341] Bump github.com/aws/aws-sdk-go-v2/config from 1.31.20 to 1.32.6 Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.31.20 to 1.32.6. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.20...v1.32.6) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.6 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 21 +++++++++++---------- go.sum | 42 ++++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index df8e1fb9..5fb06d17 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/ClusterCockpit/cc-lib v1.0.2 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.0 - github.com/aws/aws-sdk-go-v2/config v1.31.20 - github.com/aws/aws-sdk-go-v2/credentials v1.18.24 + github.com/aws/aws-sdk-go-v2/config v1.32.6 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 github.com/coreos/go-oidc/v3 v3.16.0 github.com/expr-lang/expr v1.17.6 @@ -53,18 +53,19 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 711c5551..10ecb4ee 100644 --- a/go.sum +++ b/go.sum @@ -34,36 +34,38 @@ github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgP github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= -github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc= -github.com/aws/aws-sdk-go-v2/config v1.31.20/go.mod h1:95Hh1Tc5VYKL9NJ7tAkDcqeKt+MCXQB1hQZaRdJIZE0= -github.com/aws/aws-sdk-go-v2/credentials v1.18.24 h1:iJ2FmPT35EaIB0+kMa6TnQ+PwG5A1prEdAw+PsMzfHg= -github.com/aws/aws-sdk-go-v2/credentials v1.18.24/go.mod h1:U91+DrfjAiXPDEGYhh/x29o4p0qHX5HDqG7y5VViv64= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc= github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 h1:NjShtS1t8r5LUfFVtFeI8xLAHQNTa7UI0VawXlrBMFQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.3/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 h1:gTsnx0xXNQ6SBbymoDvcoRHL+q4l/dAFsQuKfDWSaGc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= -github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 h1:HK5ON3KmQV2HcAunnx4sKLB9aPf3gKGwVAf7xnx0QT0= -github.com/aws/aws-sdk-go-v2/service/sts v1.40.2/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From 89875db4a9a8d7dc53dacb480a6c144ae847772a Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 22 Dec 2025 10:39:40 +0100 Subject: [PATCH 048/341] dashboard layout fixes --- web/frontend/src/DashPublic.root.svelte | 8 ++++---- web/frontend/src/generic/plots/Stacked.svelte | 2 +- web/frontend/src/status/DashInternal.svelte | 10 ++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 25e2683c..c69b28f6 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -338,7 +338,7 @@ - + - + @@ -540,7 +540,7 @@ Date: Mon, 22 Dec 2025 17:26:56 +0100 Subject: [PATCH 049/341] Rework info panel in public dashboard - change to bootstrap grid from table - add infos, use badges - remove non required query --- internal/metricdata/cc-metric-store.go | 30 ++-- web/frontend/src/DashPublic.root.svelte | 219 ++++++++++++++---------- web/frontend/src/generic/units.js | 2 +- 3 files changed, 146 insertions(+), 105 deletions(-) diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 6d446d17..be2e956e 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -770,21 +770,25 @@ func (ccms *CCMetricStore) LoadNodeData( } mc := archive.GetMetricConfig(cluster, metric) - hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ - Unit: mc.Unit, - Timestep: mc.Timestep, - Series: []schema.Series{ - { - Hostname: query.Hostname, - Data: qdata.Data, - Statistics: schema.MetricStatistics{ - Avg: float64(qdata.Avg), - Min: float64(qdata.Min), - Max: float64(qdata.Max), + if mc != nil { + hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ + Unit: mc.Unit, + Timestep: mc.Timestep, + Series: []schema.Series{ + { + Hostname: query.Hostname, + Data: qdata.Data, + Statistics: schema.MetricStatistics{ + Avg: float64(qdata.Avg), + Min: float64(qdata.Min), + Max: float64(qdata.Max), + }, }, }, - }, - }) + }) + } else { + cclog.Warnf("Metric '%s' not configured for cluster '%s': Skipped in LoadNodeData() Return!", metric, cluster) + } } if len(errors) != 0 { diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index c69b28f6..fbbf486d 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -30,7 +30,8 @@ Table, Progress, Icon, - Button + Button, + Badge } from "@sveltestrap/sveltestrap"; import Roofline from "./generic/plots/Roofline.svelte"; import Pie, { colors } from "./generic/plots/Pie.svelte"; @@ -85,7 +86,8 @@ query: gql` query ( $cluster: String! - $metrics: [String!] + $nmetrics: [String!] + $cmetrics: [String!] $from: Time! $to: Time! $clusterFrom: Time! @@ -97,7 +99,7 @@ # Node 5 Minute Averages for Roofline nodeMetrics( cluster: $cluster - metrics: $metrics + metrics: $nmetrics from: $from to: $to ) { @@ -106,6 +108,10 @@ metrics { name metric { + unit { + base + prefix + } series { statistics { avg @@ -114,21 +120,6 @@ } } } - # Running Job Metric Average for Rooflines - jobsMetricStats(filter: $jobFilter, metrics: $metrics) { - id - jobId - duration - numNodes - numAccelerators - subCluster - stats { - name - data { - avg - } - } - } # Get Jobs for Per-Node Counts jobs(filter: $jobFilter, order: $sorting, page: $paging) { items { @@ -175,7 +166,7 @@ # ClusterMetrics for doubleMetricPlot clusterMetrics( cluster: $cluster - metrics: $metrics + metrics: $cmetrics from: $clusterFrom to: $to ) { @@ -194,7 +185,8 @@ `, variables: { cluster: presetCluster, - metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline + nmetrics: ["flops_any", "mem_bw", "cpu_power", "acc_power"], // Metrics For Roofline and Stats + cmetrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot from: from.toISOString(), clusterFrom: clusterFrom.toISOString(), to: to.toISOString(), @@ -258,6 +250,11 @@ } } + // Get Idle Infos after Sums + if (!rawInfos['idleNodes']) rawInfos['idleNodes'] = rawInfos['totalNodes'] - rawInfos['allocatedNodes']; + if (!rawInfos['idleCores']) rawInfos['idleCores'] = rawInfos['totalCores'] - rawInfos['allocatedCores']; + if (!rawInfos['idleAccs']) rawInfos['idleAccs'] = rawInfos['totalAccs'] - rawInfos['allocatedAccs']; + // Keymetrics (Data on Cluster-Scope) let rawFlops = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) => sum + (node.metrics.find((m) => m.name == 'flops_any')?.metric?.series[0]?.statistics?.avg || 0), @@ -271,6 +268,26 @@ ) || 0; rawInfos['memBwRate'] = Math.floor((rawMemBw * 100) / 100) + let rawCpuPwr = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) => + sum + (node.metrics.find((m) => m.name == 'cpu_power')?.metric?.series[0]?.statistics?.avg || 0), + 0, // Initial Value + ) || 0; + rawInfos['cpuPwr'] = Math.floor((rawCpuPwr * 100) / 100) + if (!rawInfos['cpuPwrUnit']) { + let rawCpuUnit = $statusQuery?.data?.nodeMetrics[0]?.metrics.find((m) => m.name == 'cpu_power')?.metric?.unit || null + rawInfos['cpuPwrUnit'] = rawCpuUnit ? rawCpuUnit.prefix + rawCpuUnit.base : '' + } + + let rawGpuPwr = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) => + sum + (node.metrics.find((m) => m.name == 'acc_power')?.metric?.series[0]?.statistics?.avg || 0), + 0, // Initial Value + ) || 0; + rawInfos['gpuPwr'] = Math.floor((rawGpuPwr * 100) / 100) + if (!rawInfos['gpuPwrUnit']) { + let rawGpuUnit = $statusQuery?.data?.nodeMetrics[0]?.metrics.find((m) => m.name == 'acc_power')?.metric?.unit || null + rawInfos['gpuPwrUnit'] = rawGpuUnit ? rawGpuUnit.prefix + rawGpuUnit.base : '' + } + return rawInfos } else { return {}; @@ -408,79 +425,99 @@ - - - - - -
    - - - - - - - - -
    - - - - - - - - - - + + + + {clusterInfo?.runningJobs} + +
    + Running Jobs +
    + + + + {clusterInfo?.activeUsers} + +
    + Active Users +
    + + + + {clusterInfo?.allocatedNodes} + +
    + Active Nodes +
    + + + + + + {clusterInfo?.flopRate} {clusterInfo?.flopRateUnit} + +
    + Total Flop Rate +
    + + + + {clusterInfo?.memBwRate} {clusterInfo?.memBwRateUnit} + +
    + Total Memory Bandwidth +
    + {#if clusterInfo?.totalAccs !== 0} - - - - - + + + {clusterInfo?.gpuPwr} {clusterInfo?.gpuPwrUnit} + +
    + Total GPU Power +
    + + {:else} + + + {clusterInfo?.cpuPwr} {clusterInfo?.cpuPwrUnit} + +
    + Total CPU Power +
    + {/if} -
    {clusterInfo?.runningJobs} Running Jobs{clusterInfo?.activeUsers} Active Users
    - Flop Rate (Any) - - Memory BW Rate -
    - {clusterInfo?.flopRate} - {clusterInfo?.flopRateUnit} - - {clusterInfo?.memBwRate} - {clusterInfo?.memBwRateUnit} -
    Allocated Nodes
    - -
    		{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes} - Nodes
    Allocated Cores
    - -
    		{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)} - Cores
    Allocated Accelerators
    - -
    		{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs} - Accelerators
    +     + + + Active Cores + + + + {formatNumber(clusterInfo?.allocatedCores)} + {formatNumber(clusterInfo?.idleCores)} + + + + Idle Cores + + + {#if clusterInfo?.totalAccs !== 0} + + + Active GPU + + + + {formatNumber(clusterInfo?.allocatedAccs)} + {formatNumber(clusterInfo?.idleAccs)} + + + + Idle GPU + + + {/if}     diff --git a/web/frontend/src/generic/units.js b/web/frontend/src/generic/units.js index 1737b977..3e251fbf 100644 --- a/web/frontend/src/generic/units.js +++ b/web/frontend/src/generic/units.js @@ -3,7 +3,7 @@ */ const power = [1, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21] -const prefix = ['', 'K', 'M', 'G', 'T', 'P', 'E'] +const prefix = ['', 'k', 'M', 'G', 'T', 'P', 'E'] export function formatNumber(x) { if ( isNaN(x) || x == null) { From 0bc26aa1943cf281165084f46a2273c95ffe8d90 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 23 Dec 2025 05:56:46 +0100 Subject: [PATCH 050/341] Add error check --- internal/api/nats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/api/nats.go b/internal/api/nats.go index 1bfe9051..61cbd979 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -224,7 +224,10 @@ func (api *NatsAPI) handleNodeState(subject string, data []byte) { JobsRunning: node.JobsRunning, } - repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState) + if err := repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState); err != nil { + cclog.Errorf("NATS %s: updating node state for %s on %s failed: %v", + subject, node.Hostname, req.Cluster, err) + } } cclog.Debugf("NATS %s: updated %d node states for cluster %s", subject, len(req.Nodes), req.Cluster) From c1135531ba26d3267791113b77c0f4bcc4f71234 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 23 Dec 2025 07:56:13 +0100 Subject: [PATCH 051/341] Port NATS api to ccMessages --- go.mod | 5 +++ go.sum | 4 ++ internal/api/nats.go | 94 +++++++++++++++++++++++++++------------ internal/config/config.go | 3 +- pkg/nats/influxDecoder.go | 59 ++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 pkg/nats/influxDecoder.go diff --git a/go.mod b/go.mod index eb061de7..b821f7bf 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect @@ -89,6 +90,8 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect + github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -101,6 +104,7 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -114,6 +118,7 @@ require ( github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/mod v0.30.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sync v0.18.0 // indirect diff --git a/go.sum b/go.sum index fd4980da..04e2514b 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObk github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= @@ -64,6 +65,7 @@ github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow= @@ -194,6 +196,7 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -260,6 +263,7 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= diff --git a/internal/api/nats.go b/internal/api/nats.go index 1bfe9051..745e7acb 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -9,6 +9,7 @@ import ( "bytes" "database/sql" "encoding/json" + "strings" "sync" "time" @@ -18,7 +19,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/nats" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/ccMessage" "github.com/ClusterCockpit/cc-lib/schema" + influx "github.com/influxdata/line-protocol/v2/lineprotocol" ) // NatsAPI provides NATS subscription-based handlers for Job and Node operations. @@ -50,11 +53,7 @@ func (api *NatsAPI) StartSubscriptions() error { s := config.Keys.APISubjects - if err := client.Subscribe(s.SubjectJobStart, api.handleStartJob); err != nil { - return err - } - - if err := client.Subscribe(s.SubjectJobStop, api.handleStopJob); err != nil { + if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil { return err } @@ -67,26 +66,63 @@ func (api *NatsAPI) StartSubscriptions() error { return nil } +func (api *NatsAPI) processJobEvent(msg lp.CCMessage) { + function, ok := msg.GetTag("function") + if !ok { + cclog.Errorf("Job event is missing tag 'function': %+v", msg) + return + } + + switch function { + case "start_job": + api.handleStartJob(msg.GetEventValue()) + + case "stop_job": + api.handleStopJob(msg.GetEventValue()) + default: + cclog.Warnf("Unimplemented job event: %+v", msg) + } +} + +func (api *NatsAPI) handleJobEvent(subject string, data []byte) { + d := influx.NewDecoderWithBytes(data) + + for d.Next() { + m, err := nats.DecodeInfluxMessage(d) + if err != nil { + cclog.Errorf("NATS %s: Failed to decode message: %v", subject, err) + return + } + + if m.IsEvent() { + if m.Name() == "job" { + api.processJobEvent(m) + } + } + + } +} + // handleStartJob processes job start messages received via NATS. // Expected JSON payload follows the schema.Job structure. -func (api *NatsAPI) handleStartJob(subject string, data []byte) { +func (api *NatsAPI) handleStartJob(payload string) { req := schema.Job{ Shared: "none", MonitoringStatus: schema.MonitoringStatusRunningOrArchiving, } - dec := json.NewDecoder(bytes.NewReader(data)) + dec := json.NewDecoder(strings.NewReader(payload)) dec.DisallowUnknownFields() if err := dec.Decode(&req); err != nil { - cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + cclog.Errorf("NATS start job: parsing request failed: %v", err) return } - cclog.Debugf("NATS %s: %s", subject, req.GoString()) + cclog.Debugf("NATS start job: %s", req.GoString()) req.State = schema.JobStateRunning if err := importer.SanityChecks(&req); err != nil { - cclog.Errorf("NATS %s: sanity check failed: %v", subject, err) + cclog.Errorf("NATS start job: sanity check failed: %v", err) return } @@ -96,14 +132,14 @@ func (api *NatsAPI) handleStartJob(subject string, data []byte) { jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil) if err != nil && err != sql.ErrNoRows { - cclog.Errorf("NATS %s: checking for duplicate failed: %v", subject, err) + cclog.Errorf("NATS start job: checking for duplicate failed: %v", err) return } if err == nil { for _, job := range jobs { if (req.StartTime - job.StartTime) < secondsPerDay { - cclog.Errorf("NATS %s: job with jobId %d, cluster %s already exists (dbid: %d)", - subject, req.JobID, req.Cluster, job.ID) + cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)", + req.JobID, req.Cluster, job.ID) return } } @@ -111,14 +147,14 @@ func (api *NatsAPI) handleStartJob(subject string, data []byte) { id, err := api.JobRepository.Start(&req) if err != nil { - cclog.Errorf("NATS %s: insert into database failed: %v", subject, err) + cclog.Errorf("NATS start job: insert into database failed: %v", err) return } unlockOnce.Do(api.RepositoryMutex.Unlock) for _, tag := range req.Tags { if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil { - cclog.Errorf("NATS %s: adding tag to new job %d failed: %v", subject, id, err) + cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err) return } } @@ -129,18 +165,18 @@ func (api *NatsAPI) handleStartJob(subject string, data []byte) { // handleStopJob processes job stop messages received via NATS. // Expected JSON payload follows the StopJobAPIRequest structure. -func (api *NatsAPI) handleStopJob(subject string, data []byte) { +func (api *NatsAPI) handleStopJob(payload string) { var req StopJobAPIRequest - dec := json.NewDecoder(bytes.NewReader(data)) + dec := json.NewDecoder(strings.NewReader(payload)) dec.DisallowUnknownFields() if err := dec.Decode(&req); err != nil { - cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + cclog.Errorf("NATS job stop: parsing request failed: %v", err) return } if req.JobID == nil { - cclog.Errorf("NATS %s: the field 'jobId' is required", subject) + cclog.Errorf("NATS job stop: the field 'jobId' is required") return } @@ -148,28 +184,28 @@ func (api *NatsAPI) handleStopJob(subject string, data []byte) { if err != nil { cachedJob, cachedErr := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime) if cachedErr != nil { - cclog.Errorf("NATS %s: finding job failed: %v (cached lookup also failed: %v)", - subject, err, cachedErr) + cclog.Errorf("NATS job stop: finding job failed: %v (cached lookup also failed: %v)", + err, cachedErr) return } job = cachedJob } if job.State != schema.JobStateRunning { - cclog.Errorf("NATS %s: jobId %d (id %d) on %s: job has already been stopped (state is: %s)", - subject, job.JobID, job.ID, job.Cluster, job.State) + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)", + job.JobID, job.ID, job.Cluster, job.State) return } if job.StartTime > req.StopTime { - cclog.Errorf("NATS %s: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d", - subject, job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime) + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d", + job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime) return } if req.State != "" && !req.State.Valid() { - cclog.Errorf("NATS %s: jobId %d (id %d) on %s: invalid job state: %#v", - subject, job.JobID, job.ID, job.Cluster, req.State) + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v", + job.JobID, job.ID, job.Cluster, req.State) return } else if req.State == "" { req.State = schema.JobStateCompleted @@ -182,8 +218,8 @@ func (api *NatsAPI) handleStopJob(subject string, data []byte) { if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - cclog.Errorf("NATS %s: jobId %d (id %d) on %s: marking job as '%s' failed: %v", - subject, job.JobID, job.ID, job.Cluster, job.State, err) + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v", + job.JobID, job.ID, job.Cluster, job.State, err) return } } diff --git a/internal/config/config.go b/internal/config/config.go index b7b8ed06..3c88bcfd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -90,8 +90,7 @@ type ResampleConfig struct { } type NATSConfig struct { - SubjectJobStart string `json:"subjectJobStart"` - SubjectJobStop string `json:"subjectJobStop"` + SubjectJobEvent string `json:"subjectJobEvent"` SubjectNodeState string `json:"subjectNodeState"` } diff --git a/pkg/nats/influxDecoder.go b/pkg/nats/influxDecoder.go new file mode 100644 index 00000000..412f85e9 --- /dev/null +++ b/pkg/nats/influxDecoder.go @@ -0,0 +1,59 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nats + +import ( + "time" + + lp "github.com/ClusterCockpit/cc-lib/ccMessage" + influx "github.com/influxdata/line-protocol/v2/lineprotocol" +) + +// DecodeInfluxMessage decodes a single InfluxDB line protocol message from the decoder +// Returns the decoded CCMessage or an error if decoding fails +func DecodeInfluxMessage(d *influx.Decoder) (lp.CCMessage, error) { + measurement, err := d.Measurement() + if err != nil { + return nil, err + } + + tags := make(map[string]string) + for { + key, value, err := d.NextTag() + if err != nil { + return nil, err + } + if key == nil { + break + } + tags[string(key)] = string(value) + } + + fields := make(map[string]interface{}) + for { + key, value, err := d.NextField() + if err != nil { + return nil, err + } + if key == nil { + break + } + fields[string(key)] = value.Interface() + } + + t, err := d.Time(influx.Nanosecond, time.Time{}) + if err != nil { + return nil, err + } + + return lp.NewMessage( + string(measurement), + tags, + nil, + fields, + t, + ) +} From 64fef9774cce32caf35b8e20adbef51896205315 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 23 Dec 2025 09:22:57 +0100 Subject: [PATCH 052/341] Add unit test for NATS API --- internal/api/api_test.go | 10 +- internal/api/nats_test.go | 892 ++++++++++++++++++++++++++++ internal/repository/dbConnection.go | 23 + 3 files changed, 920 insertions(+), 5 deletions(-) create mode 100644 internal/api/nats_test.go diff --git a/internal/api/api_test.go b/internal/api/api_test.go index d311767c..3030b1c1 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -36,6 +36,8 @@ import ( ) func setup(t *testing.T) *api.RestAPI { + repository.ResetConnection() + const testconfig = `{ "main": { "addr": "0.0.0.0:8080", @@ -190,11 +192,9 @@ func setup(t *testing.T) *api.RestAPI { } func cleanup() { - // Gracefully shutdown archiver with timeout if err := archiver.Shutdown(5 * time.Second); err != nil { cclog.Warnf("Archiver shutdown timeout in tests: %v", err) } - // TODO: Clear all caches, reset all modules, etc... } /* @@ -230,7 +230,7 @@ func TestRestApi(t *testing.T) { r.StrictSlash(true) restapi.MountAPIRoutes(r) - var TestJobId int64 = 123 + var TestJobID int64 = 123 TestClusterName := "testcluster" var TestStartTime int64 = 123456789 @@ -280,7 +280,7 @@ func TestRestApi(t *testing.T) { } // resolver := graph.GetResolverInstance() restapi.JobRepository.SyncJobs() - job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime) + job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime) if err != nil { t.Fatal(err) } @@ -338,7 +338,7 @@ func TestRestApi(t *testing.T) { } // Archiving happens asynchronously, will be completed in cleanup - job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime) + job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime) if err != nil { t.Fatal(err) } diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go new file mode 100644 index 00000000..420a359c --- /dev/null +++ b/internal/api/nats_test.go @@ -0,0 +1,892 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package api + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/archiver" + "github.com/ClusterCockpit/cc-backend/internal/auth" + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/graph" + "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/ccMessage" + "github.com/ClusterCockpit/cc-lib/schema" + + _ "github.com/mattn/go-sqlite3" +) + +func setupNatsTest(t *testing.T) *NatsAPI { + repository.ResetConnection() + + const testconfig = `{ + "main": { + "addr": "0.0.0.0:8080", + "validate": false, + "apiAllowedIPs": [ + "*" + ] + }, + "archive": { + "kind": "file", + "path": "./var/job-archive" + }, + "auth": { + "jwts": { + "max-age": "2m" + } + }, + "clusters": [ + { + "name": "testcluster", + "metricDataRepository": {"kind": "test", "url": "bla:8081"}, + "filterRanges": { + "numNodes": { "from": 1, "to": 64 }, + "duration": { "from": 0, "to": 86400 }, + "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } + } + } + ] +}` + const testclusterJSON = `{ + "name": "testcluster", + "subClusters": [ + { + "name": "sc1", + "nodes": "host123,host124,host125", + "processorType": "Intel Core i7-4770", + "socketsPerNode": 1, + "coresPerSocket": 4, + "threadsPerCore": 2, + "flopRateScalar": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 14 + }, + "flopRateSimd": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 112 + }, + "memoryBandwidth": { + "unit": { + "prefix": "G", + "base": "B/s" + }, + "value": 24 + }, + "numberOfNodes": 70, + "topology": { + "node": [0, 1, 2, 3, 4, 5, 6, 7], + "socket": [[0, 1, 2, 3, 4, 5, 6, 7]], + "memoryDomain": [[0, 1, 2, 3, 4, 5, 6, 7]], + "die": [[0, 1, 2, 3, 4, 5, 6, 7]], + "core": [[0], [1], [2], [3], [4], [5], [6], [7]] + } + } + ], + "metricConfig": [ + { + "name": "load_one", + "unit": { "base": ""}, + "scope": "node", + "timestep": 60, + "aggregation": "avg", + "peak": 8, + "normal": 0, + "caution": 0, + "alert": 0 + } + ] + }` + + cclog.Init("info", true) + tmpdir := t.TempDir() + jobarchive := filepath.Join(tmpdir, "job-archive") + if err := os.Mkdir(jobarchive, 0o777); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 3), 0o666); err != nil { + t.Fatal(err) + } + + if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJSON), 0o666); err != nil { + t.Fatal(err) + } + + dbfilepath := filepath.Join(tmpdir, "test.db") + err := repository.MigrateDB(dbfilepath) + if err != nil { + t.Fatal(err) + } + + cfgFilePath := filepath.Join(tmpdir, "config.json") + if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil { + t.Fatal(err) + } + + ccconf.Init(cfgFilePath) + + // Load and check main configuration + if cfg := ccconf.GetPackageConfig("main"); cfg != nil { + if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { + config.Init(cfg, clustercfg) + } else { + cclog.Abort("Cluster configuration must be present") + } + } else { + cclog.Abort("Main configuration must be present") + } + archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) + + repository.Connect("sqlite3", dbfilepath) + + if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil { + t.Fatal(err) + } + + if err := metricdata.Init(); err != nil { + t.Fatal(err) + } + + archiver.Start(repository.GetJobRepository(), context.Background()) + + if cfg := ccconf.GetPackageConfig("auth"); cfg != nil { + auth.Init(&cfg) + } else { + cclog.Warn("Authentication disabled due to missing configuration") + auth.Init(nil) + } + + graph.Init() + + return NewNatsAPI() +} + +func cleanupNatsTest() { + if err := archiver.Shutdown(5 * time.Second); err != nil { + cclog.Warnf("Archiver shutdown timeout in tests: %v", err) + } +} + +func TestNatsHandleStartJob(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + tests := []struct { + name string + payload string + expectError bool + validateJob func(t *testing.T, job *schema.Job) + shouldFindJob bool + }{ + { + name: "valid job start", + payload: `{ + "jobId": 1001, + "user": "testuser1", + "project": "testproj1", + "cluster": "testcluster", + "partition": "main", + "walltime": 7200, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7] + } + ], + "startTime": 1234567890 + }`, + expectError: false, + shouldFindJob: true, + validateJob: func(t *testing.T, job *schema.Job) { + if job.JobID != 1001 { + t.Errorf("expected JobID 1001, got %d", job.JobID) + } + if job.User != "testuser1" { + t.Errorf("expected user testuser1, got %s", job.User) + } + if job.State != schema.JobStateRunning { + t.Errorf("expected state running, got %s", job.State) + } + }, + }, + { + name: "invalid JSON", + payload: `{ + "jobId": "not a number", + "user": "testuser2" + }`, + expectError: true, + shouldFindJob: false, + }, + { + name: "missing required fields", + payload: `{ + "jobId": 1002 + }`, + expectError: true, + shouldFindJob: false, + }, + { + name: "job with unknown fields (should fail due to DisallowUnknownFields)", + payload: `{ + "jobId": 1003, + "user": "testuser3", + "project": "testproj3", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "unknownField": "should cause error", + "startTime": 1234567900 + }`, + expectError: true, + shouldFindJob: false, + }, + { + name: "job with tags", + payload: `{ + "jobId": 1004, + "user": "testuser4", + "project": "testproj4", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3] + } + ], + "tags": [ + { + "type": "test", + "name": "testtag", + "scope": "testuser4" + } + ], + "startTime": 1234567910 + }`, + expectError: false, + shouldFindJob: true, + validateJob: func(t *testing.T, job *schema.Job) { + if job.JobID != 1004 { + t.Errorf("expected JobID 1004, got %d", job.JobID) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + natsAPI.handleStartJob(tt.payload) + natsAPI.JobRepository.SyncJobs() + + // Allow some time for async operations + time.Sleep(100 * time.Millisecond) + + if tt.shouldFindJob { + // Extract jobId from payload + var payloadMap map[string]any + json.Unmarshal([]byte(tt.payload), &payloadMap) + jobID := int64(payloadMap["jobId"].(float64)) + cluster := payloadMap["cluster"].(string) + startTime := int64(payloadMap["startTime"].(float64)) + + job, err := natsAPI.JobRepository.Find(&jobID, &cluster, &startTime) + if err != nil { + if !tt.expectError { + t.Fatalf("expected to find job, but got error: %v", err) + } + return + } + + if tt.validateJob != nil { + tt.validateJob(t, job) + } + } + }) + } +} + +func TestNatsHandleStopJob(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + // First, create a running job + startPayload := `{ + "jobId": 2001, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7] + } + ], + "startTime": 1234567890 + }` + + natsAPI.handleStartJob(startPayload) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + + tests := []struct { + name string + payload string + expectError bool + validateJob func(t *testing.T, job *schema.Job) + setupJobFunc func() // Optional: create specific test job + }{ + { + name: "valid job stop - completed", + payload: `{ + "jobId": 2001, + "cluster": "testcluster", + "startTime": 1234567890, + "jobState": "completed", + "stopTime": 1234571490 + }`, + expectError: false, + validateJob: func(t *testing.T, job *schema.Job) { + if job.State != schema.JobStateCompleted { + t.Errorf("expected state completed, got %s", job.State) + } + expectedDuration := int32(1234571490 - 1234567890) + if job.Duration != expectedDuration { + t.Errorf("expected duration %d, got %d", expectedDuration, job.Duration) + } + }, + }, + { + name: "valid job stop - failed", + setupJobFunc: func() { + startPayloadFailed := `{ + "jobId": 2002, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3] + } + ], + "startTime": 1234567900 + }` + natsAPI.handleStartJob(startPayloadFailed) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + }, + payload: `{ + "jobId": 2002, + "cluster": "testcluster", + "startTime": 1234567900, + "jobState": "failed", + "stopTime": 1234569900 + }`, + expectError: false, + validateJob: func(t *testing.T, job *schema.Job) { + if job.State != schema.JobStateFailed { + t.Errorf("expected state failed, got %s", job.State) + } + }, + }, + { + name: "invalid JSON", + payload: `{ + "jobId": "not a number" + }`, + expectError: true, + }, + { + name: "missing jobId", + payload: `{ + "cluster": "testcluster", + "jobState": "completed", + "stopTime": 1234571490 + }`, + expectError: true, + }, + { + name: "invalid job state", + setupJobFunc: func() { + startPayloadInvalid := `{ + "jobId": 2003, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1] + } + ], + "startTime": 1234567910 + }` + natsAPI.handleStartJob(startPayloadInvalid) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + }, + payload: `{ + "jobId": 2003, + "cluster": "testcluster", + "startTime": 1234567910, + "jobState": "invalid_state", + "stopTime": 1234571510 + }`, + expectError: true, + }, + { + name: "stopTime before startTime", + setupJobFunc: func() { + startPayloadTime := `{ + "jobId": 2004, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0] + } + ], + "startTime": 1234567920 + }` + natsAPI.handleStartJob(startPayloadTime) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + }, + payload: `{ + "jobId": 2004, + "cluster": "testcluster", + "startTime": 1234567920, + "jobState": "completed", + "stopTime": 1234567900 + }`, + expectError: true, + }, + { + name: "job not found", + payload: `{ + "jobId": 99999, + "cluster": "testcluster", + "startTime": 1234567890, + "jobState": "completed", + "stopTime": 1234571490 + }`, + expectError: true, + }, + } + + testData := schema.JobData{ + "load_one": map[schema.MetricScope]*schema.JobMetric{ + schema.MetricScopeNode: { + Unit: schema.Unit{Base: "load"}, + Timestep: 60, + Series: []schema.Series{ + { + Hostname: "host123", + Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3}, + Data: []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3}, + }, + }, + }, + }, + } + + metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { + return testData, nil + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupJobFunc != nil { + tt.setupJobFunc() + } + + natsAPI.handleStopJob(tt.payload) + + // Allow some time for async operations + time.Sleep(100 * time.Millisecond) + + if !tt.expectError && tt.validateJob != nil { + // Extract job details from payload + var payloadMap map[string]any + json.Unmarshal([]byte(tt.payload), &payloadMap) + jobID := int64(payloadMap["jobId"].(float64)) + cluster := payloadMap["cluster"].(string) + + var startTime *int64 + if st, ok := payloadMap["startTime"]; ok { + t := int64(st.(float64)) + startTime = &t + } + + job, err := natsAPI.JobRepository.Find(&jobID, &cluster, startTime) + if err != nil { + t.Fatalf("expected to find job, but got error: %v", err) + } + + tt.validateJob(t, job) + } + }) + } +} + +func TestNatsHandleNodeState(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + tests := []struct { + name string + payload string + expectError bool + validateFn func(t *testing.T) + }{ + { + name: "valid node state update", + payload: `{ + "cluster": "testcluster", + "nodes": [ + { + "hostname": "host123", + "states": ["allocated"], + "cpusAllocated": 8, + "memoryAllocated": 16384, + "gpusAllocated": 0, + "jobsRunning": 1 + } + ] + }`, + expectError: false, + validateFn: func(t *testing.T) { + // In a full test, we would verify the node state was updated in the database + // For now, just ensure no error occurred + }, + }, + { + name: "multiple nodes", + payload: `{ + "cluster": "testcluster", + "nodes": [ + { + "hostname": "host123", + "states": ["idle"], + "cpusAllocated": 0, + "memoryAllocated": 0, + "gpusAllocated": 0, + "jobsRunning": 0 + }, + { + "hostname": "host124", + "states": ["allocated"], + "cpusAllocated": 4, + "memoryAllocated": 8192, + "gpusAllocated": 1, + "jobsRunning": 1 + } + ] + }`, + expectError: false, + }, + { + name: "invalid JSON", + payload: `{ + "cluster": "testcluster", + "nodes": "not an array" + }`, + expectError: true, + }, + { + name: "empty nodes array", + payload: `{ + "cluster": "testcluster", + "nodes": [] + }`, + expectError: false, // Empty array should not cause error + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + natsAPI.handleNodeState("test.subject", []byte(tt.payload)) + + // Allow some time for async operations + time.Sleep(50 * time.Millisecond) + + if tt.validateFn != nil { + tt.validateFn(t) + } + }) + } +} + +func TestNatsProcessJobEvent(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + msgStartJob, err := lp.NewMessage( + "job", + map[string]string{"function": "start_job"}, + nil, + map[string]any{ + "event": `{ + "jobId": 3001, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3] + } + ], + "startTime": 1234567890 + }`, + }, + time.Now(), + ) + if err != nil { + t.Fatalf("failed to create test message: %v", err) + } + + msgMissingTag, err := lp.NewMessage( + "job", + map[string]string{}, + nil, + map[string]any{ + "event": `{}`, + }, + time.Now(), + ) + if err != nil { + t.Fatalf("failed to create test message: %v", err) + } + + msgUnknownFunc, err := lp.NewMessage( + "job", + map[string]string{"function": "unknown_function"}, + nil, + map[string]any{ + "event": `{}`, + }, + time.Now(), + ) + if err != nil { + t.Fatalf("failed to create test message: %v", err) + } + + tests := []struct { + name string + message lp.CCMessage + expectError bool + }{ + { + name: "start_job function", + message: msgStartJob, + expectError: false, + }, + { + name: "missing function tag", + message: msgMissingTag, + expectError: true, + }, + { + name: "unknown function", + message: msgUnknownFunc, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + natsAPI.processJobEvent(tt.message) + time.Sleep(50 * time.Millisecond) + }) + } +} + +func TestNatsHandleJobEvent(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + tests := []struct { + name string + data []byte + expectError bool + }{ + { + name: "valid influx line protocol", + data: []byte(`job,function=start_job event="{\"jobId\":4001,\"user\":\"testuser\",\"project\":\"testproj\",\"cluster\":\"testcluster\",\"partition\":\"main\",\"walltime\":3600,\"numNodes\":1,\"numHwthreads\":8,\"numAcc\":0,\"shared\":\"none\",\"monitoringStatus\":1,\"smt\":1,\"resources\":[{\"hostname\":\"host123\",\"hwthreads\":[0,1,2,3]}],\"startTime\":1234567890}"`), + expectError: false, + }, + { + name: "invalid influx line protocol", + data: []byte(`invalid line protocol format`), + expectError: true, + }, + { + name: "empty data", + data: []byte(``), + expectError: false, // Decoder should handle empty input gracefully + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // HandleJobEvent doesn't return errors, it logs them + // We're just ensuring it doesn't panic + natsAPI.handleJobEvent("test.subject", tt.data) + time.Sleep(50 * time.Millisecond) + }) + } +} + +func TestNatsHandleStartJobDuplicatePrevention(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + // Start a job + payload := `{ + "jobId": 5001, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3] + } + ], + "startTime": 1234567890 + }` + + natsAPI.handleStartJob(payload) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + + // Try to start the same job again (within 24 hours) + duplicatePayload := `{ + "jobId": 5001, + "user": "testuser", + "project": "testproj", + "cluster": "testcluster", + "partition": "main", + "walltime": 3600, + "numNodes": 1, + "numHwthreads": 8, + "numAcc": 0, + "shared": "none", + "monitoringStatus": 1, + "smt": 1, + "resources": [ + { + "hostname": "host123", + "hwthreads": [0, 1, 2, 3] + } + ], + "startTime": 1234567900 + }` + + natsAPI.handleStartJob(duplicatePayload) + natsAPI.JobRepository.SyncJobs() + time.Sleep(100 * time.Millisecond) + + // Verify only one job exists + jobID := int64(5001) + cluster := "testcluster" + jobs, err := natsAPI.JobRepository.FindAll(&jobID, &cluster, nil) + if err != nil && err != sql.ErrNoRows { + t.Fatalf("unexpected error: %v", err) + } + + if len(jobs) != 1 { + t.Errorf("expected 1 job, got %d", len(jobs)) + } +} diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 0f7536b7..be0b161b 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -115,3 +115,26 @@ func GetConnection() *DBConnection { return dbConnInstance } + +// ResetConnection closes the current database connection and resets the connection state. +// This function is intended for testing purposes only to allow test isolation. +func ResetConnection() error { + if dbConnInstance != nil && dbConnInstance.DB != nil { + if err := dbConnInstance.DB.Close(); err != nil { + return fmt.Errorf("failed to close database connection: %w", err) + } + } + + dbConnInstance = nil + dbConnOnce = sync.Once{} + jobRepoInstance = nil + jobRepoOnce = sync.Once{} + nodeRepoInstance = nil + nodeRepoOnce = sync.Once{} + userRepoInstance = nil + userRepoOnce = sync.Once{} + userCfgRepoInstance = nil + userCfgRepoOnce = sync.Once{} + + return nil +} From 9bf5c5dc1a0b5eae739a8ef8971d470059def827 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 23 Dec 2025 09:34:09 +0100 Subject: [PATCH 053/341] Update README and config schema --- CLAUDE.md | 94 ++++++++++++++++++++++++++++++++++++++- README.md | 62 +++++++++++++++++++++----- configs/config-demo.json | 12 ++--- configs/config.json | 4 ++ internal/config/schema.go | 15 +++++++ 5 files changed, 165 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 379b4dbb..67412a76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,11 +100,15 @@ The backend follows a layered architecture with clear separation of concerns: - Pluggable backends: cc-metric-store, Prometheus, InfluxDB - Each cluster can have a different metric data backend - **internal/archiver**: Job archiving to file-based archive +- **internal/api/nats.go**: NATS-based API for job and node operations + - Subscribes to NATS subjects for job events (start/stop) + - Handles node state updates via NATS + - Uses InfluxDB line protocol message format - **pkg/archive**: Job archive backend implementations - File system backend (default) - S3 backend - SQLite backend (experimental) -- **pkg/nats**: NATS integration for metric ingestion +- **pkg/nats**: NATS client and message decoding utilities ### Frontend Structure @@ -146,6 +150,14 @@ applied automatically on startup. Version tracking in `version` table. ## Configuration - **config.json**: Main configuration (clusters, metric repositories, archive settings) + - `main.apiSubjects`: NATS subject configuration (optional) + - `subjectJobEvent`: Subject for job start/stop events (e.g., "cc.job.event") + - `subjectNodeState`: Subject for node state updates (e.g., "cc.node.state") + - `nats`: NATS client connection configuration (optional) + - `address`: NATS server address (e.g., "nats://localhost:4222") + - `username`: Authentication username (optional) + - `password`: Authentication password (optional) + - `creds-file-path`: Path to NATS credentials file (optional) - **.env**: Environment variables (secrets like JWT keys) - Copy from `configs/env-template.txt` - NEVER commit this file @@ -207,9 +219,87 @@ applied automatically on startup. Version tracking in `version` table. 2. Increment `repository.Version` 3. Test with fresh database and existing database +## NATS API + +The backend supports a NATS-based API as an alternative to the REST API for job and node operations. + +### Setup + +1. Configure NATS client connection in `config.json`: + ```json + { + "nats": { + "address": "nats://localhost:4222", + "username": "user", + "password": "pass" + } + } + ``` + +2. Configure API subjects in `config.json` under `main`: + ```json + { + "main": { + "apiSubjects": { + "subjectJobEvent": "cc.job.event", + "subjectNodeState": "cc.node.state" + } + } + } + ``` + +### Message Format + +Messages use **InfluxDB line protocol** format with the following structure: + +#### Job Events + +**Start Job:** +``` +job,function=start_job event="{\"jobId\":123,\"user\":\"alice\",\"cluster\":\"test\", ...}" 1234567890000000000 +``` + +**Stop Job:** +``` +job,function=stop_job event="{\"jobId\":123,\"cluster\":\"test\",\"startTime\":1234567890,\"stopTime\":1234571490,\"jobState\":\"completed\"}" 1234571490000000000 +``` + +**Tags:** +- `function`: Either `start_job` or `stop_job` + +**Fields:** +- `event`: JSON payload containing job data (see REST API documentation for schema) + +#### Node State Updates + +```json +{ + "cluster": "testcluster", + "nodes": [ + { + "hostname": "node001", + "states": ["allocated"], + "cpusAllocated": 8, + "memoryAllocated": 16384, + "gpusAllocated": 0, + "jobsRunning": 1 + } + ] +} +``` + +### Implementation Notes + +- NATS API mirrors REST API functionality but uses messaging +- Job start/stop events are processed asynchronously +- Duplicate job detection is handled (same as REST API) +- All validation rules from REST API apply +- Messages are logged; no responses are sent back to publishers +- If NATS client is unavailable, API subscriptions are skipped (logged as warning) + ## Dependencies - Go 1.24.0+ (check go.mod for exact version) - Node.js (for frontend builds) - SQLite 3 (only supported database) -- Optional: NATS server for metric ingestion +- Optional: NATS server for NATS API integration diff --git a/README.md b/README.md index a0352d17..468a12ad 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,12 @@ switching from PHP Symfony to a Golang based solution are explained ## Overview This is a Golang web backend for the ClusterCockpit job-specific performance -monitoring framework. It provides a REST API for integrating ClusterCockpit with -an HPC cluster batch system and external analysis scripts. Data exchange between -the web front-end and the back-end is based on a GraphQL API. The web frontend -is also served by the backend using [Svelte](https://svelte.dev/) components. -Layout and styling are based on [Bootstrap 5](https://getbootstrap.com/) using +monitoring framework. It provides a REST API and an optional NATS-based messaging +API for integrating ClusterCockpit with an HPC cluster batch system and external +analysis scripts. Data exchange between the web front-end and the back-end is +based on a GraphQL API. The web frontend is also served by the backend using +[Svelte](https://svelte.dev/) components. Layout and styling are based on +[Bootstrap 5](https://getbootstrap.com/) using [Bootstrap Icons](https://icons.getbootstrap.com/). The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database. @@ -35,6 +36,10 @@ databases, the only tested and supported setup is to use cc-metric-store as the metric data backend. Documentation on how to integrate ClusterCockpit with other time series databases will be added in the future. +For real-time integration with HPC systems, the backend can subscribe to +[NATS](https://nats.io/) subjects to receive job start/stop events and node +state updates, providing an alternative to REST API polling. + Completed batch jobs are stored in a file-based job archive according to [this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive). The backend supports authentication via local accounts, an external LDAP @@ -130,27 +135,60 @@ ln -s ./var/job-archive ## Project file structure +- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/master/.github) + GitHub Actions workflows and dependabot configuration for CI/CD. - [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api) contains the API schema files for the REST and GraphQL APIs. The REST API is documented in the OpenAPI 3.0 format in - [./api/openapi.yaml](./api/openapi.yaml). + [./api/swagger.yaml](./api/swagger.yaml). The GraphQL schema is in + [./api/schema.graphqls](./api/schema.graphqls). - [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend) - contains `main.go` for the main application. + contains the main application entry point and CLI implementation. - [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs) contains documentation about configuration and command line options and required - environment variables. A sample configuration file is provided. -- [`docs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/docs) - contains more in-depth documentation. + environment variables. Sample configuration files are provided. - [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init) contains an example of setting up systemd for production use. - [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal) contains library source code that is not intended for use by others. + - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/api) + REST API handlers and NATS integration + - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/archiver) + Job archiving functionality + - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/auth) + Authentication (local, LDAP, OIDC) and JWT token handling + - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/config) + Configuration management and validation + - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/graph) + GraphQL schema and resolvers + - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer) + Job data import and database initialization + - [`memorystore`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/memorystore) + In-memory metric data store with checkpointing + - [`metricdata`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdata) + Metric data repository implementations (cc-metric-store, Prometheus) + - [`metricDataDispatcher`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricDataDispatcher) + Dispatches metric data loading to appropriate backends + - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository) + Database repository layer for jobs and metadata + - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/routerConfig) + HTTP router configuration and middleware + - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/tagger) + Job classification and application detection + - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager) + Background task management and scheduled jobs - [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg) contains Go packages that can be used by other projects. + - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive) + Job archive backend implementations (filesystem, S3) + - [`nats`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/nats) + NATS client and message handling - [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools) Additional command line helper tools. - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager) - Commands for getting infos about and existing job archive. + Commands for getting infos about an existing job archive. + - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration) + Tool for migrating job archives between formats. - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey) Tool to convert external pubkey for use in `cc-backend`. - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair) @@ -162,7 +200,7 @@ ln -s ./var/job-archive - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend) Svelte components and static assets for the frontend UI - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates) - Server-side Go templates + Server-side Go templates, including monitoring views - [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml) Configures the behaviour and generation of [gqlgen](https://github.com/99designs/gqlgen). diff --git a/configs/config-demo.json b/configs/config-demo.json index 58366fb5..aa388316 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -5,14 +5,9 @@ "resampling": { "minimumPoints": 600, "trigger": 180, - "resolutions": [ - 240, - 60 - ] + "resolutions": [240, 60] }, - "apiAllowedIPs": [ - "*" - ], + "apiAllowedIPs": ["*"], "emission-constant": 317 }, "cron": { @@ -103,4 +98,5 @@ } ] } -} \ No newline at end of file +} + diff --git a/configs/config.json b/configs/config.json index 88a9e930..41d8ecac 100644 --- a/configs/config.json +++ b/configs/config.json @@ -15,6 +15,10 @@ 240, 60 ] + }, + "apiSubjects": { + "subjectJobEvent": "cc.job.event", + "subjectNodeState": "cc.node.state" } }, "cron": { diff --git a/internal/config/schema.go b/internal/config/schema.go index b171f96a..ff8d0c92 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -119,6 +119,21 @@ var configSchema = ` } }, "required": ["trigger", "resolutions"] + }, + "apiSubjects": { + "description": "NATS subjects configuration for subscribing to job and node events.", + "type": "object", + "properties": { + "subjectJobEvent": { + "description": "NATS subject for job events (start_job, stop_job)", + "type": "string" + }, + "subjectNodeState": { + "description": "NATS subject for node state updates", + "type": "string" + } + }, + "required": ["subjectJobEvent", "subjectNodeState"] } }, "required": ["apiAllowedIPs"] From 29a20f7b0b995f99f8f198cfe69b8ace9fff4ef7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 09:07:01 +0000 Subject: [PATCH 054/341] Bump github.com/expr-lang/expr from 1.17.6 to 1.17.7 Bumps [github.com/expr-lang/expr](https://github.com/expr-lang/expr) from 1.17.6 to 1.17.7. - [Release notes](https://github.com/expr-lang/expr/releases) - [Commits](https://github.com/expr-lang/expr/compare/v1.17.6...v1.17.7) --- updated-dependencies: - dependency-name: github.com/expr-lang/expr dependency-version: 1.17.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a443875..4da3b80e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.6 github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 github.com/coreos/go-oidc/v3 v3.17.0 - github.com/expr-lang/expr v1.17.6 + github.com/expr-lang/expr v1.17.7 github.com/go-co-op/gocron/v2 v2.18.2 github.com/go-ldap/ldap/v3 v3.4.12 github.com/go-sql-driver/mysql v1.9.3 diff --git a/go.sum b/go.sum index e2f5ba37..773bf31c 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec= -github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= +github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= From 8576ae458d11d5df74faca4088f9ff4d0a0c7774 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 24 Dec 2025 09:24:18 +0100 Subject: [PATCH 055/341] Switch to cc-lib v2 --- cmd/cc-backend/init.go | 4 +- cmd/cc-backend/main.go | 14 ++--- cmd/cc-backend/server.go | 6 +- go.mod | 16 ++--- go.sum | 32 +++++----- gqlgen.yml | 52 +++++++-------- internal/api/api_test.go | 6 +- internal/api/cluster.go | 2 +- internal/api/job.go | 4 +- internal/api/memorystore.go | 2 +- internal/api/nats.go | 21 +++++-- internal/api/nats_test.go | 8 +-- internal/api/node.go | 2 +- internal/api/rest.go | 6 +- internal/api/user.go | 4 +- internal/archiver/archiveWorker.go | 4 +- internal/archiver/archiver.go | 4 +- internal/auth/auth.go | 6 +- internal/auth/jwt.go | 4 +- internal/auth/jwtCookieSession.go | 4 +- internal/auth/jwtHelpers.go | 4 +- internal/auth/jwtHelpers_test.go | 2 +- internal/auth/jwtSession.go | 4 +- internal/auth/ldap.go | 4 +- internal/auth/local.go | 4 +- internal/auth/oidc.go | 4 +- internal/config/config.go | 4 +- internal/config/config_test.go | 4 +- internal/config/validate.go | 2 +- internal/graph/generated/generated.go | 2 +- internal/graph/model/models_gen.go | 2 +- internal/graph/resolver.go | 2 +- internal/graph/schema.resolvers.go | 4 +- internal/graph/util.go | 4 +- internal/importer/handleImport.go | 4 +- internal/importer/importer_test.go | 4 +- internal/importer/initDB.go | 4 +- internal/importer/normalize.go | 2 +- internal/importer/normalize_test.go | 2 +- internal/memorystore/api.go | 4 +- internal/memorystore/archive.go | 2 +- internal/memorystore/avroCheckpoint.go | 4 +- internal/memorystore/avroHelper.go | 2 +- internal/memorystore/avroStruct.go | 2 +- internal/memorystore/buffer.go | 2 +- internal/memorystore/checkpoint.go | 4 +- internal/memorystore/level.go | 2 +- internal/memorystore/lineprotocol.go | 4 +- internal/memorystore/memorystore.go | 8 +-- internal/memorystore/memorystore_test.go | 2 +- internal/memorystore/stats.go | 2 +- internal/metricDataDispatcher/dataLoader.go | 8 +-- .../metricdata/cc-metric-store-internal.go | 4 +- internal/metricdata/cc-metric-store.go | 4 +- internal/metricdata/metricdata.go | 4 +- internal/metricdata/prometheus.go | 4 +- internal/metricdata/utils.go | 2 +- internal/repository/dbConnection.go | 2 +- internal/repository/hooks.go | 2 +- internal/repository/job.go | 6 +- internal/repository/jobCreate.go | 4 +- internal/repository/jobFind.go | 4 +- internal/repository/jobHooks.go | 2 +- internal/repository/jobQuery.go | 4 +- internal/repository/job_test.go | 2 +- internal/repository/migration.go | 2 +- internal/repository/node.go | 6 +- internal/repository/node_test.go | 6 +- internal/repository/repository_test.go | 4 +- internal/repository/stats.go | 4 +- internal/repository/tags.go | 4 +- internal/repository/testdata/job.db | Bin 987136 -> 987136 bytes internal/repository/user.go | 4 +- internal/repository/userConfig.go | 6 +- internal/repository/userConfig_test.go | 6 +- internal/routerConfig/routes.go | 6 +- internal/tagger/classifyJob.go | 6 +- internal/tagger/classifyJob_test.go | 2 +- internal/tagger/detectApp.go | 6 +- internal/tagger/detectApp_test.go | 2 +- internal/tagger/tagger.go | 4 +- internal/tagger/tagger_test.go | 2 +- internal/taskmanager/commitJobService.go | 2 +- internal/taskmanager/compressionService.go | 4 +- internal/taskmanager/ldapSyncService.go | 2 +- internal/taskmanager/retentionService.go | 2 +- internal/taskmanager/stopJobsExceedTime.go | 2 +- internal/taskmanager/taskManager.go | 2 +- internal/taskmanager/updateDurationService.go | 2 +- .../taskmanager/updateFootprintService.go | 4 +- pkg/archive/archive.go | 6 +- pkg/archive/archive_test.go | 4 +- pkg/archive/clusterConfig.go | 4 +- pkg/archive/fsBackend.go | 6 +- pkg/archive/fsBackend_test.go | 4 +- pkg/archive/json.go | 4 +- pkg/archive/nodelist.go | 2 +- pkg/archive/s3Backend.go | 6 +- pkg/archive/s3Backend_test.go | 2 +- pkg/archive/sqliteBackend.go | 6 +- pkg/archive/sqliteBackend_test.go | 2 +- pkg/nats/client.go | 2 +- pkg/nats/config.go | 2 +- pkg/nats/influxDecoder.go | 59 ------------------ tools/archive-manager/import_test.go | 4 +- tools/archive-manager/main.go | 4 +- tools/archive-migration/main.go | 2 +- tools/archive-migration/transforms.go | 2 +- web/web.go | 6 +- web/webConfig_test.go | 2 +- 110 files changed, 261 insertions(+), 311 deletions(-) delete mode 100644 pkg/nats/influxDecoder.go diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index 151eee9e..025396be 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -15,8 +15,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/util" ) const envString = ` diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 9464ccf4..f8b4aea1 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -32,11 +32,11 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/runtimeEnv" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/runtime" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/google/gops/agent" "github.com/joho/godotenv" @@ -371,7 +371,7 @@ func runServer(ctx context.Context) error { case <-ctx.Done(): } - runtimeEnv.SystemdNotifiy(false, "Shutting down ...") + runtime.SystemdNotify(false, "Shutting down ...") srv.Shutdown(ctx) util.FsWatcherShutdown() taskmanager.Shutdown() @@ -381,7 +381,7 @@ func runServer(ctx context.Context) error { if os.Getenv(envGOGC) == "" { debug.SetGCPercent(25) } - runtimeEnv.SystemdNotifiy(true, "running") + runtime.SystemdNotify(true, "running") // Wait for completion or error go func() { diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 4ed79622..53e24c88 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -33,8 +33,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/routerConfig" "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/runtimeEnv" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/runtime" "github.com/gorilla/handlers" "github.com/gorilla/mux" httpSwagger "github.com/swaggo/http-swagger" @@ -347,7 +347,7 @@ func (s *Server) Start(ctx context.Context) error { // Because this program will want to bind to a privileged port (like 80), the listener must // be established first, then the user can be changed, and after that, // the actual http server can be started. - if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { + if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { return fmt.Errorf("dropping privileges: %w", err) } diff --git a/go.mod b/go.mod index b821f7bf..36ce47b9 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,14 @@ tool ( require ( github.com/99designs/gqlgen v0.17.84 - github.com/ClusterCockpit/cc-lib v1.0.2 + github.com/ClusterCockpit/cc-lib/v2 v2.0.0 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.0 github.com/aws/aws-sdk-go-v2/config v1.31.20 github.com/aws/aws-sdk-go-v2/credentials v1.18.24 github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 github.com/coreos/go-oidc/v3 v3.16.0 - github.com/expr-lang/expr v1.17.6 + github.com/expr-lang/expr v1.17.7 github.com/go-co-op/gocron/v2 v2.18.2 github.com/go-ldap/ldap/v3 v3.4.12 github.com/golang-jwt/jwt/v5 v5.3.0 @@ -41,7 +41,7 @@ require ( github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 github.com/vektah/gqlparser/v2 v2.5.31 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 golang.org/x/oauth2 v0.32.0 golang.org/x/time v0.14.0 ) @@ -95,14 +95,14 @@ require ( github.com/jonboulle/clockwork v0.5.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -121,9 +121,9 @@ require ( golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/mod v0.30.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.39.0 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 04e2514b..9038d960 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/99designs/gqlgen v0.17.84 h1:iVMdiStgUVx/BFkMb0J5GAXlqfqtQ7bqMCYK6v52 github.com/99designs/gqlgen v0.17.84/go.mod h1:qjoUqzTeiejdo+bwUg8unqSpeYG42XrcrQboGIezmFA= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/ClusterCockpit/cc-lib v1.0.2 h1:ZWn3oZkXgxrr3zSigBdlOOfayZ4Om4xL20DhmritPPg= -github.com/ClusterCockpit/cc-lib v1.0.2/go.mod h1:UGdOvXEnjFqlnPSxtvtFwO6BtXYW6NnXFoud9FtN93k= +github.com/ClusterCockpit/cc-lib/v2 v2.0.0 h1:OjDADx8mf9SflqeeKUuhy5pamu4YDucae6wUX6vvNNA= +github.com/ClusterCockpit/cc-lib/v2 v2.0.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= @@ -79,8 +79,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec= -github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= +github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= @@ -197,8 +197,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -230,8 +230,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= +github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -298,8 +298,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -315,16 +315,16 @@ golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -332,8 +332,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/gqlgen.yml b/gqlgen.yml index 5f5272b4..40410b48 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -52,51 +52,51 @@ models: - github.com/99designs/gqlgen/graphql.Int64 - github.com/99designs/gqlgen/graphql.Int32 Job: - model: "github.com/ClusterCockpit/cc-lib/schema.Job" + model: "github.com/ClusterCockpit/cc-lib/v2/schema.Job" fields: tags: resolver: true metaData: resolver: true Cluster: - model: "github.com/ClusterCockpit/cc-lib/schema.Cluster" + model: "github.com/ClusterCockpit/cc-lib/v2/schema.Cluster" fields: partitions: resolver: true # Node: - # model: "github.com/ClusterCockpit/cc-lib/schema.Node" + # model: "github.com/ClusterCockpit/cc-lib/v2/schema.Node" # fields: # metaData: # resolver: true - NullableFloat: { model: "github.com/ClusterCockpit/cc-lib/schema.Float" } - MetricScope: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricScope" } - MetricValue: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricValue" } + NullableFloat: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Float" } + MetricScope: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricScope" } + MetricValue: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricValue" } JobStatistics: - { model: "github.com/ClusterCockpit/cc-lib/schema.JobStatistics" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobStatistics" } GlobalMetricListItem: - { model: "github.com/ClusterCockpit/cc-lib/schema.GlobalMetricListItem" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.GlobalMetricListItem" } ClusterSupport: - { model: "github.com/ClusterCockpit/cc-lib/schema.ClusterSupport" } - Tag: { model: "github.com/ClusterCockpit/cc-lib/schema.Tag" } - Resource: { model: "github.com/ClusterCockpit/cc-lib/schema.Resource" } - JobState: { model: "github.com/ClusterCockpit/cc-lib/schema.JobState" } - Node: { model: "github.com/ClusterCockpit/cc-lib/schema.Node" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.ClusterSupport" } + Tag: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Tag" } + Resource: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Resource" } + JobState: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobState" } + Node: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Node" } SchedulerState: - { model: "github.com/ClusterCockpit/cc-lib/schema.SchedulerState" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SchedulerState" } HealthState: - { model: "github.com/ClusterCockpit/cc-lib/schema.MonitoringState" } - JobMetric: { model: "github.com/ClusterCockpit/cc-lib/schema.JobMetric" } - Series: { model: "github.com/ClusterCockpit/cc-lib/schema.Series" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MonitoringState" } + JobMetric: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobMetric" } + Series: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Series" } MetricStatistics: - { model: "github.com/ClusterCockpit/cc-lib/schema.MetricStatistics" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricStatistics" } MetricConfig: - { model: "github.com/ClusterCockpit/cc-lib/schema.MetricConfig" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricConfig" } SubClusterConfig: - { model: "github.com/ClusterCockpit/cc-lib/schema.SubClusterConfig" } - Accelerator: { model: "github.com/ClusterCockpit/cc-lib/schema.Accelerator" } - Topology: { model: "github.com/ClusterCockpit/cc-lib/schema.Topology" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SubClusterConfig" } + Accelerator: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Accelerator" } + Topology: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Topology" } FilterRanges: - { model: "github.com/ClusterCockpit/cc-lib/schema.FilterRanges" } - SubCluster: { model: "github.com/ClusterCockpit/cc-lib/schema.SubCluster" } - StatsSeries: { model: "github.com/ClusterCockpit/cc-lib/schema.StatsSeries" } - Unit: { model: "github.com/ClusterCockpit/cc-lib/schema.Unit" } + { model: "github.com/ClusterCockpit/cc-lib/v2/schema.FilterRanges" } + SubCluster: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SubCluster" } + StatsSeries: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.StatsSeries" } + Unit: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Unit" } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 3030b1c1..50605f7b 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -27,9 +27,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/gorilla/mux" _ "github.com/mattn/go-sqlite3" diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 28d7c109..b6f41244 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -13,7 +13,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // GetClustersAPIResponse model diff --git a/internal/api/job.go b/internal/api/job.go index 919772f4..9b970c2e 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -25,8 +25,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/gorilla/mux" ) diff --git a/internal/api/memorystore.go b/internal/api/memorystore.go index 1b883792..56c396e2 100644 --- a/internal/api/memorystore.go +++ b/internal/api/memorystore.go @@ -16,7 +16,7 @@ import ( "strings" "github.com/ClusterCockpit/cc-backend/internal/memorystore" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/influxdata/line-protocol/v2/lineprotocol" ) diff --git a/internal/api/nats.go b/internal/api/nats.go index a309a915..efd04406 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -18,9 +18,10 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/nats" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - lp "github.com/ClusterCockpit/cc-lib/ccMessage" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" + "github.com/ClusterCockpit/cc-lib/v2/receivers" + "github.com/ClusterCockpit/cc-lib/v2/schema" influx "github.com/influxdata/line-protocol/v2/lineprotocol" ) @@ -75,10 +76,18 @@ func (api *NatsAPI) processJobEvent(msg lp.CCMessage) { switch function { case "start_job": - api.handleStartJob(msg.GetEventValue()) + v, ok := msg.GetEventValue() + if !ok { + cclog.Errorf("Job event is missing event value: %+v", msg) + } + api.handleStartJob(v) case "stop_job": - api.handleStopJob(msg.GetEventValue()) + v, ok := msg.GetEventValue() + if !ok { + cclog.Errorf("Job event is missing event value: %+v", msg) + } + api.handleStopJob(v) default: cclog.Warnf("Unimplemented job event: %+v", msg) } @@ -88,7 +97,7 @@ func (api *NatsAPI) handleJobEvent(subject string, data []byte) { d := influx.NewDecoderWithBytes(data) for d.Next() { - m, err := nats.DecodeInfluxMessage(d) + m, err := receivers.DecodeInfluxMessage(d) if err != nil { cclog.Errorf("NATS %s: Failed to decode message: %v", subject, err) return diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index 420a359c..c9415afc 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -21,10 +21,10 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - lp "github.com/ClusterCockpit/cc-lib/ccMessage" - "github.com/ClusterCockpit/cc-lib/schema" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" + "github.com/ClusterCockpit/cc-lib/v2/schema" _ "github.com/mattn/go-sqlite3" ) diff --git a/internal/api/node.go b/internal/api/node.go index 8953e5b9..350f097d 100644 --- a/internal/api/node.go +++ b/internal/api/node.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/repository" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) type UpdateNodeStatesRequest struct { diff --git a/internal/api/rest.go b/internal/api/rest.go index ebcf31ed..195de826 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -22,9 +22,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/gorilla/mux" ) diff --git a/internal/api/user.go b/internal/api/user.go index f9ddee33..1821b69b 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -11,8 +11,8 @@ import ( "net/http" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/gorilla/mux" ) diff --git a/internal/archiver/archiveWorker.go b/internal/archiver/archiveWorker.go index 0434844d..ecdd1756 100644 --- a/internal/archiver/archiveWorker.go +++ b/internal/archiver/archiveWorker.go @@ -54,8 +54,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index b88199aa..46ce8126 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -12,8 +12,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // ArchiveJob archives a completed job's metric data to the configured archive backend. diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 5d947353..3be1768e 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -25,9 +25,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/gorilla/sessions" ) diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index 4f1f3f54..be642219 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -14,8 +14,8 @@ import ( "strings" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/golang-jwt/jwt/v5" ) diff --git a/internal/auth/jwtCookieSession.go b/internal/auth/jwtCookieSession.go index 44c64a0c..42f7439e 100644 --- a/internal/auth/jwtCookieSession.go +++ b/internal/auth/jwtCookieSession.go @@ -12,8 +12,8 @@ import ( "net/http" "os" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/golang-jwt/jwt/v5" ) diff --git a/internal/auth/jwtHelpers.go b/internal/auth/jwtHelpers.go index 792722a8..5bfc91ef 100644 --- a/internal/auth/jwtHelpers.go +++ b/internal/auth/jwtHelpers.go @@ -11,8 +11,8 @@ import ( "fmt" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/golang-jwt/jwt/v5" ) diff --git a/internal/auth/jwtHelpers_test.go b/internal/auth/jwtHelpers_test.go index 5cee1df5..84a1f2e0 100644 --- a/internal/auth/jwtHelpers_test.go +++ b/internal/auth/jwtHelpers_test.go @@ -8,7 +8,7 @@ package auth import ( "testing" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/golang-jwt/jwt/v5" ) diff --git a/internal/auth/jwtSession.go b/internal/auth/jwtSession.go index 15e58347..107afcb8 100644 --- a/internal/auth/jwtSession.go +++ b/internal/auth/jwtSession.go @@ -13,8 +13,8 @@ import ( "os" "strings" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/golang-jwt/jwt/v5" ) diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index e96e732b..4cbb80c5 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -13,8 +13,8 @@ import ( "strings" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/go-ldap/ldap/v3" ) diff --git a/internal/auth/local.go b/internal/auth/local.go index 1c9b0372..b1a7362c 100644 --- a/internal/auth/local.go +++ b/internal/auth/local.go @@ -9,8 +9,8 @@ import ( "fmt" "net/http" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "golang.org/x/crypto/bcrypt" ) diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go index 9e361302..a3fc09cc 100644 --- a/internal/auth/oidc.go +++ b/internal/auth/oidc.go @@ -15,8 +15,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/coreos/go-oidc/v3/oidc" "github.com/gorilla/mux" "golang.org/x/oauth2" diff --git a/internal/config/config.go b/internal/config/config.go index 3c88bcfd..af8ec944 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,8 +11,8 @@ import ( "encoding/json" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/resampler" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/resampler" ) type ProgramConfig struct { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 35e1c65e..396a80a1 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -8,8 +8,8 @@ package config import ( "testing" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func TestInit(t *testing.T) { diff --git a/internal/config/validate.go b/internal/config/validate.go index 6ac67f5e..af8591ca 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -8,7 +8,7 @@ package config import ( "encoding/json" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/santhosh-tekuri/jsonschema/v5" ) diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 1cb348e5..d96ccf1d 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -16,7 +16,7 @@ import ( "github.com/99designs/gqlgen/graphql/introspection" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" gqlparser "github.com/vektah/gqlparser/v2" "github.com/vektah/gqlparser/v2/ast" ) diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 63b2da5d..31ba03ab 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -10,7 +10,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) type ClusterMetricWithName struct { diff --git a/internal/graph/resolver.go b/internal/graph/resolver.go index 990014c7..d1b04de6 100644 --- a/internal/graph/resolver.go +++ b/internal/graph/resolver.go @@ -4,7 +4,7 @@ import ( "sync" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/jmoiron/sqlx" ) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index cd4af057..32499b8c 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -22,8 +22,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // Partitions is the resolver for the partitions field. diff --git a/internal/graph/util.go b/internal/graph/util.go index 220c3a84..42a1d2fb 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -14,8 +14,8 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) const MAX_JOBS_FOR_ANALYSIS = 500 diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go index 482b328c..4b217475 100644 --- a/internal/importer/handleImport.go +++ b/internal/importer/handleImport.go @@ -14,8 +14,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // HandleImportFlag imports jobs from file pairs specified in a comma-separated flag string. diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 470f7603..bffb8bf6 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -16,8 +16,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) // copyFile copies a file from source path to destination path. diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index 12f49010..d88be7c7 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -22,8 +22,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) const ( diff --git a/internal/importer/normalize.go b/internal/importer/normalize.go index 943ceb26..c6e84d4b 100644 --- a/internal/importer/normalize.go +++ b/internal/importer/normalize.go @@ -7,7 +7,7 @@ package importer import ( "math" - ccunits "github.com/ClusterCockpit/cc-lib/ccUnits" + ccunits "github.com/ClusterCockpit/cc-lib/v2/ccUnits" ) // getNormalizationFactor calculates the scaling factor needed to normalize a value diff --git a/internal/importer/normalize_test.go b/internal/importer/normalize_test.go index 6aa1ed2e..039a3cfc 100644 --- a/internal/importer/normalize_test.go +++ b/internal/importer/normalize_test.go @@ -8,7 +8,7 @@ import ( "fmt" "testing" - ccunits "github.com/ClusterCockpit/cc-lib/ccUnits" + ccunits "github.com/ClusterCockpit/cc-lib/v2/ccUnits" ) // TestNormalizeFactor tests the normalization of large byte values to gigabyte prefix. diff --git a/internal/memorystore/api.go b/internal/memorystore/api.go index b96dc1fd..41c53a18 100644 --- a/internal/memorystore/api.go +++ b/internal/memorystore/api.go @@ -9,8 +9,8 @@ import ( "errors" "math" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) var ( diff --git a/internal/memorystore/archive.go b/internal/memorystore/archive.go index 5019ee7a..fc46dac6 100644 --- a/internal/memorystore/archive.go +++ b/internal/memorystore/archive.go @@ -18,7 +18,7 @@ import ( "sync/atomic" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func Archiving(wg *sync.WaitGroup, ctx context.Context) { diff --git a/internal/memorystore/avroCheckpoint.go b/internal/memorystore/avroCheckpoint.go index 42e5f623..b0b0cf42 100644 --- a/internal/memorystore/avroCheckpoint.go +++ b/internal/memorystore/avroCheckpoint.go @@ -19,8 +19,8 @@ import ( "sync/atomic" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/linkedin/goavro/v2" ) diff --git a/internal/memorystore/avroHelper.go b/internal/memorystore/avroHelper.go index a6f6c9bf..93a293bd 100644 --- a/internal/memorystore/avroHelper.go +++ b/internal/memorystore/avroHelper.go @@ -11,7 +11,7 @@ import ( "strconv" "sync" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func DataStaging(wg *sync.WaitGroup, ctx context.Context) { diff --git a/internal/memorystore/avroStruct.go b/internal/memorystore/avroStruct.go index bde9e02b..2643a9a7 100644 --- a/internal/memorystore/avroStruct.go +++ b/internal/memorystore/avroStruct.go @@ -8,7 +8,7 @@ package memorystore import ( "sync" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) var ( diff --git a/internal/memorystore/buffer.go b/internal/memorystore/buffer.go index 55be2ada..15e29b3a 100644 --- a/internal/memorystore/buffer.go +++ b/internal/memorystore/buffer.go @@ -9,7 +9,7 @@ import ( "errors" "sync" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // BufferCap is the default buffer capacity. diff --git a/internal/memorystore/checkpoint.go b/internal/memorystore/checkpoint.go index c676977c..c48c2fd8 100644 --- a/internal/memorystore/checkpoint.go +++ b/internal/memorystore/checkpoint.go @@ -23,8 +23,8 @@ import ( "sync/atomic" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/linkedin/goavro/v2" ) diff --git a/internal/memorystore/level.go b/internal/memorystore/level.go index f3b3d3f5..bce2a7a6 100644 --- a/internal/memorystore/level.go +++ b/internal/memorystore/level.go @@ -9,7 +9,7 @@ import ( "sync" "unsafe" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/util" ) // Could also be called "node" as this forms a node in a tree structure. diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index 6404361f..ca8cc811 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -12,8 +12,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/nats" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/influxdata/line-protocol/v2/lineprotocol" ) diff --git a/internal/memorystore/memorystore.go b/internal/memorystore/memorystore.go index 259a86ed..7c5ea0eb 100644 --- a/internal/memorystore/memorystore.go +++ b/internal/memorystore/memorystore.go @@ -30,10 +30,10 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/resampler" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/resampler" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) var ( diff --git a/internal/memorystore/memorystore_test.go b/internal/memorystore/memorystore_test.go index b8ab090a..57ea6938 100644 --- a/internal/memorystore/memorystore_test.go +++ b/internal/memorystore/memorystore_test.go @@ -8,7 +8,7 @@ package memorystore import ( "testing" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) func TestAssignAggregationStrategy(t *testing.T) { diff --git a/internal/memorystore/stats.go b/internal/memorystore/stats.go index b2cb539a..c931ab35 100644 --- a/internal/memorystore/stats.go +++ b/internal/memorystore/stats.go @@ -9,7 +9,7 @@ import ( "errors" "math" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/util" ) type Stats struct { diff --git a/internal/metricDataDispatcher/dataLoader.go b/internal/metricDataDispatcher/dataLoader.go index 780eb73e..6d1338fa 100644 --- a/internal/metricDataDispatcher/dataLoader.go +++ b/internal/metricDataDispatcher/dataLoader.go @@ -13,10 +13,10 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/lrucache" - "github.com/ClusterCockpit/cc-lib/resampler" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/resampler" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) diff --git a/internal/metricdata/cc-metric-store-internal.go b/internal/metricdata/cc-metric-store-internal.go index 9f0cd74a..741ce358 100644 --- a/internal/metricdata/cc-metric-store-internal.go +++ b/internal/metricdata/cc-metric-store-internal.go @@ -15,8 +15,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/memorystore" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // Bloat Code diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index be2e956e..6c146f22 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -15,8 +15,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) type CCMetricStoreConfig struct { diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 0748a8d5..ab0e19fb 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -13,8 +13,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/memorystore" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) type MetricDataRepository interface { diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index 66c5bc1e..3fb94d51 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -21,8 +21,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" promapi "github.com/prometheus/client_golang/api" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" promcfg "github.com/prometheus/common/config" diff --git a/internal/metricdata/utils.go b/internal/metricdata/utils.go index 0b2bb7ec..21dfbcac 100644 --- a/internal/metricdata/utils.go +++ b/internal/metricdata/utils.go @@ -10,7 +10,7 @@ import ( "encoding/json" "time" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index be0b161b..3141cf86 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -12,7 +12,7 @@ import ( "sync" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/jmoiron/sqlx" "github.com/mattn/go-sqlite3" "github.com/qustavo/sqlhooks/v2" diff --git a/internal/repository/hooks.go b/internal/repository/hooks.go index 54330723..c916b57e 100644 --- a/internal/repository/hooks.go +++ b/internal/repository/hooks.go @@ -8,7 +8,7 @@ import ( "context" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) // Hooks satisfies the sqlhook.Hooks interface diff --git a/internal/repository/job.go b/internal/repository/job.go index 47959379..99970ce1 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -71,9 +71,9 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/lrucache" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" ) diff --git a/internal/repository/jobCreate.go b/internal/repository/jobCreate.go index efd262b8..6114ae5e 100644 --- a/internal/repository/jobCreate.go +++ b/internal/repository/jobCreate.go @@ -9,8 +9,8 @@ import ( "encoding/json" "fmt" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index c4051e7f..ff2c27aa 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -12,8 +12,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/repository/jobHooks.go b/internal/repository/jobHooks.go index 824b5cde..c449d308 100644 --- a/internal/repository/jobHooks.go +++ b/internal/repository/jobHooks.go @@ -7,7 +7,7 @@ package repository import ( "sync" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) type JobHook interface { diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 00dabea3..8c341afb 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -14,8 +14,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index c89225b3..17766c69 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" _ "github.com/mattn/go-sqlite3" ) diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 43e913cc..a47f9fcd 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -10,7 +10,7 @@ import ( "embed" "fmt" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/sqlite3" "github.com/golang-migrate/migrate/v4/source/iofs" diff --git a/internal/repository/node.go b/internal/repository/node.go index 3b597eda..752a36fa 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -17,9 +17,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/lrucache" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" ) diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index 466f51ee..e1d6ca93 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -15,9 +15,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" _ "github.com/mattn/go-sqlite3" ) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index e3dec7fc..9d07b026 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" _ "github.com/mattn/go-sqlite3" ) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index c92f5193..d1e16eb8 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -14,8 +14,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 8a076e8a..9bc9abae 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -11,8 +11,8 @@ import ( "strings" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" ) diff --git a/internal/repository/testdata/job.db b/internal/repository/testdata/job.db index 5c5a692585a4736a9e0456d1caf7b4878c936f74..729cac965265a81bf187475d23d21de268a42ccd 100644 GIT binary patch delta 914 zcma)4O=}ZT6n$^r?U+fEUfYnG7-A{`$-=;VDY$Se z#O{lKKrvu9Lfwix7t*>hG!#i8Vy#+Rvhcp7q_I_Sao(HZoO3@8Z)I(IWo`Ne=qkq+ z!&YNl-@;^st1J=^u*GuKHO!P-s=7sIsp^&tQH3)RX%XGxuv%9-iDZdN(d%W&P}W~r=gXVQjc+vXq5P8X?f-n!}otTX(EukkI` zX^%%(OF(pv|Hl&GgFuiUPb&WDIyH^rhd@{(S7q5$co=8{2qDyS~Q@D(N|Zhe~oPDSH2k z(m1S*PSJa4Gi;Oa6JDN#NL)Mn807dIn+%&I`+!rIxR+5$#S;r^_ijFm-{8%HQv47>Ofzv3<_KjOrQkU7dz2ze^XWjO@`bTufF zk`+7x;!9?pcyAMrHt{H>AfZ%gLpycl{cSQdG?h>8b^Fa(7>4WBe7R@~AtV=q3*%{d z=Y6sCYm!UBrSW3*;H-??!M`U~mQNQN8^YzEQ$;$VKw{zhAF*{nmXUt3c}LO-&}0SM fUsDxRTboO$KotlS)SwOxXo3SSw4iNl-l6FS=Yv%5 diff --git a/internal/repository/user.go b/internal/repository/user.go index 5cab2b0d..770915b6 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -15,8 +15,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" "golang.org/x/crypto/bcrypt" diff --git a/internal/repository/userConfig.go b/internal/repository/userConfig.go index beeffbf5..75e7119f 100644 --- a/internal/repository/userConfig.go +++ b/internal/repository/userConfig.go @@ -12,9 +12,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/web" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/lrucache" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/jmoiron/sqlx" ) diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index b6f68430..02c70d0f 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -10,9 +10,9 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/internal/config" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" _ "github.com/mattn/go-sqlite3" ) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 4466034d..436031ef 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -17,9 +17,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/web" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/gorilla/mux" ) diff --git a/internal/tagger/classifyJob.go b/internal/tagger/classifyJob.go index 4e46f370..70399218 100644 --- a/internal/tagger/classifyJob.go +++ b/internal/tagger/classifyJob.go @@ -16,9 +16,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" ) diff --git a/internal/tagger/classifyJob_test.go b/internal/tagger/classifyJob_test.go index 3795a60a..bed7a8f0 100644 --- a/internal/tagger/classifyJob_test.go +++ b/internal/tagger/classifyJob_test.go @@ -3,7 +3,7 @@ package tagger import ( "testing" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/internal/tagger/detectApp.go b/internal/tagger/detectApp.go index 4e8f858d..0b8e3e7e 100644 --- a/internal/tagger/detectApp.go +++ b/internal/tagger/detectApp.go @@ -16,9 +16,9 @@ import ( "strings" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) //go:embed apps/* diff --git a/internal/tagger/detectApp_test.go b/internal/tagger/detectApp_test.go index 7145d04f..1c44f670 100644 --- a/internal/tagger/detectApp_test.go +++ b/internal/tagger/detectApp_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func setup(tb testing.TB) *repository.JobRepository { diff --git a/internal/tagger/tagger.go b/internal/tagger/tagger.go index 2ba18a14..0839603d 100644 --- a/internal/tagger/tagger.go +++ b/internal/tagger/tagger.go @@ -13,8 +13,8 @@ import ( "sync" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // Tagger is the interface that must be implemented by all tagging components. diff --git a/internal/tagger/tagger_test.go b/internal/tagger/tagger_test.go index fb4bc54e..d24ad7f7 100644 --- a/internal/tagger/tagger_test.go +++ b/internal/tagger/tagger_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/internal/repository" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) func TestInit(t *testing.T) { diff --git a/internal/taskmanager/commitJobService.go b/internal/taskmanager/commitJobService.go index 4f21c86b..4a070284 100644 --- a/internal/taskmanager/commitJobService.go +++ b/internal/taskmanager/commitJobService.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/compressionService.go b/internal/taskmanager/compressionService.go index 1da2f68d..ab01ce8f 100644 --- a/internal/taskmanager/compressionService.go +++ b/internal/taskmanager/compressionService.go @@ -9,8 +9,8 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/ldapSyncService.go b/internal/taskmanager/ldapSyncService.go index e410af9e..9e99a261 100644 --- a/internal/taskmanager/ldapSyncService.go +++ b/internal/taskmanager/ldapSyncService.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/auth" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/retentionService.go b/internal/taskmanager/retentionService.go index acd07307..5678cd14 100644 --- a/internal/taskmanager/retentionService.go +++ b/internal/taskmanager/retentionService.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/stopJobsExceedTime.go b/internal/taskmanager/stopJobsExceedTime.go index b763f561..ce9cfd77 100644 --- a/internal/taskmanager/stopJobsExceedTime.go +++ b/internal/taskmanager/stopJobsExceedTime.go @@ -9,7 +9,7 @@ import ( "runtime" "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/taskManager.go b/internal/taskmanager/taskManager.go index 57f2d883..06e4f28f 100644 --- a/internal/taskmanager/taskManager.go +++ b/internal/taskmanager/taskManager.go @@ -13,7 +13,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/updateDurationService.go b/internal/taskmanager/updateDurationService.go index 9c52da79..f1dde74a 100644 --- a/internal/taskmanager/updateDurationService.go +++ b/internal/taskmanager/updateDurationService.go @@ -8,7 +8,7 @@ package taskmanager import ( "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/go-co-op/gocron/v2" ) diff --git a/internal/taskmanager/updateFootprintService.go b/internal/taskmanager/updateFootprintService.go index ae9512cd..979a6137 100644 --- a/internal/taskmanager/updateFootprintService.go +++ b/internal/taskmanager/updateFootprintService.go @@ -12,8 +12,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/go-co-op/gocron/v2" ) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 71933f2b..f9ce4314 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -85,9 +85,9 @@ import ( "sync" "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/lrucache" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) // Version is the current archive schema version. diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index 34ea831a..595315c3 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -11,8 +11,8 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) var jobs []*schema.Job diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 696601b7..6e4866eb 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -8,8 +8,8 @@ package archive import ( "fmt" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) var ( diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index b8d2a94b..020f2aa4 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -23,9 +23,9 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/santhosh-tekuri/jsonschema/v5" ) diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go index a43a6c3a..05491f61 100644 --- a/pkg/archive/fsBackend_test.go +++ b/pkg/archive/fsBackend_test.go @@ -10,8 +10,8 @@ import ( "path/filepath" "testing" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) func TestInitEmptyPath(t *testing.T) { diff --git a/pkg/archive/json.go b/pkg/archive/json.go index 75c39531..cf1b0a38 100644 --- a/pkg/archive/json.go +++ b/pkg/archive/json.go @@ -10,8 +10,8 @@ import ( "io" "time" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { diff --git a/pkg/archive/nodelist.go b/pkg/archive/nodelist.go index ffb5f563..7a3784c3 100644 --- a/pkg/archive/nodelist.go +++ b/pkg/archive/nodelist.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) type NodeList [][]interface { diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index c874a320..a9933a9f 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -22,9 +22,9 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" diff --git a/pkg/archive/s3Backend_test.go b/pkg/archive/s3Backend_test.go index 06324cd3..2b79db7f 100644 --- a/pkg/archive/s3Backend_test.go +++ b/pkg/archive/s3Backend_test.go @@ -13,7 +13,7 @@ import ( "strings" "testing" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 0b7a22d2..5bce9cea 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -21,9 +21,9 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" _ "github.com/mattn/go-sqlite3" ) diff --git a/pkg/archive/sqliteBackend_test.go b/pkg/archive/sqliteBackend_test.go index b72b8f6c..5d05e14e 100644 --- a/pkg/archive/sqliteBackend_test.go +++ b/pkg/archive/sqliteBackend_test.go @@ -9,7 +9,7 @@ import ( "os" "testing" - "github.com/ClusterCockpit/cc-lib/schema" + "github.com/ClusterCockpit/cc-lib/v2/schema" ) func TestSqliteInitEmptyPath(t *testing.T) { diff --git a/pkg/nats/client.go b/pkg/nats/client.go index 822a7b26..a32ebdca 100644 --- a/pkg/nats/client.go +++ b/pkg/nats/client.go @@ -54,7 +54,7 @@ import ( "fmt" "sync" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/nats-io/nats.go" ) diff --git a/pkg/nats/config.go b/pkg/nats/config.go index 32a0bbda..c9ab48a5 100644 --- a/pkg/nats/config.go +++ b/pkg/nats/config.go @@ -9,7 +9,7 @@ import ( "bytes" "encoding/json" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) // NatsConfig holds the configuration for connecting to a NATS server. diff --git a/pkg/nats/influxDecoder.go b/pkg/nats/influxDecoder.go deleted file mode 100644 index 412f85e9..00000000 --- a/pkg/nats/influxDecoder.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package nats - -import ( - "time" - - lp "github.com/ClusterCockpit/cc-lib/ccMessage" - influx "github.com/influxdata/line-protocol/v2/lineprotocol" -) - -// DecodeInfluxMessage decodes a single InfluxDB line protocol message from the decoder -// Returns the decoded CCMessage or an error if decoding fails -func DecodeInfluxMessage(d *influx.Decoder) (lp.CCMessage, error) { - measurement, err := d.Measurement() - if err != nil { - return nil, err - } - - tags := make(map[string]string) - for { - key, value, err := d.NextTag() - if err != nil { - return nil, err - } - if key == nil { - break - } - tags[string(key)] = string(value) - } - - fields := make(map[string]interface{}) - for { - key, value, err := d.NextField() - if err != nil { - return nil, err - } - if key == nil { - break - } - fields[string(key)] = value.Interface() - } - - t, err := d.Time(influx.Nanosecond, time.Time{}) - if err != nil { - return nil, err - } - - return lp.NewMessage( - string(measurement), - tags, - nil, - fields, - t, - ) -} diff --git a/tools/archive-manager/import_test.go b/tools/archive-manager/import_test.go index b1032118..57294d50 100644 --- a/tools/archive-manager/import_test.go +++ b/tools/archive-manager/import_test.go @@ -12,8 +12,8 @@ import ( "testing" "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) // TestImportFileToSqlite tests importing jobs from file backend to SQLite backend diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 4972fe96..f5f8b836 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -23,8 +23,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func parseDate(in string) int64 { diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go index 9bbed121..8375ee98 100644 --- a/tools/archive-migration/main.go +++ b/tools/archive-migration/main.go @@ -12,7 +12,7 @@ import ( "path/filepath" "strings" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) func main() { diff --git a/tools/archive-migration/transforms.go b/tools/archive-migration/transforms.go index 6558e47a..ef4ba5eb 100644 --- a/tools/archive-migration/transforms.go +++ b/tools/archive-migration/transforms.go @@ -12,7 +12,7 @@ import ( "sync" "sync/atomic" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) // transformExclusiveToShared converts the old 'exclusive' field to the new 'shared' field diff --git a/web/web.go b/web/web.go index 31d7002e..d2ae8700 100644 --- a/web/web.go +++ b/web/web.go @@ -16,9 +16,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/ccLogger" - "github.com/ClusterCockpit/cc-lib/schema" - "github.com/ClusterCockpit/cc-lib/util" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" ) type WebConfig struct { diff --git a/web/webConfig_test.go b/web/webConfig_test.go index 4bd84330..514fdabb 100644 --- a/web/webConfig_test.go +++ b/web/webConfig_test.go @@ -10,7 +10,7 @@ import ( "fmt" "testing" - ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" ) func TestInit(t *testing.T) { From 11ec2267daaf67cbc6602e3ba395a16ede96503e Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 25 Dec 2025 08:42:54 +0100 Subject: [PATCH 056/341] Major refactor of metric data handling - make the internal memory store required and default - Rename memorystore to metricstore - Rename metricDataDispatcher to metricdispatch - Remove metricdata package - Introduce metricsync package for upstream metric data pull --- CLAUDE.md | 10 +- README.md | 8 +- cmd/cc-backend/main.go | 21 +- cmd/cc-backend/server.go | 10 +- internal/api/api_test.go | 12 +- internal/api/job.go | 6 +- .../api/{memorystore.go => metricstore.go} | 12 +- internal/api/nats_test.go | 9 +- internal/archiver/README.md | 4 +- internal/archiver/archiver.go | 4 +- internal/graph/schema.resolvers.go | 16 +- internal/graph/util.go | 6 +- internal/metricDataDispatcher/dataLoader.go | 381 ----- internal/metricdata/cc-metric-store.go | 1226 ----------------- internal/metricdata/metricdata.go | 88 -- internal/metricdata/prometheus.go | 587 -------- internal/metricdata/utils.go | 118 -- internal/metricdispatch/dataLoader.go | 490 +++++++ internal/metricdispatch/dataLoader_test.go | 125 ++ internal/{memorystore => metricstore}/api.go | 6 +- .../{memorystore => metricstore}/archive.go | 2 +- .../avroCheckpoint.go | 2 +- .../avroHelper.go | 2 +- .../avroStruct.go | 2 +- .../{memorystore => metricstore}/buffer.go | 2 +- .../checkpoint.go | 2 +- .../{memorystore => metricstore}/config.go | 4 +- .../configSchema.go | 2 +- .../{memorystore => metricstore}/debug.go | 2 +- .../healthcheck.go | 2 +- .../{memorystore => metricstore}/level.go | 2 +- .../lineprotocol.go | 2 +- .../memorystore.go | 4 +- .../memorystore_test.go | 2 +- .../query.go} | 146 +- .../{memorystore => metricstore}/stats.go | 2 +- internal/metricsync/metricdata.go | 60 + internal/repository/stats.go | 4 +- .../taskmanager/updateFootprintService.go | 10 +- 39 files changed, 815 insertions(+), 2578 deletions(-) rename internal/api/{memorystore.go => metricstore.go} (95%) delete mode 100644 internal/metricDataDispatcher/dataLoader.go delete mode 100644 internal/metricdata/cc-metric-store.go delete mode 100644 internal/metricdata/metricdata.go delete mode 100644 internal/metricdata/prometheus.go delete mode 100644 internal/metricdata/utils.go create mode 100644 internal/metricdispatch/dataLoader.go create mode 100644 internal/metricdispatch/dataLoader_test.go rename internal/{memorystore => metricstore}/api.go (98%) rename internal/{memorystore => metricstore}/archive.go (99%) rename internal/{memorystore => metricstore}/avroCheckpoint.go (99%) rename internal/{memorystore => metricstore}/avroHelper.go (99%) rename internal/{memorystore => metricstore}/avroStruct.go (99%) rename internal/{memorystore => metricstore}/buffer.go (99%) rename internal/{memorystore => metricstore}/checkpoint.go (99%) rename internal/{memorystore => metricstore}/config.go (98%) rename internal/{memorystore => metricstore}/configSchema.go (99%) rename internal/{memorystore => metricstore}/debug.go (99%) rename internal/{memorystore => metricstore}/healthcheck.go (99%) rename internal/{memorystore => metricstore}/level.go (99%) rename internal/{memorystore => metricstore}/lineprotocol.go (99%) rename internal/{memorystore => metricstore}/memorystore.go (99%) rename internal/{memorystore => metricstore}/memorystore_test.go (99%) rename internal/{metricdata/cc-metric-store-internal.go => metricstore/query.go} (87%) rename internal/{memorystore => metricstore}/stats.go (99%) create mode 100644 internal/metricsync/metricdata.go diff --git a/CLAUDE.md b/CLAUDE.md index 67412a76..f30c3923 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,9 +96,9 @@ The backend follows a layered architecture with clear separation of concerns: - **internal/auth**: Authentication layer - Supports local accounts, LDAP, OIDC, and JWT tokens - Implements rate limiting for login attempts -- **internal/metricdata**: Metric data repository abstraction - - Pluggable backends: cc-metric-store, Prometheus, InfluxDB - - Each cluster can have a different metric data backend +- **internal/metricstore**: Metric store with data loading API + - In-memory metric storage with checkpointing + - Query API for loading job metric data - **internal/archiver**: Job archiving to file-based archive - **internal/api/nats.go**: NATS-based API for job and node operations - Subscribes to NATS subjects for job events (start/stop) @@ -209,8 +209,8 @@ applied automatically on startup. Version tracking in `version` table. ### Adding a new metric data backend -1. Implement `MetricDataRepository` interface in `internal/metricdata/` -2. Register in `metricdata.Init()` switch statement +1. Implement metric loading functions in `internal/metricstore/query.go` +2. Add cluster configuration to metric store initialization 3. Update config.json schema documentation ### Modifying database schema diff --git a/README.md b/README.md index 468a12ad..00bcb119 100644 --- a/README.md +++ b/README.md @@ -163,11 +163,9 @@ ln -s ./var/job-archive GraphQL schema and resolvers - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer) Job data import and database initialization - - [`memorystore`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/memorystore) - In-memory metric data store with checkpointing - - [`metricdata`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdata) - Metric data repository implementations (cc-metric-store, Prometheus) - - [`metricDataDispatcher`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricDataDispatcher) + - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstore) + In-memory metric data store with checkpointing and metric loading + - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdispatch) Dispatches metric data loading to appropriate backends - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository) Database repository layer for jobs and metadata diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index f8b4aea1..331df4f6 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -24,8 +24,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/importer" - "github.com/ClusterCockpit/cc-backend/internal/memorystore" - "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/internal/tagger" "github.com/ClusterCockpit/cc-backend/internal/taskmanager" @@ -283,10 +282,7 @@ func initSubsystems() error { return fmt.Errorf("initializing archive: %w", err) } - // Initialize metricdata - if err := metricdata.Init(); err != nil { - return fmt.Errorf("initializing metricdata repository: %w", err) - } + // Note: metricstore.Init() is called later in runServer() with proper configuration // Handle database re-initialization if flagReinitDB { @@ -322,13 +318,12 @@ func initSubsystems() error { func runServer(ctx context.Context) error { var wg sync.WaitGroup - // Start metric store if enabled - if memorystore.InternalCCMSFlag { - mscfg := ccconf.GetPackageConfig("metric-store") - if mscfg == nil { - return fmt.Errorf("metric store configuration must be present") - } - memorystore.Init(mscfg, &wg) + // Initialize metric store if configuration is provided + mscfg := ccconf.GetPackageConfig("metric-store") + if mscfg != nil { + metricstore.Init(mscfg, &wg) + } else { + cclog.Debug("Metric store configuration not found, skipping metricstore initialization") } // Start archiver and task manager diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 53e24c88..8d700823 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -29,7 +29,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/generated" - "github.com/ClusterCockpit/cc-backend/internal/memorystore" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" @@ -253,9 +253,7 @@ func (s *Server) init() error { } } - if memorystore.InternalCCMSFlag { - s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi) - } + s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi) if config.Keys.EmbedStaticFiles { if i, err := os.Stat("./var/img"); err == nil { @@ -383,9 +381,7 @@ func (s *Server) Shutdown(ctx context.Context) { } // Archive all the metric store data - if memorystore.InternalCCMSFlag { - memorystore.Shutdown() - } + metricstore.Shutdown() // Shutdown archiver with 10 second timeout for fast shutdown if err := archiver.Shutdown(10 * time.Second); err != nil { diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 50605f7b..a2283013 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -23,8 +23,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" - "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" @@ -173,9 +173,7 @@ func setup(t *testing.T) *api.RestAPI { t.Fatal(err) } - if err := metricdata.Init(); err != nil { - t.Fatal(err) - } + // metricstore initialization removed - it's initialized via callback in tests archiver.Start(repository.GetJobRepository(), context.Background()) @@ -221,7 +219,7 @@ func TestRestApi(t *testing.T) { }, } - metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { + metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { return testData, nil } @@ -366,7 +364,7 @@ func TestRestApi(t *testing.T) { } t.Run("CheckArchive", func(t *testing.T) { - data, err := metricDataDispatcher.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60) + data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60) if err != nil { t.Fatal(err) } diff --git a/internal/api/job.go b/internal/api/job.go index 9b970c2e..09f7b22c 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -22,7 +22,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/importer" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" @@ -293,7 +293,7 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request) } if r.URL.Query().Get("all-metrics") == "true" { - data, err = metricDataDispatcher.LoadData(job, nil, scopes, r.Context(), resolution) + data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution) if err != nil { cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster) return @@ -389,7 +389,7 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) { resolution = max(resolution, mc.Timestep) } - data, err := metricDataDispatcher.LoadData(job, metrics, scopes, r.Context(), resolution) + data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution) if err != nil { cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster) return diff --git a/internal/api/memorystore.go b/internal/api/metricstore.go similarity index 95% rename from internal/api/memorystore.go rename to internal/api/metricstore.go index 56c396e2..d4ab1dfe 100644 --- a/internal/api/memorystore.go +++ b/internal/api/metricstore.go @@ -15,7 +15,7 @@ import ( "strconv" "strings" - "github.com/ClusterCockpit/cc-backend/internal/memorystore" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/influxdata/line-protocol/v2/lineprotocol" @@ -58,7 +58,7 @@ func freeMetrics(rw http.ResponseWriter, r *http.Request) { return } - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() n := 0 for _, sel := range selectors { bn, err := ms.Free(sel, to) @@ -97,9 +97,9 @@ func writeMetrics(rw http.ResponseWriter, r *http.Request) { return } - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() dec := lineprotocol.NewDecoderWithBytes(bytes) - if err := memorystore.DecodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { + if err := metricstore.DecodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { cclog.Errorf("/api/write error: %s", err.Error()) handleError(err, http.StatusBadRequest, rw) return @@ -129,7 +129,7 @@ func debugMetrics(rw http.ResponseWriter, r *http.Request) { selector = strings.Split(raw, ":") } - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil { handleError(err, http.StatusBadRequest, rw) return @@ -162,7 +162,7 @@ func metricsHealth(rw http.ResponseWriter, r *http.Request) { selector := []string{rawCluster, rawNode} - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() if err := ms.HealthCheck(bufio.NewWriter(rw), selector); err != nil { handleError(err, http.StatusBadRequest, rw) return diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index c9415afc..9e1fa2b5 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -18,7 +18,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph" - "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/internal/importer" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" @@ -167,9 +168,7 @@ func setupNatsTest(t *testing.T) *NatsAPI { t.Fatal(err) } - if err := metricdata.Init(); err != nil { - t.Fatal(err) - } + // metricstore initialization removed - it's initialized via callback in tests archiver.Start(repository.GetJobRepository(), context.Background()) @@ -564,7 +563,7 @@ func TestNatsHandleStopJob(t *testing.T) { }, } - metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { + metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { return testData, nil } diff --git a/internal/archiver/README.md b/internal/archiver/README.md index 0fae04ea..48aed797 100644 --- a/internal/archiver/README.md +++ b/internal/archiver/README.md @@ -106,7 +106,7 @@ Data is archived at the highest available resolution (typically 60s intervals). ```go // In archiver.go ArchiveJob() function -jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 300) +jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 300) // 0 = highest resolution // 300 = 5-minute resolution ``` @@ -185,6 +185,6 @@ Internal state is protected by: ## Dependencies - `internal/repository`: Database operations for job metadata -- `internal/metricDataDispatcher`: Loading metric data from various backends +- `internal/metricdispatch`: Loading metric data from various backends - `pkg/archive`: Archive backend abstraction (filesystem, S3, SQLite) - `cc-lib/schema`: Job and metric data structures diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 46ce8126..4e0b6473 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -10,7 +10,7 @@ import ( "math" "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" @@ -60,7 +60,7 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) { scopes = append(scopes, schema.MetricScopeAccelerator) } - jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s) + jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s) if err != nil { cclog.Error("Error wile loading job data for archiving") return nil, err diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 32499b8c..34bbf393 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -19,7 +19,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/generated" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" @@ -484,7 +484,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str return nil, err } - data, err := metricDataDispatcher.LoadData(job, metrics, scopes, ctx, *resolution) + data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution) if err != nil { cclog.Warn("Error while loading job data") return nil, err @@ -512,7 +512,7 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin return nil, err } - data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx) + data, err := metricdispatch.LoadJobStats(job, metrics, ctx) if err != nil { cclog.Warnf("Error while loading jobStats data for job id %s", id) return nil, err @@ -537,7 +537,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ return nil, err } - data, err := metricDataDispatcher.LoadScopedJobStats(job, metrics, scopes, ctx) + data, err := metricdispatch.LoadScopedJobStats(job, metrics, scopes, ctx) if err != nil { cclog.Warnf("Error while loading scopedJobStats data for job id %s", id) return nil, err @@ -702,7 +702,7 @@ func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.Job res := []*model.JobStats{} for _, job := range jobs { - data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx) + data, err := metricdispatch.LoadJobStats(job, metrics, ctx) if err != nil { cclog.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID) continue @@ -759,7 +759,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [ } } - data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) + data, err := metricdispatch.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) if err != nil { cclog.Warn("error while loading node data") return nil, err @@ -825,7 +825,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub } } - data, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx) + data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx) if err != nil { cclog.Warn("error while loading node data (Resolver.NodeMetricsList") return nil, err @@ -880,7 +880,7 @@ func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metr // 'nodes' == nil -> Defaults to all nodes of cluster for existing query workflow scopes := []schema.MetricScope{"node"} - data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx) + data, err := metricdispatch.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx) if err != nil { cclog.Warn("error while loading node data") return nil, err diff --git a/internal/graph/util.go b/internal/graph/util.go index 42a1d2fb..4135ca72 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -13,7 +13,7 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" ) @@ -55,7 +55,7 @@ func (r *queryResolver) rooflineHeatmap( // resolution = max(resolution, mc.Timestep) // } - jobdata, err := metricDataDispatcher.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0) + jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0) if err != nil { cclog.Errorf("Error while loading roofline metrics for job %d", job.ID) return nil, err @@ -128,7 +128,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF continue } - if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil { + if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil { cclog.Error("Error while loading averages for footprint") return nil, err } diff --git a/internal/metricDataDispatcher/dataLoader.go b/internal/metricDataDispatcher/dataLoader.go deleted file mode 100644 index 6d1338fa..00000000 --- a/internal/metricDataDispatcher/dataLoader.go +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package metricDataDispatcher - -import ( - "context" - "fmt" - "math" - "time" - - "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/internal/metricdata" - "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/lrucache" - "github.com/ClusterCockpit/cc-lib/v2/resampler" - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) - -func cacheKey( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - resolution int, -) string { - // Duration and StartTime do not need to be in the cache key as StartTime is less unique than - // job.ID and the TTL of the cache entry makes sure it does not stay there forever. - return fmt.Sprintf("%d(%s):[%v],[%v]-%d", - job.ID, job.State, metrics, scopes, resolution) -} - -// Fetches the metric data for a job. -func LoadData(job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, - resolution int, -) (schema.JobData, error) { - data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) { - var jd schema.JobData - var err error - - if job.State == schema.JobStateRunning || - job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving || - config.Keys.DisableArchive { - - repo, err := metricdata.GetMetricDataRepo(job.Cluster) - if err != nil { - return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster), 0, 0 - } - - if scopes == nil { - scopes = append(scopes, schema.MetricScopeNode) - } - - if metrics == nil { - cluster := archive.GetCluster(job.Cluster) - for _, mc := range cluster.MetricConfig { - metrics = append(metrics, mc.Name) - } - } - - jd, err = repo.LoadData(job, metrics, scopes, ctx, resolution) - if err != nil { - if len(jd) != 0 { - cclog.Warnf("partial error: %s", err.Error()) - // return err, 0, 0 // Reactivating will block archiving on one partial error - } else { - cclog.Error("Error while loading job data from metric repository") - return err, 0, 0 - } - } - size = jd.Size() - } else { - var jd_temp schema.JobData - jd_temp, err = archive.GetHandle().LoadJobData(job) - if err != nil { - cclog.Error("Error while loading job data from archive") - return err, 0, 0 - } - - // Deep copy the cached archive hashmap - jd = metricdata.DeepCopy(jd_temp) - - // Resampling for archived data. - // Pass the resolution from frontend here. - for _, v := range jd { - for _, v_ := range v { - timestep := int64(0) - for i := 0; i < len(v_.Series); i += 1 { - v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution)) - if err != nil { - return err, 0, 0 - } - } - v_.Timestep = int(timestep) - } - } - - // Avoid sending unrequested data to the client: - if metrics != nil || scopes != nil { - if metrics == nil { - metrics = make([]string, 0, len(jd)) - for k := range jd { - metrics = append(metrics, k) - } - } - - res := schema.JobData{} - for _, metric := range metrics { - if perscope, ok := jd[metric]; ok { - if len(perscope) > 1 { - subset := make(map[schema.MetricScope]*schema.JobMetric) - for _, scope := range scopes { - if jm, ok := perscope[scope]; ok { - subset[scope] = jm - } - } - - if len(subset) > 0 { - perscope = subset - } - } - - res[metric] = perscope - } - } - jd = res - } - size = jd.Size() - } - - ttl = 5 * time.Hour - if job.State == schema.JobStateRunning { - ttl = 2 * time.Minute - } - - // FIXME: Review: Is this really necessary or correct. - // Note: Lines 147-170 formerly known as prepareJobData(jobData, scopes) - // For /monitoring/job/ and some other places, flops_any and mem_bw need - // to be available at the scope 'node'. If a job has a lot of nodes, - // statisticsSeries should be available so that a min/median/max Graph can be - // used instead of a lot of single lines. - // NOTE: New StatsSeries will always be calculated as 'min/median/max' - // Existing (archived) StatsSeries can be 'min/mean/max'! - const maxSeriesSize int = 15 - for _, scopes := range jd { - for _, jm := range scopes { - if jm.StatisticsSeries != nil || len(jm.Series) <= maxSeriesSize { - continue - } - - jm.AddStatisticsSeries() - } - } - - nodeScopeRequested := false - for _, scope := range scopes { - if scope == schema.MetricScopeNode { - nodeScopeRequested = true - } - } - - if nodeScopeRequested { - jd.AddNodeScope("flops_any") - jd.AddNodeScope("mem_bw") - } - - // Round Resulting Stat Values - jd.RoundMetricStats() - - return jd, ttl, size - }) - - if err, ok := data.(error); ok { - cclog.Error("Error in returned dataset") - return nil, err - } - - return data.(schema.JobData), nil -} - -// Used for the jobsFootprint GraphQL-Query. TODO: Rename/Generalize. -func LoadAverages( - job *schema.Job, - metrics []string, - data [][]schema.Float, - ctx context.Context, -) error { - if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { - return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here? - } - - repo, err := metricdata.GetMetricDataRepo(job.Cluster) - if err != nil { - return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster) - } - - stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion? - if err != nil { - cclog.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project) - return err - } - - for i, m := range metrics { - nodes, ok := stats[m] - if !ok { - data[i] = append(data[i], schema.NaN) - continue - } - - sum := 0.0 - for _, node := range nodes { - sum += node.Avg - } - data[i] = append(data[i], schema.Float(sum)) - } - - return nil -} - -// Used for statsTable in frontend: Return scoped statistics by metric. -func LoadScopedJobStats( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, -) (schema.ScopedJobStats, error) { - if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { - return archive.LoadScopedStatsFromArchive(job, metrics, scopes) - } - - repo, err := metricdata.GetMetricDataRepo(job.Cluster) - if err != nil { - return nil, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster) - } - - scopedStats, err := repo.LoadScopedStats(job, metrics, scopes, ctx) - if err != nil { - cclog.Errorf("error while loading scoped statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project) - return nil, err - } - - return scopedStats, nil -} - -// Used for polar plots in frontend: Aggregates statistics for all nodes to single values for job per metric. -func LoadJobStats( - job *schema.Job, - metrics []string, - ctx context.Context, -) (map[string]schema.MetricStatistics, error) { - if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { - return archive.LoadStatsFromArchive(job, metrics) - } - - data := make(map[string]schema.MetricStatistics, len(metrics)) - repo, err := metricdata.GetMetricDataRepo(job.Cluster) - if err != nil { - return data, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster) - } - - stats, err := repo.LoadStats(job, metrics, ctx) - if err != nil { - cclog.Errorf("error while loading statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project) - return data, err - } - - for _, m := range metrics { - sum, avg, min, max := 0.0, 0.0, 0.0, 0.0 - nodes, ok := stats[m] - if !ok { - data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max} - continue - } - - for _, node := range nodes { - sum += node.Avg - min = math.Min(min, node.Min) - max = math.Max(max, node.Max) - } - - data[m] = schema.MetricStatistics{ - Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100), - Min: (math.Round(min*100) / 100), - Max: (math.Round(max*100) / 100), - } - } - - return data, nil -} - -// Used for the classic node/system view. Returns a map of nodes to a map of metrics. -func LoadNodeData( - cluster string, - metrics, nodes []string, - scopes []schema.MetricScope, - from, to time.Time, - ctx context.Context, -) (map[string]map[string][]*schema.JobMetric, error) { - repo, err := metricdata.GetMetricDataRepo(cluster) - if err != nil { - return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) - } - - if metrics == nil { - for _, m := range archive.GetCluster(cluster).MetricConfig { - metrics = append(metrics, m.Name) - } - } - - data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) - if err != nil { - if len(data) != 0 { - cclog.Warnf("partial error: %s", err.Error()) - } else { - cclog.Error("Error while loading node data from metric repository") - return nil, err - } - } - - if data == nil { - return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster) - } - - return data, nil -} - -func LoadNodeListData( - cluster, subCluster string, - nodes []string, - metrics []string, - scopes []schema.MetricScope, - resolution int, - from, to time.Time, - ctx context.Context, -) (map[string]schema.JobData, error) { - repo, err := metricdata.GetMetricDataRepo(cluster) - if err != nil { - return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) - } - - if metrics == nil { - for _, m := range archive.GetCluster(cluster).MetricConfig { - metrics = append(metrics, m.Name) - } - } - - data, err := repo.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx) - if err != nil { - if len(data) != 0 { - cclog.Warnf("partial error: %s", err.Error()) - } else { - cclog.Error("Error while loading node data from metric repository") - return nil, err - } - } - - // NOTE: New StatsSeries will always be calculated as 'min/median/max' - const maxSeriesSize int = 8 - for _, jd := range data { - for _, scopes := range jd { - for _, jm := range scopes { - if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize { - continue - } - jm.AddStatisticsSeries() - } - } - } - - if data == nil { - return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster) - } - - return data, nil -} diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go deleted file mode 100644 index 6c146f22..00000000 --- a/internal/metricdata/cc-metric-store.go +++ /dev/null @@ -1,1226 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package metricdata - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - "time" - - "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -type CCMetricStoreConfig struct { - Kind string `json:"kind"` - Url string `json:"url"` - Token string `json:"token"` - - // If metrics are known to this MetricDataRepository under a different - // name than in the `metricConfig` section of the 'cluster.json', - // provide this optional mapping of local to remote name for this metric. - Renamings map[string]string `json:"metricRenamings"` -} - -type CCMetricStore struct { - here2there map[string]string - there2here map[string]string - client http.Client - jwt string - url string - queryEndpoint string -} - -type ApiQueryRequest struct { - Cluster string `json:"cluster"` - Queries []ApiQuery `json:"queries"` - ForAllNodes []string `json:"for-all-nodes"` - From int64 `json:"from"` - To int64 `json:"to"` - WithStats bool `json:"with-stats"` - WithData bool `json:"with-data"` -} - -type ApiQuery struct { - Type *string `json:"type,omitempty"` - SubType *string `json:"subtype,omitempty"` - Metric string `json:"metric"` - Hostname string `json:"host"` - Resolution int `json:"resolution"` - TypeIds []string `json:"type-ids,omitempty"` - SubTypeIds []string `json:"subtype-ids,omitempty"` - Aggregate bool `json:"aggreg"` -} - -type ApiQueryResponse struct { - Queries []ApiQuery `json:"queries,omitempty"` - Results [][]ApiMetricData `json:"results"` -} - -type ApiMetricData struct { - Error *string `json:"error"` - Data []schema.Float `json:"data"` - From int64 `json:"from"` - To int64 `json:"to"` - Resolution int `json:"resolution"` - Avg schema.Float `json:"avg"` - Min schema.Float `json:"min"` - Max schema.Float `json:"max"` -} - -func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error { - var config CCMetricStoreConfig - if err := json.Unmarshal(rawConfig, &config); err != nil { - cclog.Warn("Error while unmarshaling raw json config") - return err - } - - ccms.url = config.Url - ccms.queryEndpoint = fmt.Sprintf("%s/api/query", config.Url) - ccms.jwt = config.Token - ccms.client = http.Client{ - Timeout: 10 * time.Second, - } - - if config.Renamings != nil { - ccms.here2there = config.Renamings - ccms.there2here = make(map[string]string, len(config.Renamings)) - for k, v := range ccms.here2there { - ccms.there2here[v] = k - } - } else { - ccms.here2there = make(map[string]string) - ccms.there2here = make(map[string]string) - } - - return nil -} - -func (ccms *CCMetricStore) toRemoteName(metric string) string { - if renamed, ok := ccms.here2there[metric]; ok { - return renamed - } - - return metric -} - -func (ccms *CCMetricStore) toLocalName(metric string) string { - if renamed, ok := ccms.there2here[metric]; ok { - return renamed - } - - return metric -} - -func (ccms *CCMetricStore) doRequest( - ctx context.Context, - body *ApiQueryRequest, -) (*ApiQueryResponse, error) { - buf := &bytes.Buffer{} - if err := json.NewEncoder(buf).Encode(body); err != nil { - cclog.Errorf("Error while encoding request body: %s", err.Error()) - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf) - if err != nil { - cclog.Errorf("Error while building request body: %s", err.Error()) - return nil, err - } - if ccms.jwt != "" { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt)) - } - - // versioning the cc-metric-store query API. - // v2 = data with resampling - // v1 = data without resampling - q := req.URL.Query() - q.Add("version", "v2") - req.URL.RawQuery = q.Encode() - - res, err := ccms.client.Do(req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status) - } - - var resBody ApiQueryResponse - if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil { - cclog.Errorf("Error while decoding result body: %s", err.Error()) - return nil, err - } - - return &resBody, nil -} - -func (ccms *CCMetricStore) LoadData( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, - resolution int, -) (schema.JobData, error) { - queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution) - if err != nil { - cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) - return nil, err - } - - req := ApiQueryRequest{ - Cluster: job.Cluster, - From: job.StartTime, - To: job.StartTime + int64(job.Duration), - Queries: queries, - WithStats: true, - WithData: true, - } - - resBody, err := ccms.doRequest(ctx, &req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - var errors []string - jobData := make(schema.JobData) - for i, row := range resBody.Results { - query := req.Queries[i] - metric := ccms.toLocalName(query.Metric) - scope := assignedScope[i] - mc := archive.GetMetricConfig(job.Cluster, metric) - if _, ok := jobData[metric]; !ok { - jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric) - } - - res := mc.Timestep - if len(row) > 0 { - res = row[0].Resolution - } - - jobMetric, ok := jobData[metric][scope] - if !ok { - jobMetric = &schema.JobMetric{ - Unit: mc.Unit, - Timestep: res, - Series: make([]schema.Series, 0), - } - jobData[metric][scope] = jobMetric - } - - for ndx, res := range row { - if res.Error != nil { - /* Build list for "partial errors", if any */ - errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) - continue - } - - id := (*string)(nil) - if query.Type != nil { - id = new(string) - *id = query.TypeIds[ndx] - } - - if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() { - // "schema.Float()" because regular float64 can not be JSONed when NaN. - res.Avg = schema.Float(0) - res.Min = schema.Float(0) - res.Max = schema.Float(0) - } - - jobMetric.Series = append(jobMetric.Series, schema.Series{ - Hostname: query.Hostname, - Id: id, - Statistics: schema.MetricStatistics{ - Avg: float64(res.Avg), - Min: float64(res.Min), - Max: float64(res.Max), - }, - Data: res.Data, - }) - } - - // So that one can later check len(jobData): - if len(jobMetric.Series) == 0 { - delete(jobData[metric], scope) - if len(jobData[metric]) == 0 { - delete(jobData, metric) - } - } - } - - if len(errors) != 0 { - /* Returns list for "partial errors" */ - return jobData, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) - } - return jobData, nil -} - -func (ccms *CCMetricStore) buildQueries( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - resolution int, -) ([]ApiQuery, []schema.MetricScope, error) { - queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) - assignedScope := []schema.MetricScope{} - - subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) - if scerr != nil { - return nil, nil, scerr - } - topology := subcluster.Topology - - for _, metric := range metrics { - remoteName := ccms.toRemoteName(metric) - mc := archive.GetMetricConfig(job.Cluster, metric) - if mc == nil { - // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster) - cclog.Infof("metric '%s' is not specified for cluster '%s'", metric, job.Cluster) - continue - } - - // Skip if metric is removed for subcluster - if len(mc.SubClusters) != 0 { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == job.SubCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } - } - - // Avoid duplicates... - handledScopes := make([]schema.MetricScope, 0, 3) - - scopesLoop: - for _, requestedScope := range scopes { - nativeScope := mc.Scope - if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 { - continue - } - - scope := nativeScope.Max(requestedScope) - for _, s := range handledScopes { - if scope == s { - continue scopesLoop - } - } - handledScopes = append(handledScopes, scope) - - for _, host := range job.Resources { - hwthreads := host.HWThreads - if hwthreads == nil { - hwthreads = topology.Node - } - - // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) - if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { - if scope != schema.MetricScopeAccelerator { - // Skip all other catched cases - continue - } - - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: false, - Type: &acceleratorString, - TypeIds: host.Accelerators, - Resolution: resolution, - }) - assignedScope = append(assignedScope, schema.MetricScopeAccelerator) - continue - } - - // Accelerator -> Node - if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { - if len(host.Accelerators) == 0 { - continue - } - - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &acceleratorString, - TypeIds: host.Accelerators, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> HWThead - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: false, - Type: &hwthreadString, - TypeIds: intToStringSlice(hwthreads), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> Core - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - for _, core := range cores { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Core[core]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Socket - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - for _, socket := range sockets { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Node - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(hwthreads), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Core - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: false, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Socket - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromCores(hwthreads) - for _, socket := range sockets { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // Core -> Node - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> MemoryDomain - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: false, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDoman -> Node - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Socket - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: false, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Node - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Aggregate: true, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Node -> Node - if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: host.Hostname, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) - } - } - } - - return queries, assignedScope, nil -} - -func (ccms *CCMetricStore) LoadStats( - job *schema.Job, - metrics []string, - ctx context.Context, -) (map[string]map[string]schema.MetricStatistics, error) { - - queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization? - if err != nil { - cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error()) - return nil, err - } - - req := ApiQueryRequest{ - Cluster: job.Cluster, - From: job.StartTime, - To: job.StartTime + int64(job.Duration), - Queries: queries, - WithStats: true, - WithData: false, - } - - resBody, err := ccms.doRequest(ctx, &req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - stats := make(map[string]map[string]schema.MetricStatistics, len(metrics)) - for i, res := range resBody.Results { - query := req.Queries[i] - metric := ccms.toLocalName(query.Metric) - data := res[0] - if data.Error != nil { - cclog.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) - continue - } - - metricdata, ok := stats[metric] - if !ok { - metricdata = make(map[string]schema.MetricStatistics, job.NumNodes) - stats[metric] = metricdata - } - - if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() { - cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname) - continue - } - - metricdata[query.Hostname] = schema.MetricStatistics{ - Avg: float64(data.Avg), - Min: float64(data.Min), - Max: float64(data.Max), - } - } - - return stats, nil -} - -// Used for Job-View Statistics Table -func (ccms *CCMetricStore) LoadScopedStats( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, -) (schema.ScopedJobStats, error) { - queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0) - if err != nil { - cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) - return nil, err - } - - req := ApiQueryRequest{ - Cluster: job.Cluster, - From: job.StartTime, - To: job.StartTime + int64(job.Duration), - Queries: queries, - WithStats: true, - WithData: false, - } - - resBody, err := ccms.doRequest(ctx, &req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - var errors []string - scopedJobStats := make(schema.ScopedJobStats) - - for i, row := range resBody.Results { - query := req.Queries[i] - metric := ccms.toLocalName(query.Metric) - scope := assignedScope[i] - - if _, ok := scopedJobStats[metric]; !ok { - scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) - } - - if _, ok := scopedJobStats[metric][scope]; !ok { - scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) - } - - for ndx, res := range row { - if res.Error != nil { - /* Build list for "partial errors", if any */ - errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) - continue - } - - id := (*string)(nil) - if query.Type != nil { - id = new(string) - *id = query.TypeIds[ndx] - } - - if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() { - // "schema.Float()" because regular float64 can not be JSONed when NaN. - res.Avg = schema.Float(0) - res.Min = schema.Float(0) - res.Max = schema.Float(0) - } - - scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ - Hostname: query.Hostname, - Id: id, - Data: &schema.MetricStatistics{ - Avg: float64(res.Avg), - Min: float64(res.Min), - Max: float64(res.Max), - }, - }) - } - - // So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty - if len(scopedJobStats[metric][scope]) == 0 { - delete(scopedJobStats[metric], scope) - if len(scopedJobStats[metric]) == 0 { - delete(scopedJobStats, metric) - } - } - } - - if len(errors) != 0 { - /* Returns list for "partial errors" */ - return scopedJobStats, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) - } - return scopedJobStats, nil -} - -// Used for Systems-View Node-Overview -func (ccms *CCMetricStore) LoadNodeData( - cluster string, - metrics, nodes []string, - scopes []schema.MetricScope, - from, to time.Time, - ctx context.Context, -) (map[string]map[string][]*schema.JobMetric, error) { - req := ApiQueryRequest{ - Cluster: cluster, - From: from.Unix(), - To: to.Unix(), - WithStats: true, - WithData: true, - } - - if nodes == nil { - for _, metric := range metrics { - req.ForAllNodes = append(req.ForAllNodes, ccms.toRemoteName(metric)) - } - } else { - for _, node := range nodes { - for _, metric := range metrics { - req.Queries = append(req.Queries, ApiQuery{ - Hostname: node, - Metric: ccms.toRemoteName(metric), - Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution - }) - } - } - } - - resBody, err := ccms.doRequest(ctx, &req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - var errors []string - data := make(map[string]map[string][]*schema.JobMetric) - for i, res := range resBody.Results { - var query ApiQuery - if resBody.Queries != nil { - query = resBody.Queries[i] - } else { - query = req.Queries[i] - } - - metric := ccms.toLocalName(query.Metric) - qdata := res[0] - if qdata.Error != nil { - /* Build list for "partial errors", if any */ - errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) - } - - if qdata.Avg.IsNaN() || qdata.Min.IsNaN() || qdata.Max.IsNaN() { - // return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") - qdata.Avg, qdata.Min, qdata.Max = 0., 0., 0. - } - - hostdata, ok := data[query.Hostname] - if !ok { - hostdata = make(map[string][]*schema.JobMetric) - data[query.Hostname] = hostdata - } - - mc := archive.GetMetricConfig(cluster, metric) - if mc != nil { - hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ - Unit: mc.Unit, - Timestep: mc.Timestep, - Series: []schema.Series{ - { - Hostname: query.Hostname, - Data: qdata.Data, - Statistics: schema.MetricStatistics{ - Avg: float64(qdata.Avg), - Min: float64(qdata.Min), - Max: float64(qdata.Max), - }, - }, - }, - }) - } else { - cclog.Warnf("Metric '%s' not configured for cluster '%s': Skipped in LoadNodeData() Return!", metric, cluster) - } - } - - if len(errors) != 0 { - /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) - } - - return data, nil -} - -// Used for Systems-View Node-List -func (ccms *CCMetricStore) LoadNodeListData( - cluster, subCluster string, - nodes []string, - metrics []string, - scopes []schema.MetricScope, - resolution int, - from, to time.Time, - ctx context.Context, -) (map[string]schema.JobData, error) { - - // Note: Order of node data is not guaranteed after this point - queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution) - if err != nil { - cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error()) - return nil, err - } - - req := ApiQueryRequest{ - Cluster: cluster, - Queries: queries, - From: from.Unix(), - To: to.Unix(), - WithStats: true, - WithData: true, - } - - resBody, err := ccms.doRequest(ctx, &req) - if err != nil { - cclog.Errorf("Error while performing request: %s", err.Error()) - return nil, err - } - - var errors []string - data := make(map[string]schema.JobData) - for i, row := range resBody.Results { - var query ApiQuery - if resBody.Queries != nil { - query = resBody.Queries[i] - } else { - query = req.Queries[i] - } - // qdata := res[0] - metric := ccms.toLocalName(query.Metric) - scope := assignedScope[i] - mc := archive.GetMetricConfig(cluster, metric) - - res := mc.Timestep - if len(row) > 0 { - res = row[0].Resolution - } - - // Init Nested Map Data Structures If Not Found - hostData, ok := data[query.Hostname] - if !ok { - hostData = make(schema.JobData) - data[query.Hostname] = hostData - } - - metricData, ok := hostData[metric] - if !ok { - metricData = make(map[schema.MetricScope]*schema.JobMetric) - data[query.Hostname][metric] = metricData - } - - scopeData, ok := metricData[scope] - if !ok { - scopeData = &schema.JobMetric{ - Unit: mc.Unit, - Timestep: res, - Series: make([]schema.Series, 0), - } - data[query.Hostname][metric][scope] = scopeData - } - - for ndx, res := range row { - if res.Error != nil { - /* Build list for "partial errors", if any */ - errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) - continue - } - - id := (*string)(nil) - if query.Type != nil { - id = new(string) - *id = query.TypeIds[ndx] - } - - if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() { - // "schema.Float()" because regular float64 can not be JSONed when NaN. - res.Avg = schema.Float(0) - res.Min = schema.Float(0) - res.Max = schema.Float(0) - } - - scopeData.Series = append(scopeData.Series, schema.Series{ - Hostname: query.Hostname, - Id: id, - Statistics: schema.MetricStatistics{ - Avg: float64(res.Avg), - Min: float64(res.Min), - Max: float64(res.Max), - }, - Data: res.Data, - }) - } - } - - if len(errors) != 0 { - /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) - } - - return data, nil -} - -func (ccms *CCMetricStore) buildNodeQueries( - cluster string, - subCluster string, - nodes []string, - metrics []string, - scopes []schema.MetricScope, - resolution int, -) ([]ApiQuery, []schema.MetricScope, error) { - - queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(nodes)) - assignedScope := []schema.MetricScope{} - - // Get Topol before loop if subCluster given - var subClusterTopol *schema.SubCluster - var scterr error - if subCluster != "" { - subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster) - if scterr != nil { - cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error()) - return nil, nil, scterr - } - } - - for _, metric := range metrics { - remoteName := ccms.toRemoteName(metric) - mc := archive.GetMetricConfig(cluster, metric) - if mc == nil { - // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, cluster) - cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster) - continue - } - - // Skip if metric is removed for subcluster - if mc.SubClusters != nil { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == subCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } - } - - // Avoid duplicates... - handledScopes := make([]schema.MetricScope, 0, 3) - - scopesLoop: - for _, requestedScope := range scopes { - nativeScope := mc.Scope - - scope := nativeScope.Max(requestedScope) - for _, s := range handledScopes { - if scope == s { - continue scopesLoop - } - } - handledScopes = append(handledScopes, scope) - - for _, hostname := range nodes { - - // If no subCluster given, get it by node - if subCluster == "" { - subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname) - if scnerr != nil { - return nil, nil, scnerr - } - subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName) - if scterr != nil { - return nil, nil, scterr - } - } - - // Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable - // Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable - topology := subClusterTopol.Topology - acceleratorIds := topology.GetAcceleratorIDs() - - // Moved check here if metric matches hardware specs - if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 { - continue scopesLoop - } - - // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) - if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { - if scope != schema.MetricScopeAccelerator { - // Skip all other catched cases - continue - } - - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: false, - Type: &acceleratorString, - TypeIds: acceleratorIds, - Resolution: resolution, - }) - assignedScope = append(assignedScope, schema.MetricScopeAccelerator) - continue - } - - // Accelerator -> Node - if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { - if len(acceleratorIds) == 0 { - continue - } - - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &acceleratorString, - TypeIds: acceleratorIds, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> HWThead - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: false, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Node), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> Core - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - for _, core := range cores { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Core[core]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Socket - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - for _, socket := range sockets { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Node - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Node), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Core - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: false, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Socket - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromCores(topology.Node) - for _, socket := range sockets { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // Core -> Node - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> MemoryDomain - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: false, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDoman -> Node - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Socket - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: false, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Node - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Aggregate: true, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Node -> Node - if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, ApiQuery{ - Metric: remoteName, - Hostname: hostname, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) - } - } - } - - return queries, assignedScope, nil -} diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go deleted file mode 100644 index ab0e19fb..00000000 --- a/internal/metricdata/metricdata.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricdata - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/internal/memorystore" - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -type MetricDataRepository interface { - // Initialize this MetricDataRepository. One instance of - // this interface will only ever be responsible for one cluster. - Init(rawConfig json.RawMessage) error - - // Return the JobData for the given job, only with the requested metrics. - LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) - - // Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only. - LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) - - // Return a map of metrics to a map of scopes to the scoped metric statistics of the job. - LoadScopedStats(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error) - - // Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node. - LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) - - // Return a map of hosts to a map of metrics to a map of scopes for multiple nodes. - LoadNodeListData(cluster, subCluster string, nodes, metrics []string, scopes []schema.MetricScope, resolution int, from, to time.Time, ctx context.Context) (map[string]schema.JobData, error) -} - -var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{} - -func Init() error { - for _, cluster := range config.Clusters { - if cluster.MetricDataRepository != nil { - var kind struct { - Kind string `json:"kind"` - } - if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { - cclog.Warn("Error while unmarshaling raw json MetricDataRepository") - return err - } - - var mdr MetricDataRepository - switch kind.Kind { - case "cc-metric-store": - mdr = &CCMetricStore{} - case "cc-metric-store-internal": - mdr = &CCMetricStoreInternal{} - memorystore.InternalCCMSFlag = true - case "prometheus": - mdr = &PrometheusDataRepository{} - case "test": - mdr = &TestMetricDataRepository{} - default: - return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - } - - if err := mdr.Init(cluster.MetricDataRepository); err != nil { - cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - return err - } - metricDataRepos[cluster.Name] = mdr - } - } - return nil -} - -func GetMetricDataRepo(cluster string) (MetricDataRepository, error) { - var err error - repo, ok := metricDataRepos[cluster] - - if !ok { - err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) - } - - return repo, err -} diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go deleted file mode 100644 index 3fb94d51..00000000 --- a/internal/metricdata/prometheus.go +++ /dev/null @@ -1,587 +0,0 @@ -// Copyright (C) 2022 DKRZ -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package metricdata - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "math" - "net/http" - "os" - "regexp" - "sort" - "strings" - "sync" - "text/template" - "time" - - "github.com/ClusterCockpit/cc-backend/pkg/archive" - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/schema" - promapi "github.com/prometheus/client_golang/api" - promv1 "github.com/prometheus/client_golang/api/prometheus/v1" - promcfg "github.com/prometheus/common/config" - promm "github.com/prometheus/common/model" -) - -type PrometheusDataRepositoryConfig struct { - Url string `json:"url"` - Username string `json:"username,omitempty"` - Suffix string `json:"suffix,omitempty"` - Templates map[string]string `json:"query-templates"` -} - -type PrometheusDataRepository struct { - client promapi.Client - queryClient promv1.API - suffix string - templates map[string]*template.Template -} - -type PromQLArgs struct { - Nodes string -} - -type Trie map[rune]Trie - -var logOnce sync.Once - -func contains(s []schema.MetricScope, str schema.MetricScope) bool { - for _, v := range s { - if v == str { - return true - } - } - return false -} - -func MinMaxMean(data []schema.Float) (float64, float64, float64) { - if len(data) == 0 { - return 0.0, 0.0, 0.0 - } - min := math.MaxFloat64 - max := -math.MaxFloat64 - var sum float64 - var n float64 - for _, val := range data { - if val.IsNaN() { - continue - } - sum += float64(val) - n += 1 - if float64(val) > max { - max = float64(val) - } - if float64(val) < min { - min = float64(val) - } - } - return min, max, sum / n -} - -// Rewritten from -// https://github.com/ermanh/trieregex/blob/master/trieregex/trieregex.py -func nodeRegex(nodes []string) string { - root := Trie{} - // add runes of each compute node to trie - for _, node := range nodes { - _trie := root - for _, c := range node { - if _, ok := _trie[c]; !ok { - _trie[c] = Trie{} - } - _trie = _trie[c] - } - _trie['*'] = Trie{} - } - // recursively build regex from rune trie - var trieRegex func(trie Trie, reset bool) string - trieRegex = func(trie Trie, reset bool) string { - if reset == true { - trie = root - } - if len(trie) == 0 { - return "" - } - if len(trie) == 1 { - for key, _trie := range trie { - if key == '*' { - return "" - } - return regexp.QuoteMeta(string(key)) + trieRegex(_trie, false) - } - } else { - sequences := []string{} - for key, _trie := range trie { - if key != '*' { - sequences = append(sequences, regexp.QuoteMeta(string(key))+trieRegex(_trie, false)) - } - } - sort.Slice(sequences, func(i, j int) bool { - return (-len(sequences[i]) < -len(sequences[j])) || (sequences[i] < sequences[j]) - }) - var result string - // single edge from this tree node - if len(sequences) == 1 { - result = sequences[0] - if len(result) > 1 { - result = "(?:" + result + ")" - } - // multiple edges, each length 1 - } else if s := strings.Join(sequences, ""); len(s) == len(sequences) { - // char or numeric range - if len(s)-1 == int(s[len(s)-1])-int(s[0]) { - result = fmt.Sprintf("[%c-%c]", s[0], s[len(s)-1]) - // char or numeric set - } else { - result = "[" + s + "]" - } - // multiple edges of different lengths - } else { - result = "(?:" + strings.Join(sequences, "|") + ")" - } - if _, ok := trie['*']; ok { - result += "?" - } - return result - } - return "" - } - return trieRegex(root, true) -} - -func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { - var config PrometheusDataRepositoryConfig - // parse config - if err := json.Unmarshal(rawConfig, &config); err != nil { - cclog.Warn("Error while unmarshaling raw json config") - return err - } - // support basic authentication - var rt http.RoundTripper = nil - if prom_pw := os.Getenv("PROMETHEUS_PASSWORD"); prom_pw != "" && config.Username != "" { - prom_pw := promcfg.Secret(prom_pw) - rt = promcfg.NewBasicAuthRoundTripper(promcfg.NewInlineSecret(config.Username), promcfg.NewInlineSecret(string(prom_pw)), promapi.DefaultRoundTripper) - } else { - if config.Username != "" { - return errors.New("METRICDATA/PROMETHEUS > Prometheus username provided, but PROMETHEUS_PASSWORD not set") - } - } - // init client - client, err := promapi.NewClient(promapi.Config{ - Address: config.Url, - RoundTripper: rt, - }) - if err != nil { - cclog.Error("Error while initializing new prometheus client") - return err - } - // init query client - pdb.client = client - pdb.queryClient = promv1.NewAPI(pdb.client) - // site config - pdb.suffix = config.Suffix - // init query templates - pdb.templates = make(map[string]*template.Template) - for metric, templ := range config.Templates { - pdb.templates[metric], err = template.New(metric).Parse(templ) - if err == nil { - cclog.Debugf("Added PromQL template for %s: %s", metric, templ) - } else { - cclog.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric) - } - } - return nil -} - -// TODO: respect scope argument -func (pdb *PrometheusDataRepository) FormatQuery( - metric string, - scope schema.MetricScope, - nodes []string, - cluster string, -) (string, error) { - args := PromQLArgs{} - if len(nodes) > 0 { - args.Nodes = fmt.Sprintf("(%s)%s", nodeRegex(nodes), pdb.suffix) - } else { - args.Nodes = fmt.Sprintf(".*%s", pdb.suffix) - } - - buf := &bytes.Buffer{} - if templ, ok := pdb.templates[metric]; ok { - err := templ.Execute(buf, args) - if err != nil { - return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > Error compiling template %v", templ)) - } else { - query := buf.String() - cclog.Debugf("PromQL: %s", query) - return query, nil - } - } else { - return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > No PromQL for metric %s configured.", metric)) - } -} - -// Convert PromAPI row to CC schema.Series -func (pdb *PrometheusDataRepository) RowToSeries( - from time.Time, - step int64, - steps int64, - row *promm.SampleStream, -) schema.Series { - ts := from.Unix() - hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix) - // init array of expected length with NaN - values := make([]schema.Float, steps+1) - for i := range values { - values[i] = schema.NaN - } - // copy recorded values from prom sample pair - for _, v := range row.Values { - idx := (v.Timestamp.Unix() - ts) / step - values[idx] = schema.Float(v.Value) - } - min, max, mean := MinMaxMean(values) - // output struct - return schema.Series{ - Hostname: hostname, - Data: values, - Statistics: schema.MetricStatistics{ - Avg: mean, - Min: min, - Max: max, - }, - } -} - -func (pdb *PrometheusDataRepository) LoadData( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, - resolution int, -) (schema.JobData, error) { - // TODO respect requested scope - if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) { - scopes = append(scopes, schema.MetricScopeNode) - } - - jobData := make(schema.JobData) - // parse job specs - nodes := make([]string, len(job.Resources)) - for i, resource := range job.Resources { - nodes[i] = resource.Hostname - } - from := time.Unix(job.StartTime, 0) - to := time.Unix(job.StartTime+int64(job.Duration), 0) - - for _, scope := range scopes { - if scope != schema.MetricScopeNode { - logOnce.Do(func() { - cclog.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) - }) - continue - } - - for _, metric := range metrics { - metricConfig := archive.GetMetricConfig(job.Cluster, metric) - if metricConfig == nil { - cclog.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) - return nil, errors.New("Prometheus config error") - } - query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster) - if err != nil { - cclog.Warn("Error while formatting prometheus query") - return nil, err - } - - // ranged query over all job nodes - r := promv1.Range{ - Start: from, - End: to, - Step: time.Duration(metricConfig.Timestep * 1e9), - } - result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) - if err != nil { - cclog.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query) - return nil, errors.New("Prometheus query error") - } - if len(warnings) > 0 { - cclog.Warnf("Warnings: %v\n", warnings) - } - - // init data structures - if _, ok := jobData[metric]; !ok { - jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric) - } - jobMetric, ok := jobData[metric][scope] - if !ok { - jobMetric = &schema.JobMetric{ - Unit: metricConfig.Unit, - Timestep: metricConfig.Timestep, - Series: make([]schema.Series, 0), - } - } - step := int64(metricConfig.Timestep) - steps := int64(to.Sub(from).Seconds()) / step - // iter rows of host, metric, values - for _, row := range result.(promm.Matrix) { - jobMetric.Series = append(jobMetric.Series, - pdb.RowToSeries(from, step, steps, row)) - } - // only add metric if at least one host returned data - if !ok && len(jobMetric.Series) > 0 { - jobData[metric][scope] = jobMetric - } - // sort by hostname to get uniform coloring - sort.Slice(jobMetric.Series, func(i, j int) bool { - return (jobMetric.Series[i].Hostname < jobMetric.Series[j].Hostname) - }) - } - } - return jobData, nil -} - -// TODO change implementation to precomputed/cached stats -func (pdb *PrometheusDataRepository) LoadStats( - job *schema.Job, - metrics []string, - ctx context.Context, -) (map[string]map[string]schema.MetricStatistics, error) { - // map of metrics of nodes of stats - stats := map[string]map[string]schema.MetricStatistics{} - - data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/) - if err != nil { - cclog.Warn("Error while loading job for stats") - return nil, err - } - for metric, metricData := range data { - stats[metric] = make(map[string]schema.MetricStatistics) - for _, series := range metricData[schema.MetricScopeNode].Series { - stats[metric][series.Hostname] = series.Statistics - } - } - - return stats, nil -} - -func (pdb *PrometheusDataRepository) LoadNodeData( - cluster string, - metrics, nodes []string, - scopes []schema.MetricScope, - from, to time.Time, - ctx context.Context, -) (map[string]map[string][]*schema.JobMetric, error) { - t0 := time.Now() - // Map of hosts of metrics of value slices - data := make(map[string]map[string][]*schema.JobMetric) - // query db for each metric - // TODO: scopes seems to be always empty - if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) { - scopes = append(scopes, schema.MetricScopeNode) - } - for _, scope := range scopes { - if scope != schema.MetricScopeNode { - logOnce.Do(func() { - cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) - }) - continue - } - for _, metric := range metrics { - metricConfig := archive.GetMetricConfig(cluster, metric) - if metricConfig == nil { - cclog.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) - return nil, errors.New("Prometheus config error") - } - query, err := pdb.FormatQuery(metric, scope, nodes, cluster) - if err != nil { - cclog.Warn("Error while formatting prometheus query") - return nil, err - } - - // ranged query over all nodes - r := promv1.Range{ - Start: from, - End: to, - Step: time.Duration(metricConfig.Timestep * 1e9), - } - result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) - if err != nil { - cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err) - return nil, errors.New("Prometheus query error") - } - if len(warnings) > 0 { - cclog.Warnf("Warnings: %v\n", warnings) - } - - step := int64(metricConfig.Timestep) - steps := int64(to.Sub(from).Seconds()) / step - - // iter rows of host, metric, values - for _, row := range result.(promm.Matrix) { - hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix) - hostdata, ok := data[hostname] - if !ok { - hostdata = make(map[string][]*schema.JobMetric) - data[hostname] = hostdata - } - // output per host and metric - hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ - Unit: metricConfig.Unit, - Timestep: metricConfig.Timestep, - Series: []schema.Series{pdb.RowToSeries(from, step, steps, row)}, - }, - ) - } - } - } - t1 := time.Since(t0) - cclog.Debugf("LoadNodeData of %v nodes took %s", len(data), t1) - return data, nil -} - -// Implemented by NHR@FAU; Used in Job-View StatsTable -func (pdb *PrometheusDataRepository) LoadScopedStats( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, -) (schema.ScopedJobStats, error) { - // Assumption: pdb.loadData() only returns series node-scope - use node scope for statsTable - scopedJobStats := make(schema.ScopedJobStats) - data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/) - if err != nil { - cclog.Warn("Error while loading job for scopedJobStats") - return nil, err - } - - for metric, metricData := range data { - for _, scope := range scopes { - if scope != schema.MetricScopeNode { - logOnce.Do(func() { - cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) - }) - continue - } - - if _, ok := scopedJobStats[metric]; !ok { - scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats) - } - - if _, ok := scopedJobStats[metric][scope]; !ok { - scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0) - } - - for _, series := range metricData[scope].Series { - scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ - Hostname: series.Hostname, - Data: &series.Statistics, - }) - } - } - } - - return scopedJobStats, nil -} - -// Implemented by NHR@FAU; Used in NodeList-View -func (pdb *PrometheusDataRepository) LoadNodeListData( - cluster, subCluster string, - nodes []string, - metrics []string, - scopes []schema.MetricScope, - resolution int, - from, to time.Time, - ctx context.Context, -) (map[string]schema.JobData, error) { - // Assumption: pdb.loadData() only returns series node-scope - use node scope for NodeList - - // Fetch Data, based on pdb.LoadNodeData() - t0 := time.Now() - // Map of hosts of jobData - data := make(map[string]schema.JobData) - - // query db for each metric - // TODO: scopes seems to be always empty - if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) { - scopes = append(scopes, schema.MetricScopeNode) - } - - for _, scope := range scopes { - if scope != schema.MetricScopeNode { - logOnce.Do(func() { - cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope) - }) - continue - } - - for _, metric := range metrics { - metricConfig := archive.GetMetricConfig(cluster, metric) - if metricConfig == nil { - cclog.Warnf("Error in LoadNodeListData: Metric %s for cluster %s not configured", metric, cluster) - return nil, errors.New("Prometheus config error") - } - query, err := pdb.FormatQuery(metric, scope, nodes, cluster) - if err != nil { - cclog.Warn("Error while formatting prometheus query") - return nil, err - } - - // ranged query over all nodes - r := promv1.Range{ - Start: from, - End: to, - Step: time.Duration(metricConfig.Timestep * 1e9), - } - result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) - if err != nil { - cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err) - return nil, errors.New("Prometheus query error") - } - if len(warnings) > 0 { - cclog.Warnf("Warnings: %v\n", warnings) - } - - step := int64(metricConfig.Timestep) - steps := int64(to.Sub(from).Seconds()) / step - - // iter rows of host, metric, values - for _, row := range result.(promm.Matrix) { - hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix) - - hostdata, ok := data[hostname] - if !ok { - hostdata = make(schema.JobData) - data[hostname] = hostdata - } - - metricdata, ok := hostdata[metric] - if !ok { - metricdata = make(map[schema.MetricScope]*schema.JobMetric) - data[hostname][metric] = metricdata - } - - // output per host, metric and scope - scopeData, ok := metricdata[scope] - if !ok { - scopeData = &schema.JobMetric{ - Unit: metricConfig.Unit, - Timestep: metricConfig.Timestep, - Series: []schema.Series{pdb.RowToSeries(from, step, steps, row)}, - } - data[hostname][metric][scope] = scopeData - } - } - } - } - t1 := time.Since(t0) - cclog.Debugf("LoadNodeListData of %v nodes took %s", len(data), t1) - return data, nil -} diff --git a/internal/metricdata/utils.go b/internal/metricdata/utils.go deleted file mode 100644 index 21dfbcac..00000000 --- a/internal/metricdata/utils.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricdata - -import ( - "context" - "encoding/json" - "time" - - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) { - panic("TODO") -} - -// TestMetricDataRepository is only a mock for unit-testing. -type TestMetricDataRepository struct{} - -func (tmdr *TestMetricDataRepository) Init(_ json.RawMessage) error { - return nil -} - -func (tmdr *TestMetricDataRepository) LoadData( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, - resolution int, -) (schema.JobData, error) { - return TestLoadDataCallback(job, metrics, scopes, ctx, resolution) -} - -func (tmdr *TestMetricDataRepository) LoadStats( - job *schema.Job, - metrics []string, - ctx context.Context, -) (map[string]map[string]schema.MetricStatistics, error) { - panic("TODO") -} - -func (tmdr *TestMetricDataRepository) LoadScopedStats( - job *schema.Job, - metrics []string, - scopes []schema.MetricScope, - ctx context.Context, -) (schema.ScopedJobStats, error) { - panic("TODO") -} - -func (tmdr *TestMetricDataRepository) LoadNodeData( - cluster string, - metrics, nodes []string, - scopes []schema.MetricScope, - from, to time.Time, - ctx context.Context, -) (map[string]map[string][]*schema.JobMetric, error) { - panic("TODO") -} - -func (tmdr *TestMetricDataRepository) LoadNodeListData( - cluster, subCluster string, - nodes []string, - metrics []string, - scopes []schema.MetricScope, - resolution int, - from, to time.Time, - ctx context.Context, -) (map[string]schema.JobData, error) { - panic("TODO") -} - -func DeepCopy(jdTemp schema.JobData) schema.JobData { - jd := make(schema.JobData, len(jdTemp)) - for k, v := range jdTemp { - jd[k] = make(map[schema.MetricScope]*schema.JobMetric, len(jdTemp[k])) - for k_, v_ := range v { - jd[k][k_] = new(schema.JobMetric) - jd[k][k_].Series = make([]schema.Series, len(v_.Series)) - for i := 0; i < len(v_.Series); i += 1 { - jd[k][k_].Series[i].Data = make([]schema.Float, len(v_.Series[i].Data)) - copy(jd[k][k_].Series[i].Data, v_.Series[i].Data) - jd[k][k_].Series[i].Hostname = v_.Series[i].Hostname - jd[k][k_].Series[i].Id = v_.Series[i].Id - jd[k][k_].Series[i].Statistics.Avg = v_.Series[i].Statistics.Avg - jd[k][k_].Series[i].Statistics.Min = v_.Series[i].Statistics.Min - jd[k][k_].Series[i].Statistics.Max = v_.Series[i].Statistics.Max - } - jd[k][k_].Timestep = v_.Timestep - jd[k][k_].Unit.Base = v_.Unit.Base - jd[k][k_].Unit.Prefix = v_.Unit.Prefix - if v_.StatisticsSeries != nil { - // Init Slices - jd[k][k_].StatisticsSeries = new(schema.StatsSeries) - jd[k][k_].StatisticsSeries.Max = make([]schema.Float, len(v_.StatisticsSeries.Max)) - jd[k][k_].StatisticsSeries.Min = make([]schema.Float, len(v_.StatisticsSeries.Min)) - jd[k][k_].StatisticsSeries.Median = make([]schema.Float, len(v_.StatisticsSeries.Median)) - jd[k][k_].StatisticsSeries.Mean = make([]schema.Float, len(v_.StatisticsSeries.Mean)) - // Copy Data - copy(jd[k][k_].StatisticsSeries.Max, v_.StatisticsSeries.Max) - copy(jd[k][k_].StatisticsSeries.Min, v_.StatisticsSeries.Min) - copy(jd[k][k_].StatisticsSeries.Median, v_.StatisticsSeries.Median) - copy(jd[k][k_].StatisticsSeries.Mean, v_.StatisticsSeries.Mean) - // Handle Percentiles - for k__, v__ := range v_.StatisticsSeries.Percentiles { - jd[k][k_].StatisticsSeries.Percentiles[k__] = make([]schema.Float, len(v__)) - copy(jd[k][k_].StatisticsSeries.Percentiles[k__], v__) - } - } else { - jd[k][k_].StatisticsSeries = v_.StatisticsSeries - } - } - } - return jd -} diff --git a/internal/metricdispatch/dataLoader.go b/internal/metricdispatch/dataLoader.go new file mode 100644 index 00000000..8bfebbd6 --- /dev/null +++ b/internal/metricdispatch/dataLoader.go @@ -0,0 +1,490 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package metricdispatch provides a unified interface for loading and caching job metric data. +// +// This package serves as a central dispatcher that routes metric data requests to the appropriate +// backend based on job state. For running jobs, data is fetched from the metric store (e.g., cc-metric-store). +// For completed jobs, data is retrieved from the file-based job archive. +// +// # Key Features +// +// - Automatic backend selection based on job state (running vs. archived) +// - LRU cache for performance optimization (128 MB default cache size) +// - Data resampling using Largest Triangle Three Bucket algorithm for archived data +// - Automatic statistics series generation for jobs with many nodes +// - Support for scoped metrics (node, socket, accelerator, core) +// +// # Cache Behavior +// +// Cached data has different TTL (time-to-live) values depending on job state: +// - Running jobs: 2 minutes (data changes frequently) +// - Completed jobs: 5 hours (data is static) +// +// The cache key is based on job ID, state, requested metrics, scopes, and resolution. +// +// # Usage +// +// The primary entry point is LoadData, which automatically handles both running and archived jobs: +// +// jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution) +// if err != nil { +// // Handle error +// } +// +// For statistics only, use LoadJobStats, LoadScopedJobStats, or LoadAverages depending on the required format. +package metricdispatch + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" + "github.com/ClusterCockpit/cc-backend/pkg/archive" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/lrucache" + "github.com/ClusterCockpit/cc-lib/v2/resampler" + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +// cache is an LRU cache with 128 MB capacity for storing loaded job metric data. +// The cache reduces load on both the metric store and archive backends. +var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024) + +// cacheKey generates a unique cache key for a job's metric data based on job ID, state, +// requested metrics, scopes, and resolution. Duration and StartTime are intentionally excluded +// because job.ID is more unique and the cache TTL ensures entries don't persist indefinitely. +func cacheKey( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + resolution int, +) string { + return fmt.Sprintf("%d(%s):[%v],[%v]-%d", + job.ID, job.State, metrics, scopes, resolution) +} + +// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs, +// archive for completed jobs) and applies caching, resampling, and statistics generation as needed. +// +// For running jobs or when archive is disabled, data is fetched from the metric store. +// For completed archived jobs, data is loaded from the job archive and resampled if needed. +// +// Parameters: +// - job: The job for which to load metric data +// - metrics: List of metric names to load (nil loads all metrics for the cluster) +// - scopes: Metric scopes to include (nil defaults to node scope) +// - ctx: Context for cancellation and timeouts +// - resolution: Target number of data points for resampling (only applies to archived data) +// +// Returns the loaded job data and any error encountered. For partial errors (some metrics failed), +// the function returns the successfully loaded data with a warning logged. +func LoadData(job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context, + resolution int, +) (schema.JobData, error) { + data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) { + var jd schema.JobData + var err error + + if job.State == schema.JobStateRunning || + job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving || + config.Keys.DisableArchive { + + if scopes == nil { + scopes = append(scopes, schema.MetricScopeNode) + } + + if metrics == nil { + cluster := archive.GetCluster(job.Cluster) + for _, mc := range cluster.MetricConfig { + metrics = append(metrics, mc.Name) + } + } + + jd, err = metricstore.LoadData(job, metrics, scopes, ctx, resolution) + if err != nil { + if len(jd) != 0 { + cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + } else { + cclog.Errorf("failed to load job data from metric store for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + return err, 0, 0 + } + } + size = jd.Size() + } else { + var jdTemp schema.JobData + jdTemp, err = archive.GetHandle().LoadJobData(job) + if err != nil { + cclog.Errorf("failed to load job data from archive for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + return err, 0, 0 + } + + jd = deepCopy(jdTemp) + + // Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points + // to the requested resolution, improving transfer performance and client-side rendering. + for _, v := range jd { + for _, v_ := range v { + timestep := int64(0) + for i := 0; i < len(v_.Series); i += 1 { + v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution)) + if err != nil { + return err, 0, 0 + } + } + v_.Timestep = int(timestep) + } + } + + // Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer. + if metrics != nil || scopes != nil { + if metrics == nil { + metrics = make([]string, 0, len(jd)) + for k := range jd { + metrics = append(metrics, k) + } + } + + res := schema.JobData{} + for _, metric := range metrics { + if perscope, ok := jd[metric]; ok { + if len(perscope) > 1 { + subset := make(map[schema.MetricScope]*schema.JobMetric) + for _, scope := range scopes { + if jm, ok := perscope[scope]; ok { + subset[scope] = jm + } + } + + if len(subset) > 0 { + perscope = subset + } + } + + res[metric] = perscope + } + } + jd = res + } + size = jd.Size() + } + + ttl = 5 * time.Hour + if job.State == schema.JobStateRunning { + ttl = 2 * time.Minute + } + + // Generate statistics series for jobs with many nodes to enable min/median/max graphs + // instead of overwhelming the UI with individual node lines. Note that newly calculated + // statistics use min/median/max, while archived statistics may use min/mean/max. + const maxSeriesSize int = 15 + for _, scopes := range jd { + for _, jm := range scopes { + if jm.StatisticsSeries != nil || len(jm.Series) <= maxSeriesSize { + continue + } + + jm.AddStatisticsSeries() + } + } + + nodeScopeRequested := false + for _, scope := range scopes { + if scope == schema.MetricScopeNode { + nodeScopeRequested = true + } + } + + if nodeScopeRequested { + jd.AddNodeScope("flops_any") + jd.AddNodeScope("mem_bw") + } + + // Round Resulting Stat Values + jd.RoundMetricStats() + + return jd, ttl, size + }) + + if err, ok := data.(error); ok { + cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error()) + return nil, err + } + + return data.(schema.JobData), nil +} + +// LoadAverages computes average values for the specified metrics across all nodes of a job. +// For running jobs, it loads statistics from the metric store. For completed jobs, it uses +// the pre-calculated averages from the job archive. The results are appended to the data slice. +func LoadAverages( + job *schema.Job, + metrics []string, + data [][]schema.Float, + ctx context.Context, +) error { + if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { + return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here? + } + + stats, err := metricstore.LoadStats(job, metrics, ctx) + if err != nil { + cclog.Errorf("failed to load statistics from metric store for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + return err + } + + for i, m := range metrics { + nodes, ok := stats[m] + if !ok { + data[i] = append(data[i], schema.NaN) + continue + } + + sum := 0.0 + for _, node := range nodes { + sum += node.Avg + } + data[i] = append(data[i], schema.Float(sum)) + } + + return nil +} + +// LoadScopedJobStats retrieves job statistics organized by metric scope (node, socket, core, accelerator). +// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated +// statistics are loaded from the job archive. +func LoadScopedJobStats( + job *schema.Job, + metrics []string, + scopes []schema.MetricScope, + ctx context.Context, +) (schema.ScopedJobStats, error) { + if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { + return archive.LoadScopedStatsFromArchive(job, metrics, scopes) + } + + scopedStats, err := metricstore.LoadScopedStats(job, metrics, scopes, ctx) + if err != nil { + cclog.Errorf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + return nil, err + } + + return scopedStats, nil +} + +// LoadJobStats retrieves aggregated statistics (min/avg/max) for each requested metric across all job nodes. +// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated +// statistics are loaded from the job archive. +func LoadJobStats( + job *schema.Job, + metrics []string, + ctx context.Context, +) (map[string]schema.MetricStatistics, error) { + if job.State != schema.JobStateRunning && !config.Keys.DisableArchive { + return archive.LoadStatsFromArchive(job, metrics) + } + + data := make(map[string]schema.MetricStatistics, len(metrics)) + + stats, err := metricstore.LoadStats(job, metrics, ctx) + if err != nil { + cclog.Errorf("failed to load statistics from metric store for job %d (user: %s, project: %s): %s", + job.JobID, job.User, job.Project, err.Error()) + return data, err + } + + for _, m := range metrics { + sum, avg, min, max := 0.0, 0.0, 0.0, 0.0 + nodes, ok := stats[m] + if !ok { + data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max} + continue + } + + for _, node := range nodes { + sum += node.Avg + min = math.Min(min, node.Min) + max = math.Max(max, node.Max) + } + + data[m] = schema.MetricStatistics{ + Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100), + Min: (math.Round(min*100) / 100), + Max: (math.Round(max*100) / 100), + } + } + + return data, nil +} + +// LoadNodeData retrieves metric data for specific nodes in a cluster within a time range. +// This is used for node monitoring views and system status pages. Data is always fetched from +// the metric store (not the archive) since it's for current/recent node status monitoring. +// +// Returns a nested map structure: node -> metric -> scoped data. +func LoadNodeData( + cluster string, + metrics, nodes []string, + scopes []schema.MetricScope, + from, to time.Time, + ctx context.Context, +) (map[string]map[string][]*schema.JobMetric, error) { + if metrics == nil { + for _, m := range archive.GetCluster(cluster).MetricConfig { + metrics = append(metrics, m.Name) + } + } + + data, err := metricstore.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) + if err != nil { + if len(data) != 0 { + cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error()) + } else { + cclog.Errorf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error()) + return nil, err + } + } + + if data == nil { + return nil, fmt.Errorf("metric store for cluster '%s' does not support node data queries", cluster) + } + + return data, nil +} + +// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range, +// with optional resampling and automatic statistics generation for large datasets. +// This is used for comparing multiple nodes or displaying node status over time. +// +// Returns a map of node names to their job-like metric data structures. +func LoadNodeListData( + cluster, subCluster string, + nodes []string, + metrics []string, + scopes []schema.MetricScope, + resolution int, + from, to time.Time, + ctx context.Context, +) (map[string]schema.JobData, error) { + if metrics == nil { + for _, m := range archive.GetCluster(cluster).MetricConfig { + metrics = append(metrics, m.Name) + } + } + + data, err := metricstore.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx) + if err != nil { + if len(data) != 0 { + cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s", + cluster, subCluster, err.Error()) + } else { + cclog.Errorf("failed to load node list data from metric store for cluster %s, subcluster %s: %s", + cluster, subCluster, err.Error()) + return nil, err + } + } + + // Generate statistics series for datasets with many series to improve visualization performance. + // Statistics are calculated as min/median/max. + const maxSeriesSize int = 8 + for _, jd := range data { + for _, scopes := range jd { + for _, jm := range scopes { + if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize { + continue + } + jm.AddStatisticsSeries() + } + } + } + + if data == nil { + return nil, fmt.Errorf("metric store for cluster '%s' does not support node list queries", cluster) + } + + return data, nil +} + +// deepCopy creates a deep copy of JobData to prevent cache corruption when modifying +// archived data (e.g., during resampling). This ensures the cached archive data remains +// immutable while allowing per-request transformations. +func deepCopy(source schema.JobData) schema.JobData { + result := make(schema.JobData, len(source)) + + for metricName, scopeMap := range source { + result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap)) + + for scope, jobMetric := range scopeMap { + result[metricName][scope] = copyJobMetric(jobMetric) + } + } + + return result +} + +func copyJobMetric(src *schema.JobMetric) *schema.JobMetric { + dst := &schema.JobMetric{ + Timestep: src.Timestep, + Unit: src.Unit, + Series: make([]schema.Series, len(src.Series)), + } + + for i := range src.Series { + dst.Series[i] = copySeries(&src.Series[i]) + } + + if src.StatisticsSeries != nil { + dst.StatisticsSeries = copyStatisticsSeries(src.StatisticsSeries) + } + + return dst +} + +func copySeries(src *schema.Series) schema.Series { + dst := schema.Series{ + Hostname: src.Hostname, + Id: src.Id, + Statistics: src.Statistics, + Data: make([]schema.Float, len(src.Data)), + } + + copy(dst.Data, src.Data) + return dst +} + +func copyStatisticsSeries(src *schema.StatsSeries) *schema.StatsSeries { + dst := &schema.StatsSeries{ + Min: make([]schema.Float, len(src.Min)), + Mean: make([]schema.Float, len(src.Mean)), + Median: make([]schema.Float, len(src.Median)), + Max: make([]schema.Float, len(src.Max)), + } + + copy(dst.Min, src.Min) + copy(dst.Mean, src.Mean) + copy(dst.Median, src.Median) + copy(dst.Max, src.Max) + + if len(src.Percentiles) > 0 { + dst.Percentiles = make(map[int][]schema.Float, len(src.Percentiles)) + for percentile, values := range src.Percentiles { + dst.Percentiles[percentile] = make([]schema.Float, len(values)) + copy(dst.Percentiles[percentile], values) + } + } + + return dst +} diff --git a/internal/metricdispatch/dataLoader_test.go b/internal/metricdispatch/dataLoader_test.go new file mode 100644 index 00000000..c4841f8d --- /dev/null +++ b/internal/metricdispatch/dataLoader_test.go @@ -0,0 +1,125 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package metricdispatch + +import ( + "testing" + + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +func TestDeepCopy(t *testing.T) { + nodeId := "0" + original := schema.JobData{ + "cpu_load": { + schema.MetricScopeNode: &schema.JobMetric{ + Timestep: 60, + Unit: schema.Unit{Base: "load", Prefix: ""}, + Series: []schema.Series{ + { + Hostname: "node001", + Id: &nodeId, + Data: []schema.Float{1.0, 2.0, 3.0}, + Statistics: schema.MetricStatistics{ + Min: 1.0, + Avg: 2.0, + Max: 3.0, + }, + }, + }, + StatisticsSeries: &schema.StatsSeries{ + Min: []schema.Float{1.0, 1.5, 2.0}, + Mean: []schema.Float{2.0, 2.5, 3.0}, + Median: []schema.Float{2.0, 2.5, 3.0}, + Max: []schema.Float{3.0, 3.5, 4.0}, + Percentiles: map[int][]schema.Float{ + 25: {1.5, 2.0, 2.5}, + 75: {2.5, 3.0, 3.5}, + }, + }, + }, + }, + } + + copied := deepCopy(original) + + original["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] = 999.0 + original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] = 888.0 + original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] = 777.0 + + if copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] != 1.0 { + t.Errorf("Series data was not deeply copied: got %v, want 1.0", + copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0]) + } + + if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] != 1.0 { + t.Errorf("StatisticsSeries was not deeply copied: got %v, want 1.0", + copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0]) + } + + if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] != 1.5 { + t.Errorf("Percentiles was not deeply copied: got %v, want 1.5", + copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0]) + } + + if copied["cpu_load"][schema.MetricScopeNode].Timestep != 60 { + t.Errorf("Timestep not copied correctly: got %v, want 60", + copied["cpu_load"][schema.MetricScopeNode].Timestep) + } + + if copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname != "node001" { + t.Errorf("Hostname not copied correctly: got %v, want node001", + copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname) + } +} + +func TestDeepCopyNilStatisticsSeries(t *testing.T) { + original := schema.JobData{ + "mem_used": { + schema.MetricScopeNode: &schema.JobMetric{ + Timestep: 60, + Series: []schema.Series{ + { + Hostname: "node001", + Data: []schema.Float{1.0, 2.0}, + }, + }, + StatisticsSeries: nil, + }, + }, + } + + copied := deepCopy(original) + + if copied["mem_used"][schema.MetricScopeNode].StatisticsSeries != nil { + t.Errorf("StatisticsSeries should be nil, got %v", + copied["mem_used"][schema.MetricScopeNode].StatisticsSeries) + } +} + +func TestDeepCopyEmptyPercentiles(t *testing.T) { + original := schema.JobData{ + "cpu_load": { + schema.MetricScopeNode: &schema.JobMetric{ + Timestep: 60, + Series: []schema.Series{}, + StatisticsSeries: &schema.StatsSeries{ + Min: []schema.Float{1.0}, + Mean: []schema.Float{2.0}, + Median: []schema.Float{2.0}, + Max: []schema.Float{3.0}, + Percentiles: nil, + }, + }, + }, + } + + copied := deepCopy(original) + + if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles != nil { + t.Errorf("Percentiles should be nil when source is nil/empty") + } +} diff --git a/internal/memorystore/api.go b/internal/metricstore/api.go similarity index 98% rename from internal/memorystore/api.go rename to internal/metricstore/api.go index 41c53a18..d8a2ea82 100644 --- a/internal/memorystore/api.go +++ b/internal/metricstore/api.go @@ -3,10 +3,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "errors" + "fmt" "math" "github.com/ClusterCockpit/cc-lib/v2/schema" @@ -124,6 +125,9 @@ func FetchData(req APIQueryRequest) (*APIQueryResponse, error) { req.WithData = true ms := GetMemoryStore() + if ms == nil { + return nil, fmt.Errorf("memorystore not initialized") + } response := APIQueryResponse{ Results: make([][]APIMetricData, 0, len(req.Queries)), diff --git a/internal/memorystore/archive.go b/internal/metricstore/archive.go similarity index 99% rename from internal/memorystore/archive.go rename to internal/metricstore/archive.go index fc46dac6..972769fd 100644 --- a/internal/memorystore/archive.go +++ b/internal/metricstore/archive.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "archive/zip" diff --git a/internal/memorystore/avroCheckpoint.go b/internal/metricstore/avroCheckpoint.go similarity index 99% rename from internal/memorystore/avroCheckpoint.go rename to internal/metricstore/avroCheckpoint.go index b0b0cf42..275a64bd 100644 --- a/internal/memorystore/avroCheckpoint.go +++ b/internal/metricstore/avroCheckpoint.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "bufio" diff --git a/internal/memorystore/avroHelper.go b/internal/metricstore/avroHelper.go similarity index 99% rename from internal/memorystore/avroHelper.go rename to internal/metricstore/avroHelper.go index 93a293bd..5587a58d 100644 --- a/internal/memorystore/avroHelper.go +++ b/internal/metricstore/avroHelper.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "context" diff --git a/internal/memorystore/avroStruct.go b/internal/metricstore/avroStruct.go similarity index 99% rename from internal/memorystore/avroStruct.go rename to internal/metricstore/avroStruct.go index 2643a9a7..78a8d137 100644 --- a/internal/memorystore/avroStruct.go +++ b/internal/metricstore/avroStruct.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "sync" diff --git a/internal/memorystore/buffer.go b/internal/metricstore/buffer.go similarity index 99% rename from internal/memorystore/buffer.go rename to internal/metricstore/buffer.go index 15e29b3a..94d3ce76 100644 --- a/internal/memorystore/buffer.go +++ b/internal/metricstore/buffer.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "errors" diff --git a/internal/memorystore/checkpoint.go b/internal/metricstore/checkpoint.go similarity index 99% rename from internal/memorystore/checkpoint.go rename to internal/metricstore/checkpoint.go index c48c2fd8..27d611c4 100644 --- a/internal/memorystore/checkpoint.go +++ b/internal/metricstore/checkpoint.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "bufio" diff --git a/internal/memorystore/config.go b/internal/metricstore/config.go similarity index 98% rename from internal/memorystore/config.go rename to internal/metricstore/config.go index fbd62341..97f16c46 100644 --- a/internal/memorystore/config.go +++ b/internal/metricstore/config.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "fmt" @@ -19,8 +19,6 @@ const ( DefaultAvroCheckpointInterval = time.Minute ) -var InternalCCMSFlag bool = false - type MetricStoreConfig struct { // Number of concurrent workers for checkpoint and archive operations. // If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10) diff --git a/internal/memorystore/configSchema.go b/internal/metricstore/configSchema.go similarity index 99% rename from internal/memorystore/configSchema.go rename to internal/metricstore/configSchema.go index 2616edc6..f1a20a73 100644 --- a/internal/memorystore/configSchema.go +++ b/internal/metricstore/configSchema.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore const configSchema = `{ "type": "object", diff --git a/internal/memorystore/debug.go b/internal/metricstore/debug.go similarity index 99% rename from internal/memorystore/debug.go rename to internal/metricstore/debug.go index b56cf254..50c91e08 100644 --- a/internal/memorystore/debug.go +++ b/internal/metricstore/debug.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "bufio" diff --git a/internal/memorystore/healthcheck.go b/internal/metricstore/healthcheck.go similarity index 99% rename from internal/memorystore/healthcheck.go rename to internal/metricstore/healthcheck.go index b1052f3b..2a49c47a 100644 --- a/internal/memorystore/healthcheck.go +++ b/internal/metricstore/healthcheck.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "bufio" diff --git a/internal/memorystore/level.go b/internal/metricstore/level.go similarity index 99% rename from internal/memorystore/level.go rename to internal/metricstore/level.go index bce2a7a6..d46f893a 100644 --- a/internal/memorystore/level.go +++ b/internal/metricstore/level.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "sync" diff --git a/internal/memorystore/lineprotocol.go b/internal/metricstore/lineprotocol.go similarity index 99% rename from internal/memorystore/lineprotocol.go rename to internal/metricstore/lineprotocol.go index ca8cc811..cc59e213 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/metricstore/lineprotocol.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "context" diff --git a/internal/memorystore/memorystore.go b/internal/metricstore/memorystore.go similarity index 99% rename from internal/memorystore/memorystore.go rename to internal/metricstore/memorystore.go index 7c5ea0eb..14a02fcd 100644 --- a/internal/memorystore/memorystore.go +++ b/internal/metricstore/memorystore.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// Package memorystore provides an efficient in-memory time-series metric storage system +// Package metricstore provides an efficient in-memory time-series metric storage system // with support for hierarchical data organization, checkpointing, and archiving. // // The package organizes metrics in a tree structure (cluster → host → component) and @@ -17,7 +17,7 @@ // - Concurrent checkpoint/archive workers // - Support for sum and average aggregation // - NATS integration for metric ingestion -package memorystore +package metricstore import ( "bytes" diff --git a/internal/memorystore/memorystore_test.go b/internal/metricstore/memorystore_test.go similarity index 99% rename from internal/memorystore/memorystore_test.go rename to internal/metricstore/memorystore_test.go index 57ea6938..29379d21 100644 --- a/internal/memorystore/memorystore_test.go +++ b/internal/metricstore/memorystore_test.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "testing" diff --git a/internal/metricdata/cc-metric-store-internal.go b/internal/metricstore/query.go similarity index 87% rename from internal/metricdata/cc-metric-store-internal.go rename to internal/metricstore/query.go index 741ce358..78c78dd5 100644 --- a/internal/metricdata/cc-metric-store-internal.go +++ b/internal/metricstore/query.go @@ -3,56 +3,41 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package metricdata +package metricstore import ( "context" - "encoding/json" "fmt" "strconv" "strings" "time" - "github.com/ClusterCockpit/cc-backend/internal/memorystore" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" ) -// Bloat Code -type CCMetricStoreConfigInternal struct { - Kind string `json:"kind"` - Url string `json:"url"` - Token string `json:"token"` +// TestLoadDataCallback allows tests to override LoadData behavior +var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) - // If metrics are known to this MetricDataRepository under a different - // name than in the `metricConfig` section of the 'cluster.json', - // provide this optional mapping of local to remote name for this metric. - Renamings map[string]string `json:"metricRenamings"` -} - -// Bloat Code -type CCMetricStoreInternal struct{} - -// Bloat Code -func (ccms *CCMetricStoreInternal) Init(rawConfig json.RawMessage) error { - return nil -} - -func (ccms *CCMetricStoreInternal) LoadData( +func LoadData( job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, ) (schema.JobData, error) { - queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, int64(resolution)) + if TestLoadDataCallback != nil { + return TestLoadDataCallback(job, metrics, scopes, ctx, resolution) + } + + queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution)) if err != nil { cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) return nil, err } - req := memorystore.APIQueryRequest{ + req := APIQueryRequest{ Cluster: job.Cluster, From: job.StartTime, To: job.StartTime + int64(job.Duration), @@ -61,7 +46,7 @@ func (ccms *CCMetricStoreInternal) LoadData( WithData: true, } - resBody, err := memorystore.FetchData(req) + resBody, err := FetchData(req) if err != nil { cclog.Errorf("Error while fetching data : %s", err.Error()) return nil, err @@ -149,13 +134,13 @@ var ( acceleratorString = string(schema.MetricScopeAccelerator) ) -func (ccms *CCMetricStoreInternal) buildQueries( +func buildQueries( job *schema.Job, metrics []string, scopes []schema.MetricScope, resolution int64, -) ([]memorystore.APIQuery, []schema.MetricScope, error) { - queries := make([]memorystore.APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) +) ([]APIQuery, []schema.MetricScope, error) { + queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) assignedScope := []schema.MetricScope{} subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) @@ -217,7 +202,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( continue } - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: false, @@ -235,7 +220,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( continue } - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -249,7 +234,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // HWThread -> HWThead if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: false, @@ -265,7 +250,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { cores, _ := topology.GetCoresFromHWThreads(hwthreads) for _, core := range cores { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -282,7 +267,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) for _, socket := range sockets { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -297,7 +282,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // HWThread -> Node if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -312,7 +297,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // Core -> Core if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: false, @@ -328,7 +313,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromCores(hwthreads) for _, socket := range sockets { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -344,7 +329,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // Core -> Node if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -359,7 +344,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // MemoryDomain -> MemoryDomain if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: false, @@ -374,7 +359,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // MemoryDoman -> Node if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -389,7 +374,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // Socket -> Socket if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: false, @@ -404,7 +389,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // Socket -> Node if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Aggregate: true, @@ -418,7 +403,7 @@ func (ccms *CCMetricStoreInternal) buildQueries( // Node -> Node if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: host.Hostname, Resolution: resolution, @@ -435,18 +420,18 @@ func (ccms *CCMetricStoreInternal) buildQueries( return queries, assignedScope, nil } -func (ccms *CCMetricStoreInternal) LoadStats( +func LoadStats( job *schema.Job, metrics []string, ctx context.Context, ) (map[string]map[string]schema.MetricStatistics, error) { - queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization? + queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization? if err != nil { cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error()) return nil, err } - req := memorystore.APIQueryRequest{ + req := APIQueryRequest{ Cluster: job.Cluster, From: job.StartTime, To: job.StartTime + int64(job.Duration), @@ -455,7 +440,7 @@ func (ccms *CCMetricStoreInternal) LoadStats( WithData: false, } - resBody, err := memorystore.FetchData(req) + resBody, err := FetchData(req) if err != nil { cclog.Errorf("Error while fetching data : %s", err.Error()) return nil, err @@ -492,20 +477,19 @@ func (ccms *CCMetricStoreInternal) LoadStats( return stats, nil } -// Used for Job-View Statistics Table -func (ccms *CCMetricStoreInternal) LoadScopedStats( +func LoadScopedStats( job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, ) (schema.ScopedJobStats, error) { - queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0) + queries, assignedScope, err := buildQueries(job, metrics, scopes, 0) if err != nil { cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error()) return nil, err } - req := memorystore.APIQueryRequest{ + req := APIQueryRequest{ Cluster: job.Cluster, From: job.StartTime, To: job.StartTime + int64(job.Duration), @@ -514,7 +498,7 @@ func (ccms *CCMetricStoreInternal) LoadScopedStats( WithData: false, } - resBody, err := memorystore.FetchData(req) + resBody, err := FetchData(req) if err != nil { cclog.Errorf("Error while fetching data : %s", err.Error()) return nil, err @@ -583,15 +567,14 @@ func (ccms *CCMetricStoreInternal) LoadScopedStats( return scopedJobStats, nil } -// Used for Systems-View Node-Overview -func (ccms *CCMetricStoreInternal) LoadNodeData( +func LoadNodeData( cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context, ) (map[string]map[string][]*schema.JobMetric, error) { - req := memorystore.APIQueryRequest{ + req := APIQueryRequest{ Cluster: cluster, From: from.Unix(), To: to.Unix(), @@ -604,7 +587,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeData( } else { for _, node := range nodes { for _, metric := range metrics { - req.Queries = append(req.Queries, memorystore.APIQuery{ + req.Queries = append(req.Queries, APIQuery{ Hostname: node, Metric: metric, Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution @@ -613,7 +596,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeData( } } - resBody, err := memorystore.FetchData(req) + resBody, err := FetchData(req) if err != nil { cclog.Errorf("Error while fetching data : %s", err.Error()) return nil, err @@ -622,7 +605,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeData( var errors []string data := make(map[string]map[string][]*schema.JobMetric) for i, res := range resBody.Results { - var query memorystore.APIQuery + var query APIQuery if resBody.Queries != nil { query = resBody.Queries[i] } else { @@ -673,8 +656,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeData( return data, nil } -// Used for Systems-View Node-List -func (ccms *CCMetricStoreInternal) LoadNodeListData( +func LoadNodeListData( cluster, subCluster string, nodes []string, metrics []string, @@ -683,15 +665,14 @@ func (ccms *CCMetricStoreInternal) LoadNodeListData( from, to time.Time, ctx context.Context, ) (map[string]schema.JobData, error) { - // Note: Order of node data is not guaranteed after this point - queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution)) + queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution)) if err != nil { cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error()) return nil, err } - req := memorystore.APIQueryRequest{ + req := APIQueryRequest{ Cluster: cluster, Queries: queries, From: from.Unix(), @@ -700,7 +681,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeListData( WithData: true, } - resBody, err := memorystore.FetchData(req) + resBody, err := FetchData(req) if err != nil { cclog.Errorf("Error while fetching data : %s", err.Error()) return nil, err @@ -709,7 +690,7 @@ func (ccms *CCMetricStoreInternal) LoadNodeListData( var errors []string data := make(map[string]schema.JobData) for i, row := range resBody.Results { - var query memorystore.APIQuery + var query APIQuery if resBody.Queries != nil { query = resBody.Queries[i] } else { @@ -789,15 +770,15 @@ func (ccms *CCMetricStoreInternal) LoadNodeListData( return data, nil } -func (ccms *CCMetricStoreInternal) buildNodeQueries( +func buildNodeQueries( cluster string, subCluster string, nodes []string, metrics []string, scopes []schema.MetricScope, resolution int64, -) ([]memorystore.APIQuery, []schema.MetricScope, error) { - queries := make([]memorystore.APIQuery, 0, len(metrics)*len(scopes)*len(nodes)) +) ([]APIQuery, []schema.MetricScope, error) { + queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes)) assignedScope := []schema.MetricScope{} // Get Topol before loop if subCluster given @@ -812,7 +793,6 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( } for _, metric := range metrics { - metric := metric mc := archive.GetMetricConfig(cluster, metric) if mc == nil { // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, cluster) @@ -880,7 +860,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( continue } - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: false, @@ -898,7 +878,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( continue } - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -912,7 +892,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // HWThread -> HWThead if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: false, @@ -928,7 +908,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { cores, _ := topology.GetCoresFromHWThreads(topology.Node) for _, core := range cores { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -945,7 +925,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) for _, socket := range sockets { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -960,7 +940,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // HWThread -> Node if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -975,7 +955,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // Core -> Core if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: false, @@ -991,7 +971,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromCores(topology.Node) for _, socket := range sockets { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -1007,7 +987,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // Core -> Node if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -1022,7 +1002,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // MemoryDomain -> MemoryDomain if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: false, @@ -1037,7 +1017,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // MemoryDoman -> Node if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -1052,7 +1032,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // Socket -> Socket if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: false, @@ -1067,7 +1047,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // Socket -> Node if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Aggregate: true, @@ -1081,7 +1061,7 @@ func (ccms *CCMetricStoreInternal) buildNodeQueries( // Node -> Node if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, memorystore.APIQuery{ + queries = append(queries, APIQuery{ Metric: metric, Hostname: hostname, Resolution: resolution, diff --git a/internal/memorystore/stats.go b/internal/metricstore/stats.go similarity index 99% rename from internal/memorystore/stats.go rename to internal/metricstore/stats.go index c931ab35..51ffafc1 100644 --- a/internal/memorystore/stats.go +++ b/internal/metricstore/stats.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package memorystore +package metricstore import ( "errors" diff --git a/internal/metricsync/metricdata.go b/internal/metricsync/metricdata.go new file mode 100644 index 00000000..772f16da --- /dev/null +++ b/internal/metricsync/metricdata.go @@ -0,0 +1,60 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package metricsync + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/config" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +type MetricDataSource interface { + // Initialize this MetricDataRepository. One instance of + // this interface will only ever be responsible for one cluster. + Init(rawConfig json.RawMessage) error + + // Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node. + Pull(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) +} + +var metricDataSourceRepos map[string]MetricDataSource = map[string]MetricDataSource{} + +func Init() error { + for _, cluster := range config.Clusters { + if cluster.MetricDataRepository != nil { + var kind struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { + cclog.Warn("Error while unmarshaling raw json MetricDataRepository") + return err + } + + var mdr MetricDataSource + switch kind.Kind { + case "cc-metric-store": + case "prometheus": + // mdr = &PrometheusDataRepository{} + case "test": + // mdr = &TestMetricDataRepository{} + default: + return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) + } + + if err := mdr.Init(cluster.MetricDataRepository); err != nil { + cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) + return err + } + metricDataSourceRepos[cluster.Name] = mdr + } + } + return nil +} diff --git a/internal/repository/stats.go b/internal/repository/stats.go index d1e16eb8..989026d1 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -12,7 +12,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" - "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" + "github.com/ClusterCockpit/cc-backend/internal/metricdispatch" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" @@ -766,7 +766,7 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram( continue } - if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil { + if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil { cclog.Errorf("Error while loading averages for histogram: %s", err) return nil } diff --git a/internal/taskmanager/updateFootprintService.go b/internal/taskmanager/updateFootprintService.go index 979a6137..c8f81e37 100644 --- a/internal/taskmanager/updateFootprintService.go +++ b/internal/taskmanager/updateFootprintService.go @@ -10,7 +10,7 @@ import ( "math" "time" - "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/pkg/archive" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" @@ -58,12 +58,6 @@ func RegisterFootprintWorker() { allMetrics = append(allMetrics, mc.Name) } - repo, err := metricdata.GetMetricDataRepo(cluster.Name) - if err != nil { - cclog.Errorf("no metric data repository configured for '%s'", cluster.Name) - continue - } - pendingStatements := []sq.UpdateBuilder{} for _, job := range jobs { @@ -72,7 +66,7 @@ func RegisterFootprintWorker() { sJob := time.Now() - jobStats, err := repo.LoadStats(job, allMetrics, context.Background()) + jobStats, err := metricstore.LoadStats(job, allMetrics, context.Background()) if err != nil { cclog.Errorf("error wile loading job data stats for footprint update: %v", err) ce++ From ecb5aef7355b498d2f84e1837e536ab5aa69a2d0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 25 Dec 2025 08:48:03 +0100 Subject: [PATCH 057/341] Fix build error in unit test --- internal/api/nats_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index 9e1fa2b5..e92ce291 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -18,7 +18,6 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph" - "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/metricstore" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" From ddc2ecf82976c75b963604744ebce522f1cc6a3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 08:02:46 +0000 Subject: [PATCH 058/341] Bump svelte from 5.44.0 to 5.46.1 in /web/frontend Bumps [svelte](https://github.com/sveltejs/svelte/tree/HEAD/packages/svelte) from 5.44.0 to 5.46.1. - [Release notes](https://github.com/sveltejs/svelte/releases) - [Changelog](https://github.com/sveltejs/svelte/blob/main/packages/svelte/CHANGELOG.md) - [Commits](https://github.com/sveltejs/svelte/commits/svelte@5.46.1/packages/svelte) --- updated-dependencies: - dependency-name: svelte dependency-version: 5.46.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- web/frontend/package-lock.json | 21 +++++++++++++-------- web/frontend/package.json | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 4c7e4bf5..648c72ce 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -27,7 +27,7 @@ "rollup": "^4.53.3", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.44.0" + "svelte": "^5.46.1" } }, "node_modules/@0no-co/graphql.web": { @@ -621,6 +621,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -746,9 +747,9 @@ "license": "MIT" }, "node_modules/esrap": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.1.3.tgz", - "integrity": "sha512-T/Dhhv/QH+yYmiaLz9SA3PW+YyenlnRKDNdtlYJrSOBmNsH4nvPux+mTwx7p+wAedlJrGoZtXNI0a0MjQ2QkVg==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.1.tgz", + "integrity": "sha512-GiYWG34AN/4CUyaWAgunGt0Rxvr1PTMlGC0vvEov/uOQYWne2bpN03Um+k8jT+q3op33mKouP2zeJ6OlM+qeUg==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" @@ -821,6 +822,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.12.0.tgz", "integrity": "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -927,6 +929,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -981,6 +984,7 @@ "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -1157,10 +1161,11 @@ } }, "node_modules/svelte": { - "version": "5.44.0", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.44.0.tgz", - "integrity": "sha512-R7387No2zEGw4CtYtI2rgsui6BqjFARzoZFGLiLN5OPla0Pq4Ra2WwcP/zBomP3MYalhSNvF1fzDMuU0P0zPJw==", + "version": "5.46.1", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.46.1.tgz", + "integrity": "sha512-ynjfCHD3nP2el70kN5Pmg37sSi0EjOm9FgHYQdC4giWG/hzO3AatzXXJJgP305uIhGQxSufJLuYWtkY8uK/8RA==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -1172,7 +1177,7 @@ "clsx": "^2.1.1", "devalue": "^5.5.0", "esm-env": "^1.2.1", - "esrap": "^2.1.0", + "esrap": "^2.2.1", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", diff --git a/web/frontend/package.json b/web/frontend/package.json index 3f7434f7..d06ea6b4 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -14,7 +14,7 @@ "rollup": "^4.53.3", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.44.0" + "svelte": "^5.46.1" }, "dependencies": { "@rollup/plugin-replace": "^6.0.3", From 4e6038d6c1a26ac74dd3b0f10bfdef335d86b3a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 08:03:41 +0000 Subject: [PATCH 059/341] Bump github.com/99designs/gqlgen from 0.17.84 to 0.17.85 Bumps [github.com/99designs/gqlgen](https://github.com/99designs/gqlgen) from 0.17.84 to 0.17.85. - [Release notes](https://github.com/99designs/gqlgen/releases) - [Changelog](https://github.com/99designs/gqlgen/blob/master/CHANGELOG.md) - [Commits](https://github.com/99designs/gqlgen/compare/v0.17.84...v0.17.85) --- updated-dependencies: - dependency-name: github.com/99designs/gqlgen dependency-version: 0.17.85 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 4da3b80e..246460a4 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ tool ( ) require ( - github.com/99designs/gqlgen v0.17.84 + github.com/99designs/gqlgen v0.17.85 github.com/ClusterCockpit/cc-lib v1.0.2 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.0 @@ -42,7 +42,7 @@ require ( github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 github.com/vektah/gqlparser/v2 v2.5.31 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 golang.org/x/oauth2 v0.32.0 golang.org/x/time v0.14.0 ) @@ -117,13 +117,13 @@ require ( github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/tools v0.39.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.40.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 773bf31c..6ce3d35b 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/99designs/gqlgen v0.17.84 h1:iVMdiStgUVx/BFkMb0J5GAXlqfqtQ7bqMCYK6v52kQ0= -github.com/99designs/gqlgen v0.17.84/go.mod h1:qjoUqzTeiejdo+bwUg8unqSpeYG42XrcrQboGIezmFA= +github.com/99designs/gqlgen v0.17.85 h1:EkGx3U2FDcxQm8YDLQSpXIAVmpDyZ3IcBMOJi2nH1S0= +github.com/99designs/gqlgen v0.17.85/go.mod h1:yvs8s0bkQlRfqg03YXr3eR4OQUowVhODT/tHzCXnbOU= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= @@ -343,33 +343,33 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -377,19 +377,19 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From faf3a19f0c526227b8c3719863d8edca91bfbbdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 08:04:58 +0000 Subject: [PATCH 060/341] Bump github.com/aws/aws-sdk-go-v2/service/s3 from 1.90.2 to 1.95.0 Bumps [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) from 1.90.2 to 1.95.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.90.2...service/s3/v1.95.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.95.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 4da3b80e..79bc1dfc 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.41.0 github.com/aws/aws-sdk-go-v2/config v1.32.6 github.com/aws/aws-sdk-go-v2/credentials v1.19.6 - github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 github.com/coreos/go-oidc/v3 v3.17.0 github.com/expr-lang/expr v1.17.7 github.com/go-co-op/gocron/v2 v2.18.2 @@ -52,16 +52,16 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect diff --git a/go.sum b/go.sum index 773bf31c..366f3656 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= @@ -46,18 +46,18 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= +github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= From a2414791bfe4b43a60f74ffd81ae71c808902592 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 08:05:04 +0000 Subject: [PATCH 061/341] Bump github.com/go-co-op/gocron/v2 from 2.18.2 to 2.19.0 Bumps [github.com/go-co-op/gocron/v2](https://github.com/go-co-op/gocron) from 2.18.2 to 2.19.0. - [Release notes](https://github.com/go-co-op/gocron/releases) - [Commits](https://github.com/go-co-op/gocron/compare/v2.18.2...v2.19.0) --- updated-dependencies: - dependency-name: github.com/go-co-op/gocron/v2 dependency-version: 2.19.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4da3b80e..95107c1a 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 github.com/coreos/go-oidc/v3 v3.17.0 github.com/expr-lang/expr v1.17.7 - github.com/go-co-op/gocron/v2 v2.18.2 + github.com/go-co-op/gocron/v2 v2.19.0 github.com/go-ldap/ldap/v3 v3.4.12 github.com/go-sql-driver/mysql v1.9.3 github.com/golang-jwt/jwt/v5 v5.3.0 diff --git a/go.sum b/go.sum index 773bf31c..40fb4dcf 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-co-op/gocron/v2 v2.18.2 h1:+5VU41FUXPWSPKLXZQ/77SGzUiPCcakU0v7ENc2H20Q= -github.com/go-co-op/gocron/v2 v2.18.2/go.mod h1:Zii6he+Zfgy5W9B+JKk/KwejFOW0kZTFvHtwIpR4aBI= +github.com/go-co-op/gocron/v2 v2.19.0 h1:OKf2y6LXPs/BgBI2fl8PxUpNAI1DA9Mg+hSeGOS38OU= +github.com/go-co-op/gocron/v2 v2.19.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= From 4d6326b8be9bef9d730ea010ae933d43e43ad9a1 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jan 2026 08:55:31 +0100 Subject: [PATCH 062/341] Remove metricsync --- internal/metricsync/metricdata.go | 60 ------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 internal/metricsync/metricdata.go diff --git a/internal/metricsync/metricdata.go b/internal/metricsync/metricdata.go deleted file mode 100644 index 772f16da..00000000 --- a/internal/metricsync/metricdata.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricsync - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/ClusterCockpit/cc-backend/internal/config" - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -type MetricDataSource interface { - // Initialize this MetricDataRepository. One instance of - // this interface will only ever be responsible for one cluster. - Init(rawConfig json.RawMessage) error - - // Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node. - Pull(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) -} - -var metricDataSourceRepos map[string]MetricDataSource = map[string]MetricDataSource{} - -func Init() error { - for _, cluster := range config.Clusters { - if cluster.MetricDataRepository != nil { - var kind struct { - Kind string `json:"kind"` - } - if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { - cclog.Warn("Error while unmarshaling raw json MetricDataRepository") - return err - } - - var mdr MetricDataSource - switch kind.Kind { - case "cc-metric-store": - case "prometheus": - // mdr = &PrometheusDataRepository{} - case "test": - // mdr = &TestMetricDataRepository{} - default: - return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - } - - if err := mdr.Init(cluster.MetricDataRepository); err != nil { - cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - return err - } - metricDataSourceRepos[cluster.Name] = mdr - } - } - return nil -} From 56399523d7e9582ff7525b4932586415bf48d7ea Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jan 2026 09:00:06 +0100 Subject: [PATCH 063/341] Update module deps --- go.mod | 17 ++++------------- go.sum | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 36ce47b9..c8899162 100644 --- a/go.mod +++ b/go.mod @@ -33,8 +33,6 @@ require ( github.com/linkedin/goavro/v2 v2.14.1 github.com/mattn/go-sqlite3 v1.14.32 github.com/nats-io/nats.go v1.47.0 - github.com/prometheus/client_golang v1.23.2 - github.com/prometheus/common v0.67.4 github.com/qustavo/sqlhooks/v2 v2.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/stretchr/testify v1.11.1 @@ -65,8 +63,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect github.com/aws/smithy-go v1.24.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -86,6 +82,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-yaml v1.19.0 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect @@ -93,24 +90,19 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.2 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.67.4 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sosodev/duration v1.3.1 // indirect + github.com/stmcginnis/gofish v0.20.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/swaggo/files v1.0.1 // indirect github.com/urfave/cli/v2 v2.27.7 // indirect @@ -125,7 +117,6 @@ require ( golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.39.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 9038d960..99c2bdb0 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM= +github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= @@ -142,7 +144,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA= +github.com/google/go-tpm v0.9.7/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark= @@ -192,10 +195,6 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= @@ -219,15 +218,14 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk= +github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g= +github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= +github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs= +github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y= github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= @@ -238,6 +236,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -253,6 +252,7 @@ github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAi github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -264,6 +264,8 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stmcginnis/gofish v0.20.0 h1:hH2V2Qe898F2wWT1loApnkDUrXXiLKqbSlMaH3Y1n08= +github.com/stmcginnis/gofish v0.20.0/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= From 8641d9053d129075a7f7036ed538b66d619f885c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:07:20 +0000 Subject: [PATCH 064/341] Bump golang.org/x/oauth2 from 0.32.0 to 0.34.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.32.0 to 0.34.0. - [Commits](https://github.com/golang/oauth2/compare/v0.32.0...v0.34.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bc428b9a..f96e4537 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/swaggo/swag v1.16.6 github.com/vektah/gqlparser/v2 v2.5.31 golang.org/x/crypto v0.46.0 - golang.org/x/oauth2 v0.32.0 + golang.org/x/oauth2 v0.34.0 golang.org/x/time v0.14.0 ) diff --git a/go.sum b/go.sum index e7b4fd95..e86cb314 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= From 78f1db7ad1b88087847f83a5f02c06977bd6d697 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:52:40 +0000 Subject: [PATCH 065/341] Bump github.com/aws/aws-sdk-go-v2/credentials from 1.19.6 to 1.19.7 Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.19.6 to 1.19.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/m2/v1.19.6...service/m2/v1.19.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 20 ++++++++++---------- go.sum | 40 ++++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index f96e4537..3857483e 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/99designs/gqlgen v0.17.85 github.com/ClusterCockpit/cc-lib v1.0.2 github.com/Masterminds/squirrel v1.5.4 - github.com/aws/aws-sdk-go-v2 v1.41.0 + github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.6 - github.com/aws/aws-sdk-go-v2/credentials v1.19.6 + github.com/aws/aws-sdk-go-v2/credentials v1.19.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 github.com/coreos/go-oidc/v3 v3.17.0 github.com/expr-lang/expr v1.17.7 @@ -53,19 +53,19 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index e86cb314..1a0c7e8b 100644 --- a/go.sum +++ b/go.sum @@ -30,20 +30,20 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= -github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= -github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= @@ -52,20 +52,20 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEd github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0= github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From fae6d9d835bc50f09921f61193d4cba88f0761c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:52:44 +0000 Subject: [PATCH 066/341] Bump github.com/mattn/go-sqlite3 from 1.14.32 to 1.14.33 Bumps [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) from 1.14.32 to 1.14.33. - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.32...v1.14.33) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.33 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f96e4537..04de109a 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/linkedin/goavro/v2 v2.14.1 - github.com/mattn/go-sqlite3 v1.14.32 + github.com/mattn/go-sqlite3 v1.14.33 github.com/nats-io/nats.go v1.47.0 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.4 diff --git a/go.sum b/go.sum index e86cb314..7ed4c1ab 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkF github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= -github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0= +github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= From ad1e87d0b8d84db4a3ababa6a3e3bae107df9f94 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jan 2026 11:17:44 +0100 Subject: [PATCH 067/341] Disable dependabot alerts --- .github/dependabot.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 87600f2c..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,15 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - -version: 2 -updates: - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "weekly" - - package-ecosystem: "npm" - directory: "/web/frontend" - schedule: - interval: "weekly" From 4cec93334964c166435e79c99af6e354d661dde7 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 06:28:33 +0100 Subject: [PATCH 068/341] Remove obsolete cluster config section --- cmd/cc-backend/main.go | 7 +-- internal/api/api_test.go | 6 +- internal/api/nats_test.go | 6 +- internal/config/config.go | 21 +------ internal/config/config_test.go | 12 +--- internal/config/schema.go | 80 -------------------------- internal/importer/importer_test.go | 6 +- internal/repository/node_test.go | 6 +- internal/repository/userConfig_test.go | 6 +- tools/archive-manager/main.go | 6 +- web/web.go | 7 --- 11 files changed, 10 insertions(+), 153 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 331df4f6..8eb3c76f 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -102,12 +102,7 @@ func initConfiguration() error { return fmt.Errorf("main configuration must be present") } - clustercfg := ccconf.GetPackageConfig("clusters") - if clustercfg == nil { - return fmt.Errorf("cluster configuration must be present") - } - - config.Init(cfg, clustercfg) + config.Init(cfg) return nil } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index a2283013..7aa935ff 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -157,11 +157,7 @@ func setup(t *testing.T) *api.RestAPI { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + config.Init(cfg) } else { cclog.Abort("Main configuration must be present") } diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index e92ce291..319668bb 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -151,11 +151,7 @@ func setupNatsTest(t *testing.T) *NatsAPI { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + config.Init(cfg) } else { cclog.Abort("Main configuration must be present") } diff --git a/internal/config/config.go b/internal/config/config.go index af8ec944..b8eea2ca 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -111,14 +111,6 @@ type FilterRanges struct { StartTime *TimeRange `json:"startTime"` } -type ClusterConfig struct { - Name string `json:"name"` - FilterRanges *FilterRanges `json:"filterRanges"` - MetricDataRepository json.RawMessage `json:"metricDataRepository"` -} - -var Clusters []*ClusterConfig - var Keys ProgramConfig = ProgramConfig{ Addr: "localhost:8080", DisableAuthentication: false, @@ -132,7 +124,7 @@ var Keys ProgramConfig = ProgramConfig{ ShortRunningJobsDuration: 5 * 60, } -func Init(mainConfig json.RawMessage, clusterConfig json.RawMessage) { +func Init(mainConfig json.RawMessage) { Validate(configSchema, mainConfig) dec := json.NewDecoder(bytes.NewReader(mainConfig)) dec.DisallowUnknownFields() @@ -140,17 +132,6 @@ func Init(mainConfig json.RawMessage, clusterConfig json.RawMessage) { cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error()) } - Validate(clustersSchema, clusterConfig) - dec = json.NewDecoder(bytes.NewReader(clusterConfig)) - dec.DisallowUnknownFields() - if err := dec.Decode(&Clusters); err != nil { - cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error()) - } - - if len(Clusters) < 1 { - cclog.Abort("Config Init: At least one cluster required in config. Exited with error.") - } - if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 { resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 396a80a1..e4a700ff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -16,11 +16,7 @@ func TestInit(t *testing.T) { fp := "../../configs/config.json" ccconf.Init(fp) if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + Init(cfg) } else { cclog.Abort("Main configuration must be present") } @@ -34,11 +30,7 @@ func TestInitMinimal(t *testing.T) { fp := "../../configs/config-demo.json" ccconf.Init(fp) if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + Init(cfg) } else { cclog.Abort("Main configuration must be present") } diff --git a/internal/config/schema.go b/internal/config/schema.go index ff8d0c92..2d068140 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -138,83 +138,3 @@ var configSchema = ` }, "required": ["apiAllowedIPs"] }` - -var clustersSchema = ` - { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "description": "The name of the cluster.", - "type": "string" - }, - "metricDataRepository": { - "description": "Type of the metric data repository for this cluster", - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": ["influxdb", "prometheus", "cc-metric-store", "cc-metric-store-internal", "test"] - }, - "url": { - "type": "string" - }, - "token": { - "type": "string" - } - }, - "required": ["kind"] - }, - "filterRanges": { - "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.", - "type": "object", - "properties": { - "numNodes": { - "description": "UI slider range for number of nodes", - "type": "object", - "properties": { - "from": { - "type": "integer" - }, - "to": { - "type": "integer" - } - }, - "required": ["from", "to"] - }, - "duration": { - "description": "UI slider range for duration", - "type": "object", - "properties": { - "from": { - "type": "integer" - }, - "to": { - "type": "integer" - } - }, - "required": ["from", "to"] - }, - "startTime": { - "description": "UI slider range for start time", - "type": "object", - "properties": { - "from": { - "type": "string", - "format": "date-time" - }, - "to": { - "type": "null" - } - }, - "required": ["from", "to"] - } - }, - "required": ["numNodes", "duration", "startTime"] - } - }, - "required": ["name", "metricDataRepository", "filterRanges"], - "minItems": 1 - } - }` diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index bffb8bf6..2d00fc84 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -121,11 +121,7 @@ func setup(t *testing.T) *repository.JobRepository { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - t.Fatal("Cluster configuration must be present") - } + config.Init(cfg) } else { t.Fatal("Main configuration must be present") } diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index e1d6ca93..fd935b53 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -144,11 +144,7 @@ func nodeTestSetup(t *testing.T) { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + config.Init(cfg) } else { cclog.Abort("Main configuration must be present") } diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index 02c70d0f..ae3adaf2 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -58,11 +58,7 @@ func setupUserTest(t *testing.T) *UserCfgRepo { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - t.Fatal("Cluster configuration must be present") - } + config.Init(cfg) } else { t.Fatal("Main configuration must be present") } diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index f5f8b836..ffcba793 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -434,11 +434,7 @@ func main() { // Load and check main configuration if cfg := ccconf.GetPackageConfig("main"); cfg != nil { - if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil { - config.Init(cfg, clustercfg) - } else { - cclog.Abort("Cluster configuration must be present") - } + config.Init(cfg) } else { cclog.Abort("Main configuration must be present") } diff --git a/web/web.go b/web/web.go index d2ae8700..37f1c2b2 100644 --- a/web/web.go +++ b/web/web.go @@ -245,7 +245,6 @@ type Page struct { User schema.User // Information about the currently logged in user (Full User Info) Roles map[string]schema.Role // Available roles for frontend render checks Build Build // Latest information about the application - Clusters []config.ClusterConfig // List of all clusters for use in the Header SubClusters map[string][]string // Map per cluster of all subClusters for use in the Header FilterPresets map[string]any // For pages with the Filter component, this can be used to set initial filters. Infos map[string]any // For generic use (e.g. username for /monitoring/user/, job id for /monitoring/job/) @@ -260,12 +259,6 @@ func RenderTemplate(rw http.ResponseWriter, file string, page *Page) { cclog.Errorf("WEB/WEB > template '%s' not found", file) } - if page.Clusters == nil { - for _, c := range config.Clusters { - page.Clusters = append(page.Clusters, config.ClusterConfig{Name: c.Name, FilterRanges: c.FilterRanges, MetricDataRepository: nil}) - } - } - if page.SubClusters == nil { page.SubClusters = make(map[string][]string) for _, cluster := range archive.Clusters { From 42809e3f75256d282e3f7a5b8bdfd9980c222882 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 07:20:26 +0100 Subject: [PATCH 069/341] Remove embedded tagger rules --- configs/tagger/README.md | 0 {internal => configs}/tagger/apps/alf.txt | 0 {internal => configs}/tagger/apps/caracal.txt | 0 {internal => configs}/tagger/apps/chroma.txt | 0 {internal => configs}/tagger/apps/cp2k.txt | 0 {internal => configs}/tagger/apps/cpmd.txt | 0 {internal => configs}/tagger/apps/flame.txt | 0 {internal => configs}/tagger/apps/gromacs.txt | 0 {internal => configs}/tagger/apps/julia.txt | 0 {internal => configs}/tagger/apps/lammps.txt | 0 {internal => configs}/tagger/apps/matlab.txt | 0 .../tagger/apps/openfoam.txt | 0 {internal => configs}/tagger/apps/orca.txt | 0 {internal => configs}/tagger/apps/python.txt | 0 {internal => configs}/tagger/apps/starccm.txt | 0 .../tagger/apps/turbomole.txt | 0 {internal => configs}/tagger/apps/vasp.txt | 0 .../tagger/jobclasses/highload.json | 0 .../tagger/jobclasses/lowUtilization.json | 0 .../tagger/jobclasses/lowload.json | 0 .../tagger/jobclasses/parameters.json | 0 internal/tagger/classifyJob.go | 105 ++++++++++-------- internal/tagger/classifyJob_test.go | 8 +- internal/tagger/detectApp.go | 61 +++++----- internal/tagger/detectApp_test.go | 70 +++++++++++- 25 files changed, 166 insertions(+), 78 deletions(-) create mode 100644 configs/tagger/README.md rename {internal => configs}/tagger/apps/alf.txt (100%) rename {internal => configs}/tagger/apps/caracal.txt (100%) rename {internal => configs}/tagger/apps/chroma.txt (100%) rename {internal => configs}/tagger/apps/cp2k.txt (100%) rename {internal => configs}/tagger/apps/cpmd.txt (100%) rename {internal => configs}/tagger/apps/flame.txt (100%) rename {internal => configs}/tagger/apps/gromacs.txt (100%) rename {internal => configs}/tagger/apps/julia.txt (100%) rename {internal => configs}/tagger/apps/lammps.txt (100%) rename {internal => configs}/tagger/apps/matlab.txt (100%) rename {internal => configs}/tagger/apps/openfoam.txt (100%) rename {internal => configs}/tagger/apps/orca.txt (100%) rename {internal => configs}/tagger/apps/python.txt (100%) rename {internal => configs}/tagger/apps/starccm.txt (100%) rename {internal => configs}/tagger/apps/turbomole.txt (100%) rename {internal => configs}/tagger/apps/vasp.txt (100%) rename {internal => configs}/tagger/jobclasses/highload.json (100%) rename {internal => configs}/tagger/jobclasses/lowUtilization.json (100%) rename {internal => configs}/tagger/jobclasses/lowload.json (100%) rename {internal => configs}/tagger/jobclasses/parameters.json (100%) diff --git a/configs/tagger/README.md b/configs/tagger/README.md new file mode 100644 index 00000000..e69de29b diff --git a/internal/tagger/apps/alf.txt b/configs/tagger/apps/alf.txt similarity index 100% rename from internal/tagger/apps/alf.txt rename to configs/tagger/apps/alf.txt diff --git a/internal/tagger/apps/caracal.txt b/configs/tagger/apps/caracal.txt similarity index 100% rename from internal/tagger/apps/caracal.txt rename to configs/tagger/apps/caracal.txt diff --git a/internal/tagger/apps/chroma.txt b/configs/tagger/apps/chroma.txt similarity index 100% rename from internal/tagger/apps/chroma.txt rename to configs/tagger/apps/chroma.txt diff --git a/internal/tagger/apps/cp2k.txt b/configs/tagger/apps/cp2k.txt similarity index 100% rename from internal/tagger/apps/cp2k.txt rename to configs/tagger/apps/cp2k.txt diff --git a/internal/tagger/apps/cpmd.txt b/configs/tagger/apps/cpmd.txt similarity index 100% rename from internal/tagger/apps/cpmd.txt rename to configs/tagger/apps/cpmd.txt diff --git a/internal/tagger/apps/flame.txt b/configs/tagger/apps/flame.txt similarity index 100% rename from internal/tagger/apps/flame.txt rename to configs/tagger/apps/flame.txt diff --git a/internal/tagger/apps/gromacs.txt b/configs/tagger/apps/gromacs.txt similarity index 100% rename from internal/tagger/apps/gromacs.txt rename to configs/tagger/apps/gromacs.txt diff --git a/internal/tagger/apps/julia.txt b/configs/tagger/apps/julia.txt similarity index 100% rename from internal/tagger/apps/julia.txt rename to configs/tagger/apps/julia.txt diff --git a/internal/tagger/apps/lammps.txt b/configs/tagger/apps/lammps.txt similarity index 100% rename from internal/tagger/apps/lammps.txt rename to configs/tagger/apps/lammps.txt diff --git a/internal/tagger/apps/matlab.txt b/configs/tagger/apps/matlab.txt similarity index 100% rename from internal/tagger/apps/matlab.txt rename to configs/tagger/apps/matlab.txt diff --git a/internal/tagger/apps/openfoam.txt b/configs/tagger/apps/openfoam.txt similarity index 100% rename from internal/tagger/apps/openfoam.txt rename to configs/tagger/apps/openfoam.txt diff --git a/internal/tagger/apps/orca.txt b/configs/tagger/apps/orca.txt similarity index 100% rename from internal/tagger/apps/orca.txt rename to configs/tagger/apps/orca.txt diff --git a/internal/tagger/apps/python.txt b/configs/tagger/apps/python.txt similarity index 100% rename from internal/tagger/apps/python.txt rename to configs/tagger/apps/python.txt diff --git a/internal/tagger/apps/starccm.txt b/configs/tagger/apps/starccm.txt similarity index 100% rename from internal/tagger/apps/starccm.txt rename to configs/tagger/apps/starccm.txt diff --git a/internal/tagger/apps/turbomole.txt b/configs/tagger/apps/turbomole.txt similarity index 100% rename from internal/tagger/apps/turbomole.txt rename to configs/tagger/apps/turbomole.txt diff --git a/internal/tagger/apps/vasp.txt b/configs/tagger/apps/vasp.txt similarity index 100% rename from internal/tagger/apps/vasp.txt rename to configs/tagger/apps/vasp.txt diff --git a/internal/tagger/jobclasses/highload.json b/configs/tagger/jobclasses/highload.json similarity index 100% rename from internal/tagger/jobclasses/highload.json rename to configs/tagger/jobclasses/highload.json diff --git a/internal/tagger/jobclasses/lowUtilization.json b/configs/tagger/jobclasses/lowUtilization.json similarity index 100% rename from internal/tagger/jobclasses/lowUtilization.json rename to configs/tagger/jobclasses/lowUtilization.json diff --git a/internal/tagger/jobclasses/lowload.json b/configs/tagger/jobclasses/lowload.json similarity index 100% rename from internal/tagger/jobclasses/lowload.json rename to configs/tagger/jobclasses/lowload.json diff --git a/internal/tagger/jobclasses/parameters.json b/configs/tagger/jobclasses/parameters.json similarity index 100% rename from internal/tagger/jobclasses/parameters.json rename to configs/tagger/jobclasses/parameters.json diff --git a/internal/tagger/classifyJob.go b/internal/tagger/classifyJob.go index 70399218..b5f30949 100644 --- a/internal/tagger/classifyJob.go +++ b/internal/tagger/classifyJob.go @@ -2,15 +2,16 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package tagger import ( "bytes" - "embed" "encoding/json" "fmt" "maps" "os" + "path/filepath" "strings" "text/template" @@ -23,8 +24,16 @@ import ( "github.com/expr-lang/expr/vm" ) -//go:embed jobclasses/* -var jobClassFiles embed.FS +const ( + // defaultJobClassConfigPath is the default path for job classification configuration + defaultJobClassConfigPath = "./var/tagger/jobclasses" + // tagTypeJobClass is the tag type identifier for job classification tags + tagTypeJobClass = "jobClass" + // jobClassConfigDirMatch is the directory name used for matching filesystem events + jobClassConfigDirMatch = "jobclasses" + // parametersFileName is the name of the parameters configuration file + parametersFileName = "parameters.json" +) // Variable defines a named expression that can be computed and reused in rules. // Variables are evaluated before the main rule and their results are added to the environment. @@ -45,21 +54,21 @@ type ruleVariable struct { // and the final rule expression that determines if the job matches the classification. type RuleFormat struct { // Name is a human-readable description of the rule - Name string `json:"name"` + Name string `json:"name"` // Tag is the classification tag to apply if the rule matches - Tag string `json:"tag"` + Tag string `json:"tag"` // Parameters are shared values referenced in the rule (e.g., thresholds) - Parameters []string `json:"parameters"` + Parameters []string `json:"parameters"` // Metrics are the job metrics required for this rule (e.g., "cpu_load", "mem_used") - Metrics []string `json:"metrics"` + Metrics []string `json:"metrics"` // Requirements are boolean expressions that must be true for the rule to apply - Requirements []string `json:"requirements"` + Requirements []string `json:"requirements"` // Variables are computed values used in the rule expression - Variables []Variable `json:"variables"` + Variables []Variable `json:"variables"` // Rule is the boolean expression that determines if the job matches - Rule string `json:"rule"` + Rule string `json:"rule"` // Hint is a template string that generates a message when the rule matches - Hint string `json:"hint"` + Hint string `json:"hint"` } type ruleInfo struct { @@ -75,29 +84,29 @@ type ruleInfo struct { // This interface allows for easier testing and decoupling from the concrete repository implementation. type JobRepository interface { // HasTag checks if a job already has a specific tag - HasTag(jobId int64, tagType string, tagName string) bool + HasTag(jobID int64, tagType string, tagName string) bool // AddTagOrCreateDirect adds a tag to a job or creates it if it doesn't exist - AddTagOrCreateDirect(jobId int64, tagType string, tagName string) (tagId int64, err error) + AddTagOrCreateDirect(jobID int64, tagType string, tagName string) (tagID int64, err error) // UpdateMetadata updates job metadata with a key-value pair UpdateMetadata(job *schema.Job, key, val string) (err error) } // JobClassTagger classifies jobs based on configurable rules that evaluate job metrics and properties. -// Rules are loaded from embedded JSON files and can be dynamically reloaded from a watched directory. +// Rules are loaded from an external configuration directory and can be dynamically reloaded when files change. // When a job matches a rule, it is tagged with the corresponding classification and an optional hint message. type JobClassTagger struct { // rules maps classification tags to their compiled rule information - rules map[string]ruleInfo + rules map[string]ruleInfo // parameters are shared values (e.g., thresholds) used across multiple rules - parameters map[string]any + parameters map[string]any // tagType is the type of tag ("jobClass") - tagType string + tagType string // cfgPath is the path to watch for configuration changes - cfgPath string + cfgPath string // repo provides access to job database operations - repo JobRepository + repo JobRepository // getStatistics retrieves job statistics for analysis - getStatistics func(job *schema.Job) (map[string]schema.JobStatistics, error) + getStatistics func(job *schema.Job) (map[string]schema.JobStatistics, error) // getMetricConfig retrieves metric configuration (limits) for a cluster getMetricConfig func(cluster, subCluster string) map[string]*schema.Metric } @@ -169,7 +178,7 @@ func (t *JobClassTagger) prepareRule(b []byte, fns string) { // EventMatch checks if a filesystem event should trigger configuration reload. // It returns true if the event path contains "jobclasses". func (t *JobClassTagger) EventMatch(s string) bool { - return strings.Contains(s, "jobclasses") + return strings.Contains(s, jobClassConfigDirMatch) } // EventCallback is triggered when the configuration directory changes. @@ -181,9 +190,10 @@ func (t *JobClassTagger) EventCallback() { cclog.Fatal(err) } - if util.CheckFileExists(t.cfgPath + "/parameters.json") { + parametersFile := filepath.Join(t.cfgPath, parametersFileName) + if util.CheckFileExists(parametersFile) { cclog.Info("Merge parameters") - b, err := os.ReadFile(t.cfgPath + "/parameters.json") + b, err := os.ReadFile(parametersFile) if err != nil { cclog.Warnf("prepareRule() > open file error: %v", err) } @@ -198,13 +208,13 @@ func (t *JobClassTagger) EventCallback() { for _, fn := range files { fns := fn.Name() - if fns != "parameters.json" { + if fns != parametersFileName { cclog.Debugf("Process: %s", fns) - filename := fmt.Sprintf("%s/%s", t.cfgPath, fns) + filename := filepath.Join(t.cfgPath, fns) b, err := os.ReadFile(filename) if err != nil { cclog.Warnf("prepareRule() > open file error: %v", err) - return + continue } t.prepareRule(b, fns) } @@ -213,7 +223,8 @@ func (t *JobClassTagger) EventCallback() { func (t *JobClassTagger) initParameters() error { cclog.Info("Initialize parameters") - b, err := jobClassFiles.ReadFile("jobclasses/parameters.json") + parametersFile := filepath.Join(t.cfgPath, parametersFileName) + b, err := os.ReadFile(parametersFile) if err != nil { cclog.Warnf("prepareRule() > open file error: %v", err) return err @@ -227,13 +238,20 @@ func (t *JobClassTagger) initParameters() error { return nil } -// Register initializes the JobClassTagger by loading parameters and classification rules. -// It loads embedded configuration files and sets up a file watch on ./var/tagger/jobclasses -// if it exists, allowing for dynamic configuration updates without restarting the application. -// Returns an error if the embedded configuration files cannot be read or parsed. +// Register initializes the JobClassTagger by loading parameters and classification rules from external folder. +// It sets up a file watch on ./var/tagger/jobclasses if it exists, allowing for +// dynamic configuration updates without restarting the application. +// Returns an error if the configuration path does not exist or cannot be read. func (t *JobClassTagger) Register() error { - t.cfgPath = "./var/tagger/jobclasses" - t.tagType = "jobClass" + if t.cfgPath == "" { + t.cfgPath = defaultJobClassConfigPath + } + t.tagType = tagTypeJobClass + t.rules = make(map[string]ruleInfo) + + if !util.CheckFileExists(t.cfgPath) { + return fmt.Errorf("configuration path does not exist: %s", t.cfgPath) + } err := t.initParameters() if err != nil { @@ -241,31 +259,28 @@ func (t *JobClassTagger) Register() error { return err } - files, err := jobClassFiles.ReadDir("jobclasses") + files, err := os.ReadDir(t.cfgPath) if err != nil { - return fmt.Errorf("error reading app folder: %#v", err) + return fmt.Errorf("error reading jobclasses folder: %#v", err) } - t.rules = make(map[string]ruleInfo) + for _, fn := range files { fns := fn.Name() - if fns != "parameters.json" { - filename := fmt.Sprintf("jobclasses/%s", fns) + if fns != parametersFileName { cclog.Infof("Process: %s", fns) + filename := filepath.Join(t.cfgPath, fns) - b, err := jobClassFiles.ReadFile(filename) + b, err := os.ReadFile(filename) if err != nil { cclog.Warnf("prepareRule() > open file error: %v", err) - return err + continue } t.prepareRule(b, fns) } } - if util.CheckFileExists(t.cfgPath) { - t.EventCallback() - cclog.Infof("Setup file watch for %s", t.cfgPath) - util.AddListener(t.cfgPath, t) - } + cclog.Infof("Setup file watch for %s", t.cfgPath) + util.AddListener(t.cfgPath, t) t.repo = repository.GetJobRepository() t.getStatistics = archive.GetStatistics diff --git a/internal/tagger/classifyJob_test.go b/internal/tagger/classifyJob_test.go index bed7a8f0..f82cf807 100644 --- a/internal/tagger/classifyJob_test.go +++ b/internal/tagger/classifyJob_test.go @@ -13,13 +13,13 @@ type MockJobRepository struct { mock.Mock } -func (m *MockJobRepository) HasTag(jobId int64, tagType string, tagName string) bool { - args := m.Called(jobId, tagType, tagName) +func (m *MockJobRepository) HasTag(jobID int64, tagType string, tagName string) bool { + args := m.Called(jobID, tagType, tagName) return args.Bool(0) } -func (m *MockJobRepository) AddTagOrCreateDirect(jobId int64, tagType string, tagName string) (tagId int64, err error) { - args := m.Called(jobId, tagType, tagName) +func (m *MockJobRepository) AddTagOrCreateDirect(jobID int64, tagType string, tagName string) (tagID int64, err error) { + args := m.Called(jobID, tagType, tagName) return args.Get(0).(int64), args.Error(1) } diff --git a/internal/tagger/detectApp.go b/internal/tagger/detectApp.go index 0b8e3e7e..2a89ea21 100644 --- a/internal/tagger/detectApp.go +++ b/internal/tagger/detectApp.go @@ -7,9 +7,7 @@ package tagger import ( "bufio" - "embed" "fmt" - "io/fs" "os" "path/filepath" "regexp" @@ -21,8 +19,14 @@ import ( "github.com/ClusterCockpit/cc-lib/v2/util" ) -//go:embed apps/* -var appFiles embed.FS +const ( + // defaultConfigPath is the default path for application tagging configuration + defaultConfigPath = "./var/tagger/apps" + // tagTypeApp is the tag type identifier for application tags + tagTypeApp = "app" + // configDirMatch is the directory name used for matching filesystem events + configDirMatch = "apps" +) type appInfo struct { tag string @@ -30,19 +34,19 @@ type appInfo struct { } // AppTagger detects applications by matching patterns in job scripts. -// It loads application patterns from embedded files and can dynamically reload -// configuration from a watched directory. When a job script matches a pattern, +// It loads application patterns from an external configuration directory and can dynamically reload +// configuration when files change. When a job script matches a pattern, // the corresponding application tag is automatically applied. type AppTagger struct { // apps maps application tags to their matching patterns - apps map[string]appInfo + apps map[string]appInfo // tagType is the type of tag ("app") tagType string // cfgPath is the path to watch for configuration changes cfgPath string } -func (t *AppTagger) scanApp(f fs.File, fns string) { +func (t *AppTagger) scanApp(f *os.File, fns string) { scanner := bufio.NewScanner(f) ai := appInfo{tag: strings.TrimSuffix(fns, filepath.Ext(fns)), strings: make([]string, 0)} @@ -56,7 +60,7 @@ func (t *AppTagger) scanApp(f fs.File, fns string) { // EventMatch checks if a filesystem event should trigger configuration reload. // It returns true if the event path contains "apps". func (t *AppTagger) EventMatch(s string) bool { - return strings.Contains(s, "apps") + return strings.Contains(s, configDirMatch) } // EventCallback is triggered when the configuration directory changes. @@ -71,43 +75,50 @@ func (t *AppTagger) EventCallback() { for _, fn := range files { fns := fn.Name() cclog.Debugf("Process: %s", fns) - f, err := os.Open(fmt.Sprintf("%s/%s", t.cfgPath, fns)) + f, err := os.Open(filepath.Join(t.cfgPath, fns)) if err != nil { cclog.Errorf("error opening app file %s: %#v", fns, err) + continue } t.scanApp(f, fns) + f.Close() } } -// Register initializes the AppTagger by loading application patterns from embedded files. -// It also sets up a file watch on ./var/tagger/apps if it exists, allowing for +// Register initializes the AppTagger by loading application patterns from external folder. +// It sets up a file watch on ./var/tagger/apps if it exists, allowing for // dynamic configuration updates without restarting the application. -// Returns an error if the embedded application files cannot be read. +// Returns an error if the configuration path does not exist or cannot be read. func (t *AppTagger) Register() error { - t.cfgPath = "./var/tagger/apps" - t.tagType = "app" + if t.cfgPath == "" { + t.cfgPath = defaultConfigPath + } + t.tagType = tagTypeApp + t.apps = make(map[string]appInfo, 0) - files, err := appFiles.ReadDir("apps") + if !util.CheckFileExists(t.cfgPath) { + return fmt.Errorf("configuration path does not exist: %s", t.cfgPath) + } + + files, err := os.ReadDir(t.cfgPath) if err != nil { return fmt.Errorf("error reading app folder: %#v", err) } - t.apps = make(map[string]appInfo, 0) + for _, fn := range files { fns := fn.Name() cclog.Debugf("Process: %s", fns) - f, err := appFiles.Open(fmt.Sprintf("apps/%s", fns)) + f, err := os.Open(filepath.Join(t.cfgPath, fns)) if err != nil { - return fmt.Errorf("error opening app file %s: %#v", fns, err) + cclog.Errorf("error opening app file %s: %#v", fns, err) + continue } - defer f.Close() t.scanApp(f, fns) + f.Close() } - if util.CheckFileExists(t.cfgPath) { - t.EventCallback() - cclog.Infof("Setup file watch for %s", t.cfgPath) - util.AddListener(t.cfgPath, t) - } + cclog.Infof("Setup file watch for %s", t.cfgPath) + util.AddListener(t.cfgPath, t) return nil } diff --git a/internal/tagger/detectApp_test.go b/internal/tagger/detectApp_test.go index 1c44f670..fe5e7a21 100644 --- a/internal/tagger/detectApp_test.go +++ b/internal/tagger/detectApp_test.go @@ -5,6 +5,8 @@ package tagger import ( + "os" + "path/filepath" "testing" "github.com/ClusterCockpit/cc-backend/internal/repository" @@ -29,28 +31,88 @@ func noErr(tb testing.TB, err error) { } } -func TestRegister(t *testing.T) { - var tagger AppTagger +func setupAppTaggerTestDir(t *testing.T) string { + t.Helper() - err := tagger.Register() + testDir := t.TempDir() + appsDir := filepath.Join(testDir, "apps") + err := os.MkdirAll(appsDir, 0o755) noErr(t, err) + srcDir := "../../configs/tagger/apps" + files, err := os.ReadDir(srcDir) + noErr(t, err) + + for _, file := range files { + if file.IsDir() { + continue + } + srcPath := filepath.Join(srcDir, file.Name()) + dstPath := filepath.Join(appsDir, file.Name()) + + data, err := os.ReadFile(srcPath) + noErr(t, err) + + err = os.WriteFile(dstPath, data, 0o644) + noErr(t, err) + } + + return appsDir +} + +func TestRegister(t *testing.T) { + appsDir := setupAppTaggerTestDir(t) + + var tagger AppTagger + tagger.cfgPath = appsDir + tagger.tagType = tagTypeApp + tagger.apps = make(map[string]appInfo, 0) + + files, err := os.ReadDir(appsDir) + noErr(t, err) + + for _, fn := range files { + if fn.IsDir() { + continue + } + fns := fn.Name() + f, err := os.Open(filepath.Join(appsDir, fns)) + noErr(t, err) + tagger.scanApp(f, fns) + f.Close() + } + if len(tagger.apps) != 16 { t.Errorf("wrong summary for diagnostic \ngot: %d \nwant: 16", len(tagger.apps)) } } func TestMatch(t *testing.T) { + appsDir := setupAppTaggerTestDir(t) r := setup(t) job, err := r.FindByIDDirect(317) noErr(t, err) var tagger AppTagger + tagger.cfgPath = appsDir + tagger.tagType = tagTypeApp + tagger.apps = make(map[string]appInfo, 0) - err = tagger.Register() + files, err := os.ReadDir(appsDir) noErr(t, err) + for _, fn := range files { + if fn.IsDir() { + continue + } + fns := fn.Name() + f, err := os.Open(filepath.Join(appsDir, fns)) + noErr(t, err) + tagger.scanApp(f, fns) + f.Close() + } + tagger.Match(job) if !r.HasTag(317, "app", "vasp") { From a9366d14c66aae29e3da4928558bd74b39662990 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 08:32:32 +0100 Subject: [PATCH 070/341] Add README for tagging. Enable tagging by flag without configuration option --- cmd/cc-backend/main.go | 2 + configs/tagger/README.md | 419 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 8eb3c76f..9f98ccbf 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -302,6 +302,8 @@ func initSubsystems() error { // Apply tags if requested if flagApplyTags { + tagger.Init() + if err := tagger.RunTaggers(); err != nil { return fmt.Errorf("running job taggers: %w", err) } diff --git a/configs/tagger/README.md b/configs/tagger/README.md index e69de29b..759cbe97 100644 --- a/configs/tagger/README.md +++ b/configs/tagger/README.md @@ -0,0 +1,419 @@ +# Job Tagging Configuration + +ClusterCockpit provides automatic job tagging functionality to classify and +categorize jobs based on configurable rules. The tagging system consists of two +main components: + +1. **Application Detection** - Identifies which application a job is running +2. **Job Classification** - Analyzes job performance characteristics and applies classification tags + +## Directory Structure + +``` +configs/tagger/ +├── apps/ # Application detection patterns +│ ├── vasp.txt +│ ├── gromacs.txt +│ └── ... +└── jobclasses/ # Job classification rules + ├── parameters.json + ├── lowUtilization.json + ├── highload.json + └── ... +``` + +## Activating Tagger Rules + +### Step 1: Copy Configuration Files + +To activate tagging, review, adapt, and copy the configuration files from +`configs/tagger/` to `var/tagger/`: + +```bash +# From the cc-backend root directory +mkdir -p var/tagger +cp -r configs/tagger/apps var/tagger/ +cp -r configs/tagger/jobclasses var/tagger/ +``` + +### Step 2: Enable Tagging in Configuration + +Add or set the following configuration key in the `main` section of your `config.json`: + +```json +{ + "enable-job-taggers": true +} +``` + +**Important**: Automatic tagging is disabled by default. You must explicitly +enable it by setting `enable-job-taggers: true` in the main configuration file. + +### Step 3: Restart cc-backend + +The tagger system automatically loads configuration from `./var/tagger/` at +startup. After copying the files and enabling the feature, restart cc-backend: + +```bash +./cc-backend -server +``` + +### Step 4: Verify Configuration Loaded + +Check the logs for messages indicating successful configuration loading: + +``` +[INFO] Setup file watch for ./var/tagger/apps +[INFO] Setup file watch for ./var/tagger/jobclasses +``` + +## How Tagging Works + +### Automatic Tagging + +When `enable-job-taggers` is set to `true` in the configuration, tags are +automatically applied when: + +- **Job Start**: Application detection runs immediately when a job starts +- **Job Stop**: Job classification runs when a job completes + +The system analyzes job metadata and metrics to determine appropriate tags. + +**Note**: Automatic tagging only works for jobs that start or stop after the +feature is enabled. Existing jobs are not automatically retagged. + +### Manual Tagging (Retroactive) + +To apply tags to existing jobs in the database, use the `-apply-tags` command +line option: + +```bash +./cc-backend -apply-tags +``` + +This processes all jobs in the database and applies current tagging rules. This +is useful when: + +- You have existing jobs that were created before tagging was enabled +- You've added new tagging rules and want to apply them to historical data +- You've modified existing rules and want to re-evaluate all jobs + +### Hot Reload + +The tagger system watches the configuration directories for changes. You can +modify or add rules without restarting `cc-backend`: + +- Changes to `var/tagger/apps/*` are detected automatically +- Changes to `var/tagger/jobclasses/*` are detected automatically + +## Application Detection + +Application detection identifies which software a job is running by matching +patterns in the job script. + +### Configuration Format + +Application patterns are stored in text files under `var/tagger/apps/`. Each +file contains one or more regular expression patterns (one per line) that match +against the job script. + +**Example: `apps/vasp.txt`** + +``` +vasp +VASP +``` + +### How It Works + +1. When a job starts, the system retrieves the job script from metadata +2. Each line in the app files is treated as a regex pattern +3. Patterns are matched case-insensitively against the lowercased job script +4. If a match is found, a tag of type `app` with the filename (without extension) is applied +5. Only the first matching application is tagged + +### Adding New Applications + +1. Create a new file in `var/tagger/apps/` (e.g., `tensorflow.txt`) +2. Add regex patterns, one per line: + + ``` + tensorflow + tf\.keras + import tensorflow + ``` + +3. The file is automatically detected and loaded + +**Note**: The tag name will be the filename without the `.txt` extension (e.g., `tensorflow`). + +## Job Classification + +Job classification analyzes completed jobs based on their metrics and properties +to identify performance issues or characteristics. + +### Configuration Format + +Job classification rules are defined in JSON files under +`var/tagger/jobclasses/`. Each rule file defines: + +- **Metrics required**: Which job metrics to analyze +- **Requirements**: Pre-conditions that must be met +- **Variables**: Computed values used in the rule +- **Rule expression**: Boolean expression that determines if the rule matches +- **Hint template**: Message displayed when the rule matches + +### Parameters File + +`jobclasses/parameters.json` defines shared threshold values used across multiple rules: + +```json +{ + "lowcpuload_threshold_factor": 0.9, + "highmemoryusage_threshold_factor": 0.9, + "job_min_duration_seconds": 600.0, + "sampling_interval_seconds": 30.0 +} +``` + +### Rule File Structure + +**Example: `jobclasses/lowUtilization.json`** + +```json +{ + "name": "Low resource utilization", + "tag": "lowutilization", + "parameters": ["job_min_duration_seconds"], + "metrics": ["flops_any", "mem_bw"], + "requirements": [ + "job.shared == \"none\"", + "job.duration > job_min_duration_seconds" + ], + "variables": [ + { + "name": "mem_bw_perc", + "expr": "1.0 - (mem_bw.avg / mem_bw.limits.peak)" + } + ], + "rule": "flops_any.avg < flops_any.limits.alert", + "hint": "Average flop rate {{.flops_any.avg}} falls below threshold {{.flops_any.limits.alert}}" +} +``` + +#### Field Descriptions + +| Field | Description | +| -------------- | ----------------------------------------------------------------------------- | +| `name` | Human-readable description of the rule | +| `tag` | Tag identifier applied when the rule matches | +| `parameters` | List of parameter names from `parameters.json` to include in rule environment | +| `metrics` | List of metrics required for evaluation (must be present in job data) | +| `requirements` | Boolean expressions that must all be true for the rule to be evaluated | +| `variables` | Named expressions computed before evaluating the main rule | +| `rule` | Boolean expression that determines if the job matches this classification | +| `hint` | Go template string for generating a user-visible message | + +### Expression Environment + +Expressions in `requirements`, `variables`, and `rule` have access to: + +**Job Properties:** + +- `job.shared` - Shared node allocation type +- `job.duration` - Job runtime in seconds +- `job.numCores` - Number of CPU cores +- `job.numNodes` - Number of nodes +- `job.jobState` - Job completion state +- `job.numAcc` - Number of accelerators +- `job.smt` - SMT setting + +**Metric Statistics (for each metric in `metrics`):** + +- `.min` - Minimum value +- `.max` - Maximum value +- `.avg` - Average value +- `.limits.peak` - Peak limit from cluster config +- `.limits.normal` - Normal threshold +- `.limits.caution` - Caution threshold +- `.limits.alert` - Alert threshold + +**Parameters:** + +- All parameters listed in the `parameters` field + +**Variables:** + +- All variables defined in the `variables` array + +### Expression Language + +Rules use the [expr](https://github.com/expr-lang/expr) language for expressions. Supported operations: + +- **Arithmetic**: `+`, `-`, `*`, `/`, `%`, `^` +- **Comparison**: `==`, `!=`, `<`, `<=`, `>`, `>=` +- **Logical**: `&&`, `||`, `!` +- **Functions**: Standard math functions (see expr documentation) + +### Hint Templates + +Hints use Go's `text/template` syntax. Variables from the evaluation environment are accessible: + +``` +{{.flops_any.avg}} # Access metric average +{{.job.duration}} # Access job property +{{.my_variable}} # Access computed variable +``` + +### Adding New Classification Rules + +1. Create a new JSON file in `var/tagger/jobclasses/` (e.g., `memoryLeak.json`) +2. Define the rule structure: + + ```json + { + "name": "Memory Leak Detection", + "tag": "memory_leak", + "parameters": ["memory_leak_slope_threshold"], + "metrics": ["mem_used"], + "requirements": ["job.duration > 3600"], + "variables": [ + { + "name": "mem_growth", + "expr": "(mem_used.max - mem_used.min) / job.duration" + } + ], + "rule": "mem_growth > memory_leak_slope_threshold", + "hint": "Memory usage grew by {{.mem_growth}} per second" + } + ``` + +3. Add any new parameters to `parameters.json` +4. The file is automatically detected and loaded + +## Configuration Paths + +The tagger system reads from these paths (relative to cc-backend working directory): + +- **Application patterns**: `./var/tagger/apps/` +- **Job classification rules**: `./var/tagger/jobclasses/` + +These paths are defined as constants in the source code and cannot be changed without recompiling. + +## Troubleshooting + +### Tags Not Applied + +1. **Check tagging is enabled**: Verify `enable-job-taggers: true` is set in `config.json` + +2. **Check configuration exists**: + + ```bash + ls -la var/tagger/apps + ls -la var/tagger/jobclasses + ``` + +3. **Check logs for errors**: + + ```bash + ./cc-backend -server -loglevel debug + ``` + +4. **Verify file permissions**: Ensure cc-backend can read the configuration files + +5. **For existing jobs**: Use `./cc-backend -apply-tags` to retroactively tag jobs + +### Rules Not Matching + +1. **Enable debug logging**: Set `loglevel: debug` to see detailed rule evaluation +2. **Check requirements**: Ensure all requirements in the rule are satisfied +3. **Verify metrics exist**: Classification rules require job metrics to be available +4. **Check metric names**: Ensure metric names match those in your cluster configuration + +### File Watch Not Working + +If changes to configuration files aren't detected: + +1. Restart cc-backend to reload all configuration +2. Check filesystem supports file watching (network filesystems may not) +3. Check logs for file watch setup messages + +## Best Practices + +1. **Start Simple**: Begin with basic rules and refine based on results +2. **Use Requirements**: Filter out irrelevant jobs early with requirements +3. **Test Incrementally**: Add one rule at a time and verify behavior +4. **Document Rules**: Use descriptive names and clear hint messages +5. **Share Parameters**: Define common thresholds in `parameters.json` for consistency +6. **Version Control**: Keep your `var/tagger/` configuration in version control +7. **Backup Before Changes**: Test new rules on a copy before deploying to production + +## Examples + +### Simple Application Detection + +**File: `var/tagger/apps/python.txt`** + +``` +python +python3 +\.py +``` + +This detects jobs running Python scripts. + +### Complex Classification Rule + +**File: `var/tagger/jobclasses/cpuImbalance.json`** + +```json +{ + "name": "CPU Load Imbalance", + "tag": "cpu_imbalance", + "parameters": ["core_load_imbalance_threshold_factor"], + "metrics": ["cpu_load"], + "requirements": ["job.numCores > 1", "job.duration > 600"], + "variables": [ + { + "name": "load_variance", + "expr": "(cpu_load.max - cpu_load.min) / cpu_load.avg" + } + ], + "rule": "load_variance > core_load_imbalance_threshold_factor", + "hint": "CPU load varies by {{printf \"%.1f%%\" (load_variance * 100)}} across cores" +} +``` + +This detects jobs where CPU load is unevenly distributed across cores. + +## Reference + +### Configuration Options + +**Main Configuration (`config.json`)**: + +- `enable-job-taggers` (boolean, default: `false`) - Enables automatic job tagging system + - Must be set to `true` to activate automatic tagging on job start/stop events + - Does not affect the `-apply-tags` command line option + +**Command Line Options**: + +- `-apply-tags` - Apply all tagging rules to existing jobs in the database + - Works independently of `enable-job-taggers` configuration + - Useful for retroactively tagging jobs or re-evaluating with updated rules + +### Default Configuration Location + +The example configurations are provided in: + +- `configs/tagger/apps/` - Example application patterns (16 applications) +- `configs/tagger/jobclasses/` - Example classification rules (3 rules) + +Copy these to `var/tagger/` and customize for your environment. + +### Tag Types + +- `app` - Application tags (e.g., "vasp", "gromacs") +- `jobClass` - Classification tags (e.g., "lowutilization", "highload") + +Tags can be queried and filtered in the ClusterCockpit UI and API. From 2ebab1e2e2579cccebaec7615b3a1e07cf6bfe49 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 09:50:57 +0100 Subject: [PATCH 071/341] Reformat with gofumpt --- internal/auth/auth.go | 10 ++++---- internal/auth/auth_test.go | 40 +++++++++++++++---------------- internal/auth/jwt.go | 6 ++--- internal/auth/jwtCookieSession.go | 4 ++-- internal/auth/jwtHelpers.go | 24 +++++++++---------- internal/auth/jwtHelpers_test.go | 37 ++++++++++++++-------------- internal/auth/jwtSession.go | 4 ++-- internal/auth/oidc.go | 4 ++-- 8 files changed, 64 insertions(+), 65 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 3be1768e..41691d00 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -40,7 +40,7 @@ type Authenticator interface { // authenticator should attempt the login. This method should not perform // expensive operations or actual authentication. CanLogin(user *schema.User, username string, rw http.ResponseWriter, r *http.Request) (*schema.User, bool) - + // Login performs the actually authentication for the user. // It returns the authenticated user or an error if authentication fails. // The user parameter may be nil if the user doesn't exist in the database yet. @@ -65,13 +65,13 @@ var ipUserLimiters sync.Map func getIPUserLimiter(ip, username string) *rate.Limiter { key := ip + ":" + username now := time.Now() - + if entry, ok := ipUserLimiters.Load(key); ok { rle := entry.(*rateLimiterEntry) rle.lastUsed = now return rle.limiter } - + // More aggressive rate limiting: 5 attempts per 15 minutes newLimiter := rate.NewLimiter(rate.Every(15*time.Minute/5), 5) ipUserLimiters.Store(key, &rateLimiterEntry{ @@ -176,7 +176,7 @@ func (auth *Authentication) AuthViaSession( func Init(authCfg *json.RawMessage) { initOnce.Do(func() { authInstance = &Authentication{} - + // Start background cleanup of rate limiters startRateLimiterCleanup() @@ -272,7 +272,7 @@ func handleUserSync(user *schema.User, syncUserOnLogin, updateUserOnLogin bool) cclog.Errorf("Error while loading user '%s': %v", user.Username, err) return } - + if err == sql.ErrNoRows && syncUserOnLogin { // Add new user if err := r.AddUser(user); err != nil { cclog.Errorf("Error while adding user '%s' to DB: %v", user.Username, err) diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 15f153e6..68961354 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -15,25 +15,25 @@ import ( func TestGetIPUserLimiter(t *testing.T) { ip := "192.168.1.1" username := "testuser" - + // Get limiter for the first time limiter1 := getIPUserLimiter(ip, username) if limiter1 == nil { t.Fatal("Expected limiter to be created") } - + // Get the same limiter again limiter2 := getIPUserLimiter(ip, username) if limiter1 != limiter2 { t.Error("Expected to get the same limiter instance") } - + // Get a different limiter for different user limiter3 := getIPUserLimiter(ip, "otheruser") if limiter1 == limiter3 { t.Error("Expected different limiter for different user") } - + // Get a different limiter for different IP limiter4 := getIPUserLimiter("192.168.1.2", username) if limiter1 == limiter4 { @@ -45,16 +45,16 @@ func TestGetIPUserLimiter(t *testing.T) { func TestRateLimiterBehavior(t *testing.T) { ip := "10.0.0.1" username := "ratelimituser" - + limiter := getIPUserLimiter(ip, username) - + // Should allow first 5 attempts for i := 0; i < 5; i++ { if !limiter.Allow() { t.Errorf("Request %d should be allowed within rate limit", i+1) } } - + // 6th attempt should be blocked if limiter.Allow() { t.Error("Request 6 should be blocked by rate limiter") @@ -65,19 +65,19 @@ func TestRateLimiterBehavior(t *testing.T) { func TestCleanupOldRateLimiters(t *testing.T) { // Clear all existing limiters first to avoid interference from other tests cleanupOldRateLimiters(time.Now().Add(24 * time.Hour)) - + // Create some new rate limiters limiter1 := getIPUserLimiter("1.1.1.1", "user1") limiter2 := getIPUserLimiter("2.2.2.2", "user2") - + if limiter1 == nil || limiter2 == nil { t.Fatal("Failed to create test limiters") } - + // Cleanup limiters older than 1 second from now (should keep both) time.Sleep(10 * time.Millisecond) // Small delay to ensure timestamp difference cleanupOldRateLimiters(time.Now().Add(-1 * time.Second)) - + // Verify they still exist (should get same instance) if getIPUserLimiter("1.1.1.1", "user1") != limiter1 { t.Error("Limiter 1 was incorrectly cleaned up") @@ -85,10 +85,10 @@ func TestCleanupOldRateLimiters(t *testing.T) { if getIPUserLimiter("2.2.2.2", "user2") != limiter2 { t.Error("Limiter 2 was incorrectly cleaned up") } - + // Cleanup limiters older than 1 hour from now (should remove both) cleanupOldRateLimiters(time.Now().Add(2 * time.Hour)) - + // Getting them again should create new instances newLimiter1 := getIPUserLimiter("1.1.1.1", "user1") if newLimiter1 == limiter1 { @@ -107,14 +107,14 @@ func TestIPv4Extraction(t *testing.T) { {"IPv4 without port", "192.168.1.1", "192.168.1.1"}, {"Localhost with port", "127.0.0.1:3000", "127.0.0.1"}, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := tt.input if host, _, err := net.SplitHostPort(result); err == nil { result = host } - + if result != tt.expected { t.Errorf("Expected %s, got %s", tt.expected, result) } @@ -122,7 +122,7 @@ func TestIPv4Extraction(t *testing.T) { } } -// TestIPv6Extraction tests extracting IPv6 addresses +// TestIPv6Extraction tests extracting IPv6 addresses func TestIPv6Extraction(t *testing.T) { tests := []struct { name string @@ -134,14 +134,14 @@ func TestIPv6Extraction(t *testing.T) { {"IPv6 without port", "2001:db8::1", "2001:db8::1"}, {"IPv6 localhost", "::1", "::1"}, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := tt.input if host, _, err := net.SplitHostPort(result); err == nil { result = host } - + if result != tt.expected { t.Errorf("Expected %s, got %s", tt.expected, result) } @@ -160,14 +160,14 @@ func TestIPExtractionEdgeCases(t *testing.T) { {"Empty string", "", ""}, {"Just port", ":8080", ""}, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := tt.input if host, _, err := net.SplitHostPort(result); err == nil { result = host } - + if result != tt.expected { t.Errorf("Expected %s, got %s", tt.expected, result) } diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index be642219..c0f641b9 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -101,20 +101,20 @@ func (ja *JWTAuthenticator) AuthViaJWT( // Token is valid, extract payload claims := token.Claims.(jwt.MapClaims) - + // Use shared helper to get user from JWT claims var user *schema.User user, err = getUserFromJWT(claims, Keys.JwtConfig.ValidateUser, schema.AuthToken, -1) if err != nil { return nil, err } - + // If not validating user, we only get roles from JWT (no projects for this auth method) if !Keys.JwtConfig.ValidateUser { user.Roles = extractRolesFromClaims(claims, false) user.Projects = nil // Standard JWT auth doesn't include projects } - + return user, nil } diff --git a/internal/auth/jwtCookieSession.go b/internal/auth/jwtCookieSession.go index 42f7439e..4c4bbeb6 100644 --- a/internal/auth/jwtCookieSession.go +++ b/internal/auth/jwtCookieSession.go @@ -146,13 +146,13 @@ func (ja *JWTCookieSessionAuthenticator) Login( } claims := token.Claims.(jwt.MapClaims) - + // Use shared helper to get user from JWT claims user, err = getUserFromJWT(claims, jc.ValidateUser, schema.AuthSession, schema.AuthViaToken) if err != nil { return nil, err } - + // Sync or update user if configured if !jc.ValidateUser && (jc.SyncUserOnLogin || jc.UpdateUserOnLogin) { handleTokenUser(user) diff --git a/internal/auth/jwtHelpers.go b/internal/auth/jwtHelpers.go index 5bfc91ef..de59145e 100644 --- a/internal/auth/jwtHelpers.go +++ b/internal/auth/jwtHelpers.go @@ -28,7 +28,7 @@ func extractStringFromClaims(claims jwt.MapClaims, key string) string { // If validateRoles is true, only valid roles are returned func extractRolesFromClaims(claims jwt.MapClaims, validateRoles bool) []string { var roles []string - + if rawroles, ok := claims["roles"].([]any); ok { for _, rr := range rawroles { if r, ok := rr.(string); ok { @@ -42,14 +42,14 @@ func extractRolesFromClaims(claims jwt.MapClaims, validateRoles bool) []string { } } } - + return roles } // extractProjectsFromClaims extracts projects from JWT claims func extractProjectsFromClaims(claims jwt.MapClaims) []string { projects := make([]string, 0) - + if rawprojs, ok := claims["projects"].([]any); ok { for _, pp := range rawprojs { if p, ok := pp.(string); ok { @@ -61,7 +61,7 @@ func extractProjectsFromClaims(claims jwt.MapClaims) []string { projects = append(projects, projSlice...) } } - + return projects } @@ -72,14 +72,14 @@ func extractNameFromClaims(claims jwt.MapClaims) string { if name, ok := claims["name"].(string); ok { return name } - + // Try nested structure: {name: {values: [...]}} if wrap, ok := claims["name"].(map[string]any); ok { if vals, ok := wrap["values"].([]any); ok { if len(vals) == 0 { return "" } - + name := fmt.Sprintf("%v", vals[0]) for i := 1; i < len(vals); i++ { name += fmt.Sprintf(" %v", vals[i]) @@ -87,7 +87,7 @@ func extractNameFromClaims(claims jwt.MapClaims) string { return name } } - + return "" } @@ -100,7 +100,7 @@ func getUserFromJWT(claims jwt.MapClaims, validateUser bool, authType schema.Aut if sub == "" { return nil, errors.New("missing 'sub' claim in JWT") } - + if validateUser { // Validate user against database ur := repository.GetUserRepository() @@ -109,22 +109,22 @@ func getUserFromJWT(claims jwt.MapClaims, validateUser bool, authType schema.Aut cclog.Errorf("Error while loading user '%v': %v", sub, err) return nil, fmt.Errorf("database error: %w", err) } - + // Deny any logins for unknown usernames if user == nil || err == sql.ErrNoRows { cclog.Warn("Could not find user from JWT in internal database.") return nil, errors.New("unknown user") } - + // Return database user (with database roles) return user, nil } - + // Create user from JWT claims name := extractNameFromClaims(claims) roles := extractRolesFromClaims(claims, true) // Validate roles projects := extractProjectsFromClaims(claims) - + return &schema.User{ Username: sub, Name: name, diff --git a/internal/auth/jwtHelpers_test.go b/internal/auth/jwtHelpers_test.go index 84a1f2e0..4627f7e5 100644 --- a/internal/auth/jwtHelpers_test.go +++ b/internal/auth/jwtHelpers_test.go @@ -19,7 +19,7 @@ func TestExtractStringFromClaims(t *testing.T) { "email": "test@example.com", "age": 25, // not a string } - + tests := []struct { name string key string @@ -30,7 +30,7 @@ func TestExtractStringFromClaims(t *testing.T) { {"Non-existent key", "missing", ""}, {"Non-string value", "age", ""}, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := extractStringFromClaims(claims, tt.key) @@ -88,16 +88,16 @@ func TestExtractRolesFromClaims(t *testing.T) { expected: []string{}, }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := extractRolesFromClaims(tt.claims, tt.validateRoles) - + if len(result) != len(tt.expected) { t.Errorf("Expected %d roles, got %d", len(tt.expected), len(result)) return } - + for i, role := range result { if i >= len(tt.expected) || role != tt.expected[i] { t.Errorf("Expected role %s at position %d, got %s", tt.expected[i], i, role) @@ -141,16 +141,16 @@ func TestExtractProjectsFromClaims(t *testing.T) { expected: []string{"project1", "project2"}, // Should skip non-strings }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := extractProjectsFromClaims(tt.claims) - + if len(result) != len(tt.expected) { t.Errorf("Expected %d projects, got %d", len(tt.expected), len(result)) return } - + for i, project := range result { if i >= len(tt.expected) || project != tt.expected[i] { t.Errorf("Expected project %s at position %d, got %s", tt.expected[i], i, project) @@ -216,7 +216,7 @@ func TestExtractNameFromClaims(t *testing.T) { expected: "123 Smith", // Should convert to string }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := extractNameFromClaims(tt.claims) @@ -235,29 +235,28 @@ func TestGetUserFromJWT_NoValidation(t *testing.T) { "roles": []any{"user", "admin"}, "projects": []any{"project1", "project2"}, } - + user, err := getUserFromJWT(claims, false, schema.AuthToken, -1) - if err != nil { t.Fatalf("Unexpected error: %v", err) } - + if user.Username != "testuser" { t.Errorf("Expected username 'testuser', got '%s'", user.Username) } - + if user.Name != "Test User" { t.Errorf("Expected name 'Test User', got '%s'", user.Name) } - + if len(user.Roles) != 2 { t.Errorf("Expected 2 roles, got %d", len(user.Roles)) } - + if len(user.Projects) != 2 { t.Errorf("Expected 2 projects, got %d", len(user.Projects)) } - + if user.AuthType != schema.AuthToken { t.Errorf("Expected AuthType %v, got %v", schema.AuthToken, user.AuthType) } @@ -268,13 +267,13 @@ func TestGetUserFromJWT_MissingSub(t *testing.T) { claims := jwt.MapClaims{ "name": "Test User", } - + _, err := getUserFromJWT(claims, false, schema.AuthToken, -1) - + if err == nil { t.Error("Expected error for missing sub claim") } - + if err.Error() != "missing 'sub' claim in JWT" { t.Errorf("Expected specific error message, got: %v", err) } diff --git a/internal/auth/jwtSession.go b/internal/auth/jwtSession.go index 107afcb8..de7e985b 100644 --- a/internal/auth/jwtSession.go +++ b/internal/auth/jwtSession.go @@ -75,13 +75,13 @@ func (ja *JWTSessionAuthenticator) Login( } claims := token.Claims.(jwt.MapClaims) - + // Use shared helper to get user from JWT claims user, err = getUserFromJWT(claims, Keys.JwtConfig.ValidateUser, schema.AuthSession, schema.AuthViaToken) if err != nil { return nil, err } - + // Sync or update user if configured if !Keys.JwtConfig.ValidateUser && (Keys.JwtConfig.SyncUserOnLogin || Keys.JwtConfig.UpdateUserOnLogin) { handleTokenUser(user) diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go index a3fc09cc..b90aca4f 100644 --- a/internal/auth/oidc.go +++ b/internal/auth/oidc.go @@ -59,7 +59,7 @@ func NewOIDC(a *Authentication) *OIDC { // Use context with timeout for provider initialization ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - + provider, err := oidc.NewProvider(ctx, Keys.OpenIDConfig.Provider) if err != nil { cclog.Fatal(err) @@ -119,7 +119,7 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) { // Exchange authorization code for token with timeout ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - + token, err := oa.client.Exchange(ctx, code, oauth2.VerifierOption(codeVerifier)) if err != nil { http.Error(rw, "Failed to exchange token: "+err.Error(), http.StatusInternalServerError) From 04a2e460ae8b1e884795bdf5200e1efb671ac958 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 09:52:00 +0100 Subject: [PATCH 072/341] Refactor metricstore. Initial stub for cluster/ subcluster specific retention times --- internal/metricstore/avroCheckpoint.go | 6 +- internal/metricstore/config.go | 70 ++++++++++- internal/metricstore/configSchema.go | 38 +++++- .../{memorystore.go => metricstore.go} | 114 ++++++++++++++---- ...emorystore_test.go => metricstore_test.go} | 2 +- 5 files changed, 203 insertions(+), 27 deletions(-) rename internal/metricstore/{memorystore.go => metricstore.go} (76%) rename internal/metricstore/{memorystore_test.go => metricstore_test.go} (99%) diff --git a/internal/metricstore/avroCheckpoint.go b/internal/metricstore/avroCheckpoint.go index 275a64bd..aa14ce5a 100644 --- a/internal/metricstore/avroCheckpoint.go +++ b/internal/metricstore/avroCheckpoint.go @@ -24,8 +24,10 @@ import ( "github.com/linkedin/goavro/v2" ) -var NumAvroWorkers int = DefaultAvroWorkers -var startUp bool = true +var ( + NumAvroWorkers int = DefaultAvroWorkers + startUp bool = true +) func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { levels := make([]*AvroLevel, 0) diff --git a/internal/metricstore/config.go b/internal/metricstore/config.go index 97f16c46..06ae774d 100644 --- a/internal/metricstore/config.go +++ b/internal/metricstore/config.go @@ -33,8 +33,19 @@ type MetricStoreConfig struct { DumpToFile string `json:"dump-to-file"` EnableGops bool `json:"gops"` } `json:"debug"` + // Global default retention duration RetentionInMemory string `json:"retention-in-memory"` - Archive struct { + // Per-cluster retention overrides + Clusters []struct { + Cluster string `json:"cluster"` + RetentionInMemory string `json:"retention-in-memory"` + // Per-subcluster retention overrides within this cluster + SubClusters []struct { + SubCluster string `json:"subcluster"` + RetentionInMemory string `json:"retention-in-memory"` + } `json:"subclusters,omitempty"` + } `json:"clusters,omitempty"` + Archive struct { Interval string `json:"interval"` RootDir string `json:"directory"` DeleteInstead bool `json:"delete-instead"` @@ -50,6 +61,14 @@ type MetricStoreConfig struct { var Keys MetricStoreConfig +type retentionConfig struct { + global time.Duration + clusterMap map[string]time.Duration + subClusterMap map[string]map[string]time.Duration +} + +var retentionLookup *retentionConfig + // AggregationStrategy for aggregation over multiple values at different cpus/sockets/..., not time! type AggregationStrategy int @@ -113,3 +132,52 @@ func AddMetric(name string, metric MetricConfig) error { return nil } + +func GetRetentionDuration(cluster, subCluster string) (time.Duration, error) { + if retentionLookup == nil { + return 0, fmt.Errorf("[METRICSTORE]> retention configuration not initialized") + } + + if subCluster != "" { + if subMap, ok := retentionLookup.subClusterMap[cluster]; ok { + if retention, ok := subMap[subCluster]; ok { + return retention, nil + } + } + } + + if retention, ok := retentionLookup.clusterMap[cluster]; ok { + return retention, nil + } + + return retentionLookup.global, nil +} + +// GetShortestRetentionDuration returns the shortest configured retention duration +// across all levels (global, cluster, and subcluster configurations). +// Returns 0 if retentionLookup is not initialized or global retention is not set. +func GetShortestRetentionDuration() time.Duration { + if retentionLookup == nil || retentionLookup.global <= 0 { + return 0 + } + + shortest := retentionLookup.global + + // Check all cluster-level retention durations + for _, clusterRetention := range retentionLookup.clusterMap { + if clusterRetention > 0 && clusterRetention < shortest { + shortest = clusterRetention + } + } + + // Check all subcluster-level retention durations + for _, subClusterMap := range retentionLookup.subClusterMap { + for _, scRetention := range subClusterMap { + if scRetention > 0 && scRetention < shortest { + shortest = scRetention + } + } + } + + return shortest +} diff --git a/internal/metricstore/configSchema.go b/internal/metricstore/configSchema.go index f1a20a73..868bacc5 100644 --- a/internal/metricstore/configSchema.go +++ b/internal/metricstore/configSchema.go @@ -46,9 +46,45 @@ const configSchema = `{ } }, "retention-in-memory": { - "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.", + "description": "Global default: Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.", "type": "string" }, + "clusters": { + "description": "Optional per-cluster retention overrides", + "type": "array", + "items": { + "type": "object", + "required": ["cluster"], + "properties": { + "cluster": { + "description": "Cluster name", + "type": "string" + }, + "retention-in-memory": { + "description": "Cluster-specific retention duration (overrides global default)", + "type": "string" + }, + "subclusters": { + "description": "Optional per-subcluster retention overrides", + "type": "array", + "items": { + "type": "object", + "required": ["subcluster"], + "properties": { + "subcluster": { + "description": "Subcluster name", + "type": "string" + }, + "retention-in-memory": { + "description": "Subcluster-specific retention duration (overrides cluster and global default)", + "type": "string" + } + } + } + } + } + } + }, "nats": { "description": "Configuration for accepting published data through NATS.", "type": "array", diff --git a/internal/metricstore/memorystore.go b/internal/metricstore/metricstore.go similarity index 76% rename from internal/metricstore/memorystore.go rename to internal/metricstore/metricstore.go index 14a02fcd..5a5c3bce 100644 --- a/internal/metricstore/memorystore.go +++ b/internal/metricstore/metricstore.go @@ -98,6 +98,49 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { } } + globalRetention, err := time.ParseDuration(Keys.RetentionInMemory) + if err != nil { + cclog.Fatal(err) + } + + retentionLookup = &retentionConfig{ + global: globalRetention, + clusterMap: make(map[string]time.Duration), + subClusterMap: make(map[string]map[string]time.Duration), + } + + for _, clusterCfg := range Keys.Clusters { + if clusterCfg.RetentionInMemory != "" { + clusterRetention, err := time.ParseDuration(clusterCfg.RetentionInMemory) + if err != nil { + cclog.Warnf("[METRICSTORE]> Invalid retention duration for cluster '%s': %s\n", clusterCfg.Cluster, err.Error()) + continue + } + retentionLookup.clusterMap[clusterCfg.Cluster] = clusterRetention + cclog.Debugf("[METRICSTORE]> Cluster '%s' retention: %s\n", clusterCfg.Cluster, clusterRetention) + } + + if len(clusterCfg.SubClusters) > 0 { + if retentionLookup.subClusterMap[clusterCfg.Cluster] == nil { + retentionLookup.subClusterMap[clusterCfg.Cluster] = make(map[string]time.Duration) + } + + for _, scCfg := range clusterCfg.SubClusters { + if scCfg.RetentionInMemory != "" { + scRetention, err := time.ParseDuration(scCfg.RetentionInMemory) + if err != nil { + cclog.Warnf("[METRICSTORE]> Invalid retention duration for subcluster '%s/%s': %s\n", + clusterCfg.Cluster, scCfg.SubCluster, err.Error()) + continue + } + retentionLookup.subClusterMap[clusterCfg.Cluster][scCfg.SubCluster] = scRetention + cclog.Debugf("[METRICSTORE]> SubCluster '%s/%s' retention: %s\n", + clusterCfg.Cluster, scCfg.SubCluster, scRetention) + } + } + } + } + // Pass the config.MetricStoreKeys InitMetrics(Metrics) @@ -208,32 +251,22 @@ func Shutdown() { cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files) } -func getName(m *MemoryStore, i int) string { - for key, val := range m.Metrics { - if val.offset == i { - return key - } - } - return "" -} - func Retention(wg *sync.WaitGroup, ctx context.Context) { ms := GetMemoryStore() go func() { defer wg.Done() - d, err := time.ParseDuration(Keys.RetentionInMemory) - if err != nil { - cclog.Fatal(err) - } - if d <= 0 { + shortestRetention := GetShortestRetentionDuration() + if shortestRetention <= 0 { return } - tickInterval := d / 2 + tickInterval := shortestRetention / 2 if tickInterval <= 0 { return } + cclog.Debugf("[METRICSTORE]> Retention ticker interval set to %s (half of shortest retention: %s)\n", + tickInterval, shortestRetention) ticker := time.NewTicker(tickInterval) defer ticker.Stop() @@ -242,13 +275,50 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - t := time.Now().Add(-d) - cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) - freed, err := ms.Free(nil, t.Unix()) - if err != nil { - cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error()) - } else { - cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed) + totalFreed := 0 + + clusters := ms.ListChildren(nil) + for _, cluster := range clusters { + retention, err := GetRetentionDuration(cluster, "") + if err != nil { + cclog.Warnf("[METRICSTORE]> Could not get retention for cluster '%s': %s\n", cluster, err.Error()) + continue + } + if retention <= 0 { + continue + } + + t := time.Now().Add(-retention) + cclog.Debugf("[METRICSTORE]> Freeing buffers for cluster '%s' (older than %s, retention: %s)...\n", + cluster, t.Format(time.RFC3339), retention) + + subClusters := ms.ListChildren([]string{cluster}) + for _, subCluster := range subClusters { + scRetention, err := GetRetentionDuration(cluster, subCluster) + if err != nil { + cclog.Warnf("[METRICSTORE]> Could not get retention for subcluster '%s/%s': %s\n", + cluster, subCluster, err.Error()) + continue + } + if scRetention <= 0 { + continue + } + + scTime := time.Now().Add(-scRetention) + freed, err := ms.Free([]string{cluster, subCluster}, scTime.Unix()) + if err != nil { + cclog.Errorf("[METRICSTORE]> freeing buffers for '%s/%s' failed: %s\n", + cluster, subCluster, err.Error()) + } else if freed > 0 { + cclog.Debugf("[METRICSTORE]> freed %d buffers for '%s/%s' (retention: %s)\n", + freed, cluster, subCluster, scRetention) + totalFreed += freed + } + } + } + + if totalFreed > 0 { + cclog.Infof("[METRICSTORE]> Total buffers freed: %d\n", totalFreed) } } } diff --git a/internal/metricstore/memorystore_test.go b/internal/metricstore/metricstore_test.go similarity index 99% rename from internal/metricstore/memorystore_test.go rename to internal/metricstore/metricstore_test.go index 29379d21..fd7c963f 100644 --- a/internal/metricstore/memorystore_test.go +++ b/internal/metricstore/metricstore_test.go @@ -131,7 +131,7 @@ func TestBufferWrite(t *testing.T) { func TestBufferRead(t *testing.T) { b := newBuffer(100, 10) - + // Write some test data b.write(100, schema.Float(1.0)) b.write(110, schema.Float(2.0)) From 754f7e16f67e0708a479831c8360064d74147633 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 09:52:31 +0100 Subject: [PATCH 073/341] Reformat with gofumpt --- internal/repository/node.go | 1 - pkg/archive/fsBackend.go | 4 ++-- pkg/archive/s3Backend_test.go | 42 ++++++++++++++++----------------- tools/archive-manager/main.go | 1 - tools/archive-migration/main.go | 3 +-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/internal/repository/node.go b/internal/repository/node.go index 752a36fa..2890cdbc 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -561,7 +561,6 @@ func (r *NodeRepository) GetNodesForList( nodeFilter string, page *model.PageRequest, ) ([]string, map[string]string, int, bool, error) { - // Init Return Vars nodes := make([]string, 0) stateMap := make(map[string]string) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 020f2aa4..61921d70 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -188,7 +188,7 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) (uint64, error) { if isEmpty { cclog.Infof("fsBackend Init() > Bootstrapping new archive at %s", fsa.path) versionStr := fmt.Sprintf("%d\n", Version) - if err := os.WriteFile(filepath.Join(fsa.path, "version.txt"), []byte(versionStr), 0644); err != nil { + if err := os.WriteFile(filepath.Join(fsa.path, "version.txt"), []byte(versionStr), 0o644); err != nil { cclog.Errorf("fsBackend Init() > failed to create version.txt: %v", err) return 0, err } @@ -674,7 +674,7 @@ func (fsa *FsArchive) ImportJob( func (fsa *FsArchive) StoreClusterCfg(name string, config *schema.Cluster) error { dir := filepath.Join(fsa.path, name) - if err := os.MkdirAll(dir, 0777); err != nil { + if err := os.MkdirAll(dir, 0o777); err != nil { cclog.Errorf("StoreClusterCfg() > mkdir error: %v", err) return err } diff --git a/pkg/archive/s3Backend_test.go b/pkg/archive/s3Backend_test.go index 2b79db7f..0b4e17a2 100644 --- a/pkg/archive/s3Backend_test.go +++ b/pkg/archive/s3Backend_test.go @@ -41,7 +41,7 @@ func (m *MockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, if !exists { return nil, fmt.Errorf("NoSuchKey: object not found") } - + contentLength := int64(len(data)) return &s3.GetObjectOutput{ Body: io.NopCloser(bytes.NewReader(data)), @@ -65,7 +65,7 @@ func (m *MockS3Client) HeadObject(ctx context.Context, params *s3.HeadObjectInpu if !exists { return nil, fmt.Errorf("NotFound") } - + contentLength := int64(len(data)) return &s3.HeadObjectOutput{ ContentLength: &contentLength, @@ -86,12 +86,12 @@ func (m *MockS3Client) CopyObject(ctx context.Context, params *s3.CopyObjectInpu return nil, fmt.Errorf("invalid CopySource") } sourceKey := parts[1] - + data, exists := m.objects[sourceKey] if !exists { return nil, fmt.Errorf("source not found") } - + destKey := aws.ToString(params.Key) m.objects[destKey] = data return &s3.CopyObjectOutput{}, nil @@ -100,15 +100,15 @@ func (m *MockS3Client) CopyObject(ctx context.Context, params *s3.CopyObjectInpu func (m *MockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { prefix := aws.ToString(params.Prefix) delimiter := aws.ToString(params.Delimiter) - + var contents []types.Object commonPrefixes := make(map[string]bool) - + for key, data := range m.objects { if !strings.HasPrefix(key, prefix) { continue } - + if delimiter != "" { // Check if there's a delimiter after the prefix remainder := strings.TrimPrefix(key, prefix) @@ -120,21 +120,21 @@ func (m *MockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjects continue } } - + size := int64(len(data)) contents = append(contents, types.Object{ Key: aws.String(key), Size: &size, }) } - + var prefixList []types.CommonPrefix for p := range commonPrefixes { prefixList = append(prefixList, types.CommonPrefix{ Prefix: aws.String(p), }) } - + return &s3.ListObjectsV2Output{ Contents: contents, CommonPrefixes: prefixList, @@ -144,10 +144,10 @@ func (m *MockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjects // Test helper to create a mock S3 archive with test data func setupMockS3Archive(t *testing.T) *MockS3Client { mock := NewMockS3Client() - + // Add version.txt mock.objects["version.txt"] = []byte("2\n") - + // Add a test cluster directory mock.objects["emmy/cluster.json"] = []byte(`{ "name": "emmy", @@ -165,7 +165,7 @@ func setupMockS3Archive(t *testing.T) *MockS3Client { } ] }`) - + // Add a test job mock.objects["emmy/1403/244/1608923076/meta.json"] = []byte(`{ "jobId": 1403244, @@ -174,7 +174,7 @@ func setupMockS3Archive(t *testing.T) *MockS3Client { "numNodes": 1, "resources": [{"hostname": "node001"}] }`) - + mock.objects["emmy/1403/244/1608923076/data.json"] = []byte(`{ "mem_used": { "node": { @@ -184,7 +184,7 @@ func setupMockS3Archive(t *testing.T) *MockS3Client { } } }`) - + return mock } @@ -213,7 +213,7 @@ func TestGetS3Key(t *testing.T) { Cluster: "emmy", StartTime: 1608923076, } - + key := getS3Key(job, "meta.json") expected := "emmy/1403/244/1608923076/meta.json" if key != expected { @@ -227,7 +227,7 @@ func TestGetS3Directory(t *testing.T) { Cluster: "emmy", StartTime: 1608923076, } - + dir := getS3Directory(job) expected := "emmy/1403/244/1608923076/" if dir != expected { @@ -247,13 +247,13 @@ func TestS3ArchiveConfigParsing(t *testing.T) { "region": "us-east-1", "usePathStyle": true }`) - + var cfg S3ArchiveConfig err := json.Unmarshal(rawConfig, &cfg) if err != nil { t.Fatalf("failed to parse config: %v", err) } - + if cfg.Bucket != "test-bucket" { t.Errorf("expected bucket 'test-bucket', got '%s'", cfg.Bucket) } @@ -277,14 +277,14 @@ func TestS3KeyGeneration(t *testing.T) { {1404397, "emmy", 1609300556, "data.json.gz", "emmy/1404/397/1609300556/data.json.gz"}, {42, "fritz", 1234567890, "meta.json", "fritz/0/042/1234567890/meta.json"}, } - + for _, tt := range tests { job := &schema.Job{ JobID: tt.jobID, Cluster: tt.cluster, StartTime: tt.startTime, } - + key := getS3Key(job, tt.file) if key != tt.expected { t.Errorf("for job %d: expected %s, got %s", tt.jobID, tt.expected, key) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index ffcba793..fff81256 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -71,7 +71,6 @@ func countJobsNative(archivePath string) (int, error) { } return nil }) - if err != nil { return 0, fmt.Errorf("failed to walk directory: %w", err) } diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go index 8375ee98..1384e065 100644 --- a/tools/archive-migration/main.go +++ b/tools/archive-migration/main.go @@ -70,7 +70,6 @@ func main() { // Run migration migrated, failed, err := migrateArchive(archivePath, dryRun, numWorkers) - if err != nil { cclog.Errorf("Migration completed with errors: %s", err.Error()) if failed > 0 { @@ -104,5 +103,5 @@ func checkVersion(archivePath string) error { func updateVersion(archivePath string) error { versionFile := filepath.Join(archivePath, "version.txt") - return os.WriteFile(versionFile, []byte("3\n"), 0644) + return os.WriteFile(versionFile, []byte("3\n"), 0o644) } From 25c8fca56136eb04cbfe14d5f18a67082512bc64 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 13 Jan 2026 14:42:24 +0100 Subject: [PATCH 074/341] Revert retention config in metricstore --- internal/metricstore/config.go | 68 ------------------ internal/metricstore/configSchema.go | 38 +--------- internal/metricstore/metricstore.go | 103 ++++----------------------- 3 files changed, 13 insertions(+), 196 deletions(-) diff --git a/internal/metricstore/config.go b/internal/metricstore/config.go index 06ae774d..c789f11c 100644 --- a/internal/metricstore/config.go +++ b/internal/metricstore/config.go @@ -33,18 +33,7 @@ type MetricStoreConfig struct { DumpToFile string `json:"dump-to-file"` EnableGops bool `json:"gops"` } `json:"debug"` - // Global default retention duration - RetentionInMemory string `json:"retention-in-memory"` - // Per-cluster retention overrides - Clusters []struct { - Cluster string `json:"cluster"` RetentionInMemory string `json:"retention-in-memory"` - // Per-subcluster retention overrides within this cluster - SubClusters []struct { - SubCluster string `json:"subcluster"` - RetentionInMemory string `json:"retention-in-memory"` - } `json:"subclusters,omitempty"` - } `json:"clusters,omitempty"` Archive struct { Interval string `json:"interval"` RootDir string `json:"directory"` @@ -61,14 +50,6 @@ type MetricStoreConfig struct { var Keys MetricStoreConfig -type retentionConfig struct { - global time.Duration - clusterMap map[string]time.Duration - subClusterMap map[string]map[string]time.Duration -} - -var retentionLookup *retentionConfig - // AggregationStrategy for aggregation over multiple values at different cpus/sockets/..., not time! type AggregationStrategy int @@ -132,52 +113,3 @@ func AddMetric(name string, metric MetricConfig) error { return nil } - -func GetRetentionDuration(cluster, subCluster string) (time.Duration, error) { - if retentionLookup == nil { - return 0, fmt.Errorf("[METRICSTORE]> retention configuration not initialized") - } - - if subCluster != "" { - if subMap, ok := retentionLookup.subClusterMap[cluster]; ok { - if retention, ok := subMap[subCluster]; ok { - return retention, nil - } - } - } - - if retention, ok := retentionLookup.clusterMap[cluster]; ok { - return retention, nil - } - - return retentionLookup.global, nil -} - -// GetShortestRetentionDuration returns the shortest configured retention duration -// across all levels (global, cluster, and subcluster configurations). -// Returns 0 if retentionLookup is not initialized or global retention is not set. -func GetShortestRetentionDuration() time.Duration { - if retentionLookup == nil || retentionLookup.global <= 0 { - return 0 - } - - shortest := retentionLookup.global - - // Check all cluster-level retention durations - for _, clusterRetention := range retentionLookup.clusterMap { - if clusterRetention > 0 && clusterRetention < shortest { - shortest = clusterRetention - } - } - - // Check all subcluster-level retention durations - for _, subClusterMap := range retentionLookup.subClusterMap { - for _, scRetention := range subClusterMap { - if scRetention > 0 && scRetention < shortest { - shortest = scRetention - } - } - } - - return shortest -} diff --git a/internal/metricstore/configSchema.go b/internal/metricstore/configSchema.go index 868bacc5..f1a20a73 100644 --- a/internal/metricstore/configSchema.go +++ b/internal/metricstore/configSchema.go @@ -46,45 +46,9 @@ const configSchema = `{ } }, "retention-in-memory": { - "description": "Global default: Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.", + "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.", "type": "string" }, - "clusters": { - "description": "Optional per-cluster retention overrides", - "type": "array", - "items": { - "type": "object", - "required": ["cluster"], - "properties": { - "cluster": { - "description": "Cluster name", - "type": "string" - }, - "retention-in-memory": { - "description": "Cluster-specific retention duration (overrides global default)", - "type": "string" - }, - "subclusters": { - "description": "Optional per-subcluster retention overrides", - "type": "array", - "items": { - "type": "object", - "required": ["subcluster"], - "properties": { - "subcluster": { - "description": "Subcluster name", - "type": "string" - }, - "retention-in-memory": { - "description": "Subcluster-specific retention duration (overrides cluster and global default)", - "type": "string" - } - } - } - } - } - } - }, "nats": { "description": "Configuration for accepting published data through NATS.", "type": "array", diff --git a/internal/metricstore/metricstore.go b/internal/metricstore/metricstore.go index 5a5c3bce..ac8948ae 100644 --- a/internal/metricstore/metricstore.go +++ b/internal/metricstore/metricstore.go @@ -98,49 +98,6 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { } } - globalRetention, err := time.ParseDuration(Keys.RetentionInMemory) - if err != nil { - cclog.Fatal(err) - } - - retentionLookup = &retentionConfig{ - global: globalRetention, - clusterMap: make(map[string]time.Duration), - subClusterMap: make(map[string]map[string]time.Duration), - } - - for _, clusterCfg := range Keys.Clusters { - if clusterCfg.RetentionInMemory != "" { - clusterRetention, err := time.ParseDuration(clusterCfg.RetentionInMemory) - if err != nil { - cclog.Warnf("[METRICSTORE]> Invalid retention duration for cluster '%s': %s\n", clusterCfg.Cluster, err.Error()) - continue - } - retentionLookup.clusterMap[clusterCfg.Cluster] = clusterRetention - cclog.Debugf("[METRICSTORE]> Cluster '%s' retention: %s\n", clusterCfg.Cluster, clusterRetention) - } - - if len(clusterCfg.SubClusters) > 0 { - if retentionLookup.subClusterMap[clusterCfg.Cluster] == nil { - retentionLookup.subClusterMap[clusterCfg.Cluster] = make(map[string]time.Duration) - } - - for _, scCfg := range clusterCfg.SubClusters { - if scCfg.RetentionInMemory != "" { - scRetention, err := time.ParseDuration(scCfg.RetentionInMemory) - if err != nil { - cclog.Warnf("[METRICSTORE]> Invalid retention duration for subcluster '%s/%s': %s\n", - clusterCfg.Cluster, scCfg.SubCluster, err.Error()) - continue - } - retentionLookup.subClusterMap[clusterCfg.Cluster][scCfg.SubCluster] = scRetention - cclog.Debugf("[METRICSTORE]> SubCluster '%s/%s' retention: %s\n", - clusterCfg.Cluster, scCfg.SubCluster, scRetention) - } - } - } - } - // Pass the config.MetricStoreKeys InitMetrics(Metrics) @@ -256,17 +213,18 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { go func() { defer wg.Done() - shortestRetention := GetShortestRetentionDuration() - if shortestRetention <= 0 { + d, err := time.ParseDuration(Keys.RetentionInMemory) + if err != nil { + cclog.Fatal(err) + } + if d <= 0 { return } - tickInterval := shortestRetention / 2 + tickInterval := d / 2 if tickInterval <= 0 { return } - cclog.Debugf("[METRICSTORE]> Retention ticker interval set to %s (half of shortest retention: %s)\n", - tickInterval, shortestRetention) ticker := time.NewTicker(tickInterval) defer ticker.Stop() @@ -275,50 +233,13 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - totalFreed := 0 - - clusters := ms.ListChildren(nil) - for _, cluster := range clusters { - retention, err := GetRetentionDuration(cluster, "") - if err != nil { - cclog.Warnf("[METRICSTORE]> Could not get retention for cluster '%s': %s\n", cluster, err.Error()) - continue - } - if retention <= 0 { - continue - } - - t := time.Now().Add(-retention) - cclog.Debugf("[METRICSTORE]> Freeing buffers for cluster '%s' (older than %s, retention: %s)...\n", - cluster, t.Format(time.RFC3339), retention) - - subClusters := ms.ListChildren([]string{cluster}) - for _, subCluster := range subClusters { - scRetention, err := GetRetentionDuration(cluster, subCluster) + t := time.Now().Add(-d) + cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) + freed, err := ms.Free(nil, t.Unix()) if err != nil { - cclog.Warnf("[METRICSTORE]> Could not get retention for subcluster '%s/%s': %s\n", - cluster, subCluster, err.Error()) - continue - } - if scRetention <= 0 { - continue - } - - scTime := time.Now().Add(-scRetention) - freed, err := ms.Free([]string{cluster, subCluster}, scTime.Unix()) - if err != nil { - cclog.Errorf("[METRICSTORE]> freeing buffers for '%s/%s' failed: %s\n", - cluster, subCluster, err.Error()) - } else if freed > 0 { - cclog.Debugf("[METRICSTORE]> freed %d buffers for '%s/%s' (retention: %s)\n", - freed, cluster, subCluster, scRetention) - totalFreed += freed - } - } - } - - if totalFreed > 0 { - cclog.Infof("[METRICSTORE]> Total buffers freed: %d\n", totalFreed) + cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error()) + } else { + cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed) } } } From 518e9950eac893f54d1d441901860d7e12de71dc Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 13 Jan 2026 16:59:52 +0100 Subject: [PATCH 075/341] add job exclusivity filter, review db indices --- api/schema.graphqls | 1 + internal/graph/generated/generated.go | 258 +++++++++--------- internal/graph/model/models_gen.go | 1 + internal/graph/schema.resolvers.go | 16 +- internal/repository/jobQuery.go | 88 +++--- .../sqlite3/09_add-job-cache.up.sql | 72 ++--- .../migrations/sqlite3/10_node-table.up.sql | 2 - internal/routerConfig/routes.go | 6 + web/frontend/src/generic/Filters.svelte | 94 ++++--- .../src/generic/filters/JobStates.svelte | 75 ++++- 10 files changed, 365 insertions(+), 248 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 1c81e6b6..7be43f73 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -458,6 +458,7 @@ input JobFilter { state: [JobState!] metricStats: [MetricStatItem!] shared: String + schedule: String node: StringInput } diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index d96ccf1d..2d3aca04 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -2741,6 +2741,7 @@ input JobFilter { state: [JobState!] metricStats: [MetricStatItem!] shared: String + schedule: String node: StringInput } @@ -3002,7 +3003,7 @@ func (ec *executionContext) field_Query_jobMetrics_args(ctx context.Context, raw return nil, err } args["metrics"] = arg1 - arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ) + arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ) if err != nil { return nil, err } @@ -3159,7 +3160,7 @@ func (ec *executionContext) field_Query_nodeMetricsList_args(ctx context.Context return nil, err } args["nodeFilter"] = arg3 - arg4, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ) + arg4, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ) if err != nil { return nil, err } @@ -3205,7 +3206,7 @@ func (ec *executionContext) field_Query_nodeMetrics_args(ctx context.Context, ra return nil, err } args["nodes"] = arg1 - arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ) + arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ) if err != nil { return nil, err } @@ -3336,7 +3337,7 @@ func (ec *executionContext) field_Query_scopedJobStats_args(ctx context.Context, return nil, err } args["metrics"] = arg1 - arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ) + arg2, err := graphql.ProcessArgField(ctx, rawArgs, "scopes", ec.unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ) if err != nil { return nil, err } @@ -3562,7 +3563,7 @@ func (ec *executionContext) _Cluster_subClusters(ctx context.Context, field grap return obj.SubClusters, nil }, nil, - ec.marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterᚄ, + ec.marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterᚄ, true, true, ) @@ -3648,7 +3649,7 @@ func (ec *executionContext) _ClusterMetricWithName_unit(ctx context.Context, fie return obj.Unit, nil }, nil, - ec.marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit, + ec.marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit, true, false, ) @@ -3712,7 +3713,7 @@ func (ec *executionContext) _ClusterMetricWithName_data(ctx context.Context, fie return obj.Data, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -4200,7 +4201,7 @@ func (ec *executionContext) _GlobalMetricListItem_unit(ctx context.Context, fiel return obj.Unit, nil }, nil, - ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit, + ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit, true, true, ) @@ -4235,7 +4236,7 @@ func (ec *executionContext) _GlobalMetricListItem_scope(ctx context.Context, fie return obj.Scope, nil }, nil, - ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope, + ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope, true, true, ) @@ -4293,7 +4294,7 @@ func (ec *executionContext) _GlobalMetricListItem_availability(ctx context.Conte return obj.Availability, nil }, nil, - ec.marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupportᚄ, + ec.marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupportᚄ, true, true, ) @@ -4966,7 +4967,7 @@ func (ec *executionContext) _Job_state(ctx context.Context, field graphql.Collec return obj.State, nil }, nil, - ec.marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobState, + ec.marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobState, true, true, ) @@ -4995,7 +4996,7 @@ func (ec *executionContext) _Job_tags(ctx context.Context, field graphql.Collect return ec.resolvers.Job().Tags(ctx, obj) }, nil, - ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTagᚄ, + ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, true, true, ) @@ -5034,7 +5035,7 @@ func (ec *executionContext) _Job_resources(ctx context.Context, field graphql.Co return obj.Resources, nil }, nil, - ec.marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐResourceᚄ, + ec.marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResourceᚄ, true, true, ) @@ -5401,7 +5402,7 @@ func (ec *executionContext) _JobMetric_unit(ctx context.Context, field graphql.C return obj.Unit, nil }, nil, - ec.marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit, + ec.marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit, true, false, ) @@ -5465,7 +5466,7 @@ func (ec *executionContext) _JobMetric_series(ctx context.Context, field graphql return obj.Series, nil }, nil, - ec.marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSeriesᚄ, + ec.marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeriesᚄ, true, false, ) @@ -5504,7 +5505,7 @@ func (ec *executionContext) _JobMetric_statisticsSeries(ctx context.Context, fie return obj.StatisticsSeries, nil }, nil, - ec.marshalOStatsSeries2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐStatsSeries, + ec.marshalOStatsSeries2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐStatsSeries, true, false, ) @@ -5572,7 +5573,7 @@ func (ec *executionContext) _JobMetricWithName_scope(ctx context.Context, field return obj.Scope, nil }, nil, - ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope, + ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope, true, true, ) @@ -5601,7 +5602,7 @@ func (ec *executionContext) _JobMetricWithName_metric(ctx context.Context, field return obj.Metric, nil }, nil, - ec.marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobMetric, + ec.marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobMetric, true, true, ) @@ -5640,7 +5641,7 @@ func (ec *executionContext) _JobResultList_items(ctx context.Context, field grap return obj.Items, nil }, nil, - ec.marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobᚄ, + ec.marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobᚄ, true, true, ) @@ -6720,7 +6721,7 @@ func (ec *executionContext) _MetricConfig_unit(ctx context.Context, field graphq return obj.Unit, nil }, nil, - ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit, + ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit, true, true, ) @@ -6755,7 +6756,7 @@ func (ec *executionContext) _MetricConfig_scope(ctx context.Context, field graph return obj.Scope, nil }, nil, - ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope, + ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope, true, true, ) @@ -6987,7 +6988,7 @@ func (ec *executionContext) _MetricConfig_subClusters(ctx context.Context, field return obj.SubClusters, nil }, nil, - ec.marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterConfigᚄ, + ec.marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfigᚄ, true, true, ) @@ -7059,7 +7060,7 @@ func (ec *executionContext) _MetricFootprints_data(ctx context.Context, field gr return obj.Data, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -7446,7 +7447,7 @@ func (ec *executionContext) _MetricValue_unit(ctx context.Context, field graphql return obj.Unit, nil }, nil, - ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit, + ec.marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit, true, true, ) @@ -7511,7 +7512,7 @@ func (ec *executionContext) _Mutation_createTag(ctx context.Context, field graph return ec.resolvers.Mutation().CreateTag(ctx, fc.Args["type"].(string), fc.Args["name"].(string), fc.Args["scope"].(string)) }, nil, - ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTag, + ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag, true, true, ) @@ -7603,7 +7604,7 @@ func (ec *executionContext) _Mutation_addTagsToJob(ctx context.Context, field gr return ec.resolvers.Mutation().AddTagsToJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) }, nil, - ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTagᚄ, + ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, true, true, ) @@ -7654,7 +7655,7 @@ func (ec *executionContext) _Mutation_removeTagsFromJob(ctx context.Context, fie return ec.resolvers.Mutation().RemoveTagsFromJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) }, nil, - ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTagᚄ, + ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, true, true, ) @@ -7815,7 +7816,7 @@ func (ec *executionContext) _NamedStats_data(ctx context.Context, field graphql. return obj.Data, nil }, nil, - ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricStatistics, + ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricStatistics, true, true, ) @@ -7881,7 +7882,7 @@ func (ec *executionContext) _NamedStatsWithScope_scope(ctx context.Context, fiel return obj.Scope, nil }, nil, - ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope, + ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope, true, true, ) @@ -8179,7 +8180,7 @@ func (ec *executionContext) _Node_schedulerState(ctx context.Context, field grap return ec.resolvers.Node().SchedulerState(ctx, obj) }, nil, - ec.marshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState, + ec.marshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState, true, true, ) @@ -8390,7 +8391,7 @@ func (ec *executionContext) _NodeStateResultList_items(ctx context.Context, fiel return obj.Items, nil }, nil, - ec.marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNodeᚄ, + ec.marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNodeᚄ, true, true, ) @@ -8801,7 +8802,7 @@ func (ec *executionContext) _Query_clusters(ctx context.Context, field graphql.C return ec.resolvers.Query().Clusters(ctx) }, nil, - ec.marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterᚄ, + ec.marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterᚄ, true, true, ) @@ -8838,7 +8839,7 @@ func (ec *executionContext) _Query_tags(ctx context.Context, field graphql.Colle return ec.resolvers.Query().Tags(ctx) }, nil, - ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTagᚄ, + ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, true, true, ) @@ -8877,7 +8878,7 @@ func (ec *executionContext) _Query_globalMetrics(ctx context.Context, field grap return ec.resolvers.Query().GlobalMetrics(ctx) }, nil, - ec.marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐGlobalMetricListItemᚄ, + ec.marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItemᚄ, true, true, ) @@ -9015,7 +9016,7 @@ func (ec *executionContext) _Query_node(ctx context.Context, field graphql.Colle return ec.resolvers.Query().Node(ctx, fc.Args["id"].(string)) }, nil, - ec.marshalONode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNode, + ec.marshalONode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode, true, false, ) @@ -9223,7 +9224,7 @@ func (ec *executionContext) _Query_job(ctx context.Context, field graphql.Collec return ec.resolvers.Query().Job(ctx, fc.Args["id"].(string)) }, nil, - ec.marshalOJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJob, + ec.marshalOJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob, true, false, ) @@ -10180,7 +10181,7 @@ func (ec *executionContext) _ScopedStats_data(ctx context.Context, field graphql return obj.Data, nil }, nil, - ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricStatistics, + ec.marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricStatistics, true, true, ) @@ -10275,7 +10276,7 @@ func (ec *executionContext) _Series_statistics(ctx context.Context, field graphq return obj.Statistics, nil }, nil, - ec.marshalOMetricStatistics2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricStatistics, + ec.marshalOMetricStatistics2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricStatistics, true, false, ) @@ -10312,7 +10313,7 @@ func (ec *executionContext) _Series_data(ctx context.Context, field graphql.Coll return obj.Data, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -10341,7 +10342,7 @@ func (ec *executionContext) _StatsSeries_mean(ctx context.Context, field graphql return obj.Mean, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -10370,7 +10371,7 @@ func (ec *executionContext) _StatsSeries_median(ctx context.Context, field graph return obj.Median, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -10399,7 +10400,7 @@ func (ec *executionContext) _StatsSeries_min(ctx context.Context, field graphql. return obj.Min, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -10428,7 +10429,7 @@ func (ec *executionContext) _StatsSeries_max(ctx context.Context, field graphql. return obj.Max, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -10660,7 +10661,7 @@ func (ec *executionContext) _SubCluster_flopRateScalar(ctx context.Context, fiel return obj.FlopRateScalar, nil }, nil, - ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricValue, + ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricValue, true, true, ) @@ -10697,7 +10698,7 @@ func (ec *executionContext) _SubCluster_flopRateSimd(ctx context.Context, field return obj.FlopRateSimd, nil }, nil, - ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricValue, + ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricValue, true, true, ) @@ -10734,7 +10735,7 @@ func (ec *executionContext) _SubCluster_memoryBandwidth(ctx context.Context, fie return obj.MemoryBandwidth, nil }, nil, - ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricValue, + ec.marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricValue, true, true, ) @@ -10771,7 +10772,7 @@ func (ec *executionContext) _SubCluster_topology(ctx context.Context, field grap return obj.Topology, nil }, nil, - ec.marshalNTopology2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTopology, + ec.marshalNTopology2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTopology, true, true, ) @@ -10814,7 +10815,7 @@ func (ec *executionContext) _SubCluster_metricConfig(ctx context.Context, field return obj.MetricConfig, nil }, nil, - ec.marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricConfigᚄ, + ec.marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ, true, true, ) @@ -11273,7 +11274,7 @@ func (ec *executionContext) _TimeWeights_nodeHours(ctx context.Context, field gr return obj.NodeHours, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -11302,7 +11303,7 @@ func (ec *executionContext) _TimeWeights_accHours(ctx context.Context, field gra return obj.AccHours, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -11331,7 +11332,7 @@ func (ec *executionContext) _TimeWeights_coreHours(ctx context.Context, field gr return obj.CoreHours, nil }, nil, - ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ, + ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ, true, true, ) @@ -11505,7 +11506,7 @@ func (ec *executionContext) _Topology_accelerators(ctx context.Context, field gr return obj.Accelerators, nil }, nil, - ec.marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐAcceleratorᚄ, + ec.marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAcceleratorᚄ, true, false, ) @@ -13198,7 +13199,7 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any asMap[k] = v } - fieldsInOrder := [...]string{"tags", "dbId", "jobId", "arrayJobId", "user", "project", "jobName", "cluster", "partition", "duration", "energy", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "metricStats", "shared", "node"} + fieldsInOrder := [...]string{"tags", "dbId", "jobId", "arrayJobId", "user", "project", "jobName", "cluster", "partition", "duration", "energy", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "metricStats", "shared", "schedule", "node"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -13319,7 +13320,7 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any it.StartTime = data case "state": ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("state")) - data, err := ec.unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobStateᚄ(ctx, v) + data, err := ec.unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobStateᚄ(ctx, v) if err != nil { return it, err } @@ -13338,6 +13339,13 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any return it, err } it.Shared = data + case "schedule": + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("schedule")) + data, err := ec.unmarshalOString2ᚖstring(ctx, v) + if err != nil { + return it, err + } + it.Schedule = data case "node": ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("node")) data, err := ec.unmarshalOStringInput2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐStringInput(ctx, v) @@ -13422,7 +13430,7 @@ func (ec *executionContext) unmarshalInputNodeFilter(ctx context.Context, obj an it.Subcluster = data case "schedulerState": ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("schedulerState")) - data, err := ec.unmarshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState(ctx, v) + data, err := ec.unmarshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState(ctx, v) if err != nil { return it, err } @@ -17481,7 +17489,7 @@ func (ec *executionContext) ___Type(ctx context.Context, sel ast.SelectionSet, o // region ***************************** type.gotpl ***************************** -func (ec *executionContext) marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐAccelerator(ctx context.Context, sel ast.SelectionSet, v *schema.Accelerator) graphql.Marshaler { +func (ec *executionContext) marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAccelerator(ctx context.Context, sel ast.SelectionSet, v *schema.Accelerator) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -17507,7 +17515,7 @@ func (ec *executionContext) marshalNBoolean2bool(ctx context.Context, sel ast.Se return res } -func (ec *executionContext) marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Cluster) graphql.Marshaler { +func (ec *executionContext) marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Cluster) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -17531,7 +17539,7 @@ func (ec *executionContext) marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpit if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐCluster(ctx, sel, v[i]) + ret[i] = ec.marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐCluster(ctx, sel, v[i]) } if isLen1 { f(i) @@ -17551,7 +17559,7 @@ func (ec *executionContext) marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpit return ret } -func (ec *executionContext) marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐCluster(ctx context.Context, sel ast.SelectionSet, v *schema.Cluster) graphql.Marshaler { +func (ec *executionContext) marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐCluster(ctx context.Context, sel ast.SelectionSet, v *schema.Cluster) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -17629,11 +17637,11 @@ func (ec *executionContext) marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCock return ec._ClusterMetrics(ctx, sel, v) } -func (ec *executionContext) marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupport(ctx context.Context, sel ast.SelectionSet, v schema.ClusterSupport) graphql.Marshaler { +func (ec *executionContext) marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupport(ctx context.Context, sel ast.SelectionSet, v schema.ClusterSupport) graphql.Marshaler { return ec._ClusterSupport(ctx, sel, &v) } -func (ec *executionContext) marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupportᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.ClusterSupport) graphql.Marshaler { +func (ec *executionContext) marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupportᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.ClusterSupport) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -17657,7 +17665,7 @@ func (ec *executionContext) marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCock if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupport(ctx, sel, v[i]) + ret[i] = ec.marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupport(ctx, sel, v[i]) } if isLen1 { f(i) @@ -17812,7 +17820,7 @@ func (ec *executionContext) unmarshalNFloatRange2ᚖgithubᚗcomᚋClusterCockpi return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐGlobalMetricListItemᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.GlobalMetricListItem) graphql.Marshaler { +func (ec *executionContext) marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItemᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.GlobalMetricListItem) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -17836,7 +17844,7 @@ func (ec *executionContext) marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋCl if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐGlobalMetricListItem(ctx, sel, v[i]) + ret[i] = ec.marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItem(ctx, sel, v[i]) } if isLen1 { f(i) @@ -17856,7 +17864,7 @@ func (ec *executionContext) marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋCl return ret } -func (ec *executionContext) marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐGlobalMetricListItem(ctx context.Context, sel ast.SelectionSet, v *schema.GlobalMetricListItem) graphql.Marshaler { +func (ec *executionContext) marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItem(ctx context.Context, sel ast.SelectionSet, v *schema.GlobalMetricListItem) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -18134,7 +18142,7 @@ func (ec *executionContext) marshalNInt2ᚖint(ctx context.Context, sel ast.Sele return res } -func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Job) graphql.Marshaler { +func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Job) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -18158,7 +18166,7 @@ func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋcc if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJob(ctx, sel, v[i]) + ret[i] = ec.marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob(ctx, sel, v[i]) } if isLen1 { f(i) @@ -18178,7 +18186,7 @@ func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋcc return ret } -func (ec *executionContext) marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJob(ctx context.Context, sel ast.SelectionSet, v *schema.Job) graphql.Marshaler { +func (ec *executionContext) marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob(ctx context.Context, sel ast.SelectionSet, v *schema.Job) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -18262,7 +18270,7 @@ func (ec *executionContext) marshalNJobLink2ᚖgithubᚗcomᚋClusterCockpitᚋc return ec._JobLink(ctx, sel, v) } -func (ec *executionContext) marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobMetric(ctx context.Context, sel ast.SelectionSet, v *schema.JobMetric) graphql.Marshaler { +func (ec *executionContext) marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobMetric(ctx context.Context, sel ast.SelectionSet, v *schema.JobMetric) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -18340,13 +18348,13 @@ func (ec *executionContext) marshalNJobResultList2ᚖgithubᚗcomᚋClusterCockp return ec._JobResultList(ctx, sel, v) } -func (ec *executionContext) unmarshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobState(ctx context.Context, v any) (schema.JobState, error) { +func (ec *executionContext) unmarshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobState(ctx context.Context, v any) (schema.JobState, error) { var res schema.JobState err := res.UnmarshalGQL(v) return res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobState(ctx context.Context, sel ast.SelectionSet, v schema.JobState) graphql.Marshaler { +func (ec *executionContext) marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobState(ctx context.Context, sel ast.SelectionSet, v schema.JobState) graphql.Marshaler { return v } @@ -18458,11 +18466,11 @@ func (ec *executionContext) marshalNJobsStatistics2ᚖgithubᚗcomᚋClusterCock return ec._JobsStatistics(ctx, sel, v) } -func (ec *executionContext) marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricConfig(ctx context.Context, sel ast.SelectionSet, v schema.MetricConfig) graphql.Marshaler { +func (ec *executionContext) marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx context.Context, sel ast.SelectionSet, v schema.MetricConfig) graphql.Marshaler { return ec._MetricConfig(ctx, sel, &v) } -func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.MetricConfig) graphql.Marshaler { +func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.MetricConfig) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -18486,7 +18494,7 @@ func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpi if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricConfig(ctx, sel, v[i]) + ret[i] = ec.marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx, sel, v[i]) } if isLen1 { f(i) @@ -18624,13 +18632,13 @@ func (ec *executionContext) marshalNMetricHistoPoints2ᚖgithubᚗcomᚋClusterC return ec._MetricHistoPoints(ctx, sel, v) } -func (ec *executionContext) unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope(ctx context.Context, v any) (schema.MetricScope, error) { +func (ec *executionContext) unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope(ctx context.Context, v any) (schema.MetricScope, error) { var res schema.MetricScope err := res.UnmarshalGQL(v) return res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope(ctx context.Context, sel ast.SelectionSet, v schema.MetricScope) graphql.Marshaler { +func (ec *executionContext) marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope(ctx context.Context, sel ast.SelectionSet, v schema.MetricScope) graphql.Marshaler { return v } @@ -18639,7 +18647,7 @@ func (ec *executionContext) unmarshalNMetricStatItem2ᚖgithubᚗcomᚋClusterCo return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricStatistics(ctx context.Context, sel ast.SelectionSet, v *schema.MetricStatistics) graphql.Marshaler { +func (ec *executionContext) marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricStatistics(ctx context.Context, sel ast.SelectionSet, v *schema.MetricStatistics) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -18649,7 +18657,7 @@ func (ec *executionContext) marshalNMetricStatistics2ᚖgithubᚗcomᚋClusterCo return ec._MetricStatistics(ctx, sel, v) } -func (ec *executionContext) marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricValue(ctx context.Context, sel ast.SelectionSet, v schema.MetricValue) graphql.Marshaler { +func (ec *executionContext) marshalNMetricValue2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricValue(ctx context.Context, sel ast.SelectionSet, v schema.MetricValue) graphql.Marshaler { return ec._MetricValue(ctx, sel, &v) } @@ -18777,7 +18785,7 @@ func (ec *executionContext) marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋCluste return ec._NamedStatsWithScope(ctx, sel, v) } -func (ec *executionContext) marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNodeᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Node) graphql.Marshaler { +func (ec *executionContext) marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNodeᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Node) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -18801,7 +18809,7 @@ func (ec *executionContext) marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋc if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNode(ctx, sel, v[i]) + ret[i] = ec.marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode(ctx, sel, v[i]) } if isLen1 { f(i) @@ -18821,7 +18829,7 @@ func (ec *executionContext) marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋc return ret } -func (ec *executionContext) marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNode(ctx context.Context, sel ast.SelectionSet, v *schema.Node) graphql.Marshaler { +func (ec *executionContext) marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode(ctx context.Context, sel ast.SelectionSet, v *schema.Node) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -19026,24 +19034,24 @@ func (ec *executionContext) marshalNNodesResultList2ᚖgithubᚗcomᚋClusterCoc return ec._NodesResultList(ctx, sel, v) } -func (ec *executionContext) unmarshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloat(ctx context.Context, v any) (schema.Float, error) { +func (ec *executionContext) unmarshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloat(ctx context.Context, v any) (schema.Float, error) { var res schema.Float err := res.UnmarshalGQL(v) return res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloat(ctx context.Context, sel ast.SelectionSet, v schema.Float) graphql.Marshaler { +func (ec *executionContext) marshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloat(ctx context.Context, sel ast.SelectionSet, v schema.Float) graphql.Marshaler { return v } -func (ec *executionContext) unmarshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ(ctx context.Context, v any) ([]schema.Float, error) { +func (ec *executionContext) unmarshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ(ctx context.Context, v any) ([]schema.Float, error) { var vSlice []any vSlice = graphql.CoerceList(v) var err error res := make([]schema.Float, len(vSlice)) for i := range vSlice { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithIndex(i)) - res[i], err = ec.unmarshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloat(ctx, vSlice[i]) + res[i], err = ec.unmarshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloat(ctx, vSlice[i]) if err != nil { return nil, err } @@ -19051,10 +19059,10 @@ func (ec *executionContext) unmarshalNNullableFloat2ᚕgithubᚗcomᚋClusterCoc return res, nil } -func (ec *executionContext) marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.Float) graphql.Marshaler { +func (ec *executionContext) marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloatᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.Float) graphql.Marshaler { ret := make(graphql.Array, len(v)) for i := range v { - ret[i] = ec.marshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloat(ctx, sel, v[i]) + ret[i] = ec.marshalNNullableFloat2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐFloat(ctx, sel, v[i]) } for _, e := range ret { @@ -19066,7 +19074,7 @@ func (ec *executionContext) marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockp return ret } -func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐResourceᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Resource) graphql.Marshaler { +func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResourceᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Resource) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -19090,7 +19098,7 @@ func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpit if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐResource(ctx, sel, v[i]) + ret[i] = ec.marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResource(ctx, sel, v[i]) } if isLen1 { f(i) @@ -19110,7 +19118,7 @@ func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpit return ret } -func (ec *executionContext) marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐResource(ctx context.Context, sel ast.SelectionSet, v *schema.Resource) graphql.Marshaler { +func (ec *executionContext) marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResource(ctx context.Context, sel ast.SelectionSet, v *schema.Resource) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -19120,13 +19128,13 @@ func (ec *executionContext) marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋ return ec._Resource(ctx, sel, v) } -func (ec *executionContext) unmarshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState(ctx context.Context, v any) (schema.SchedulerState, error) { +func (ec *executionContext) unmarshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState(ctx context.Context, v any) (schema.SchedulerState, error) { tmp, err := graphql.UnmarshalString(v) res := schema.SchedulerState(tmp) return res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState(ctx context.Context, sel ast.SelectionSet, v schema.SchedulerState) graphql.Marshaler { +func (ec *executionContext) marshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState(ctx context.Context, sel ast.SelectionSet, v schema.SchedulerState) graphql.Marshaler { _ = sel res := graphql.MarshalString(string(v)) if res == graphql.Null { @@ -19191,7 +19199,7 @@ func (ec *executionContext) marshalNScopedStats2ᚖgithubᚗcomᚋClusterCockpit return ec._ScopedStats(ctx, sel, v) } -func (ec *executionContext) marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSeries(ctx context.Context, sel ast.SelectionSet, v schema.Series) graphql.Marshaler { +func (ec *executionContext) marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeries(ctx context.Context, sel ast.SelectionSet, v schema.Series) graphql.Marshaler { return ec._Series(ctx, sel, &v) } @@ -19251,7 +19259,7 @@ func (ec *executionContext) marshalNString2ᚕstringᚄ(ctx context.Context, sel return ret } -func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubCluster) graphql.Marshaler { +func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubCluster) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -19275,7 +19283,7 @@ func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockp if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubCluster(ctx, sel, v[i]) + ret[i] = ec.marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubCluster(ctx, sel, v[i]) } if isLen1 { f(i) @@ -19295,7 +19303,7 @@ func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockp return ret } -func (ec *executionContext) marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubCluster(ctx context.Context, sel ast.SelectionSet, v *schema.SubCluster) graphql.Marshaler { +func (ec *executionContext) marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubCluster(ctx context.Context, sel ast.SelectionSet, v *schema.SubCluster) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -19305,7 +19313,7 @@ func (ec *executionContext) marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpit return ec._SubCluster(ctx, sel, v) } -func (ec *executionContext) marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubClusterConfig) graphql.Marshaler { +func (ec *executionContext) marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubClusterConfig) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -19329,7 +19337,7 @@ func (ec *executionContext) marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋCluste if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterConfig(ctx, sel, v[i]) + ret[i] = ec.marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfig(ctx, sel, v[i]) } if isLen1 { f(i) @@ -19349,7 +19357,7 @@ func (ec *executionContext) marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋCluste return ret } -func (ec *executionContext) marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSubClusterConfig(ctx context.Context, sel ast.SelectionSet, v *schema.SubClusterConfig) graphql.Marshaler { +func (ec *executionContext) marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfig(ctx context.Context, sel ast.SelectionSet, v *schema.SubClusterConfig) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -19359,11 +19367,11 @@ func (ec *executionContext) marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCo return ec._SubClusterConfig(ctx, sel, v) } -func (ec *executionContext) marshalNTag2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTag(ctx context.Context, sel ast.SelectionSet, v schema.Tag) graphql.Marshaler { +func (ec *executionContext) marshalNTag2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag(ctx context.Context, sel ast.SelectionSet, v schema.Tag) graphql.Marshaler { return ec._Tag(ctx, sel, &v) } -func (ec *executionContext) marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTagᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Tag) graphql.Marshaler { +func (ec *executionContext) marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Tag) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup isLen1 := len(v) == 1 @@ -19387,7 +19395,7 @@ func (ec *executionContext) marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋcc if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTag(ctx, sel, v[i]) + ret[i] = ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag(ctx, sel, v[i]) } if isLen1 { f(i) @@ -19407,7 +19415,7 @@ func (ec *executionContext) marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋcc return ret } -func (ec *executionContext) marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTag(ctx context.Context, sel ast.SelectionSet, v *schema.Tag) graphql.Marshaler { +func (ec *executionContext) marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag(ctx context.Context, sel ast.SelectionSet, v *schema.Tag) graphql.Marshaler { if v == nil { if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow") @@ -19465,11 +19473,11 @@ func (ec *executionContext) marshalNTimeWeights2ᚖgithubᚗcomᚋClusterCockpit return ec._TimeWeights(ctx, sel, v) } -func (ec *executionContext) marshalNTopology2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐTopology(ctx context.Context, sel ast.SelectionSet, v schema.Topology) graphql.Marshaler { +func (ec *executionContext) marshalNTopology2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTopology(ctx context.Context, sel ast.SelectionSet, v schema.Topology) graphql.Marshaler { return ec._Topology(ctx, sel, &v) } -func (ec *executionContext) marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v schema.Unit) graphql.Marshaler { +func (ec *executionContext) marshalNUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v schema.Unit) graphql.Marshaler { return ec._Unit(ctx, sel, &v) } @@ -19726,7 +19734,7 @@ func (ec *executionContext) marshalN__TypeKind2string(ctx context.Context, sel a return res } -func (ec *executionContext) marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐAcceleratorᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Accelerator) graphql.Marshaler { +func (ec *executionContext) marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAcceleratorᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Accelerator) graphql.Marshaler { if v == nil { return graphql.Null } @@ -19753,7 +19761,7 @@ func (ec *executionContext) marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCock if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐAccelerator(ctx, sel, v[i]) + ret[i] = ec.marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAccelerator(ctx, sel, v[i]) } if isLen1 { f(i) @@ -20141,7 +20149,7 @@ func (ec *executionContext) unmarshalOIntRange2ᚖgithubᚗcomᚋClusterCockpit return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalOJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJob(ctx context.Context, sel ast.SelectionSet, v *schema.Job) graphql.Marshaler { +func (ec *executionContext) marshalOJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob(ctx context.Context, sel ast.SelectionSet, v *schema.Job) graphql.Marshaler { if v == nil { return graphql.Null } @@ -20173,7 +20181,7 @@ func (ec *executionContext) marshalOJobLinkResultList2ᚖgithubᚗcomᚋClusterC return ec._JobLinkResultList(ctx, sel, v) } -func (ec *executionContext) unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobStateᚄ(ctx context.Context, v any) ([]schema.JobState, error) { +func (ec *executionContext) unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobStateᚄ(ctx context.Context, v any) ([]schema.JobState, error) { if v == nil { return nil, nil } @@ -20183,7 +20191,7 @@ func (ec *executionContext) unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpit res := make([]schema.JobState, len(vSlice)) for i := range vSlice { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithIndex(i)) - res[i], err = ec.unmarshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobState(ctx, vSlice[i]) + res[i], err = ec.unmarshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobState(ctx, vSlice[i]) if err != nil { return nil, err } @@ -20191,13 +20199,13 @@ func (ec *executionContext) unmarshalOJobState2ᚕgithubᚗcomᚋClusterCockpit return res, nil } -func (ec *executionContext) marshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobStateᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.JobState) graphql.Marshaler { +func (ec *executionContext) marshalOJobState2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobStateᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.JobState) graphql.Marshaler { if v == nil { return graphql.Null } ret := make(graphql.Array, len(v)) for i := range v { - ret[i] = ec.marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐJobState(ctx, sel, v[i]) + ret[i] = ec.marshalNJobState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobState(ctx, sel, v[i]) } for _, e := range ret { @@ -20256,7 +20264,7 @@ func (ec *executionContext) marshalOMetricHistoPoint2ᚕᚖgithubᚗcomᚋCluste return ret } -func (ec *executionContext) unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ(ctx context.Context, v any) ([]schema.MetricScope, error) { +func (ec *executionContext) unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ(ctx context.Context, v any) ([]schema.MetricScope, error) { if v == nil { return nil, nil } @@ -20266,7 +20274,7 @@ func (ec *executionContext) unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockp res := make([]schema.MetricScope, len(vSlice)) for i := range vSlice { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithIndex(i)) - res[i], err = ec.unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope(ctx, vSlice[i]) + res[i], err = ec.unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope(ctx, vSlice[i]) if err != nil { return nil, err } @@ -20274,13 +20282,13 @@ func (ec *executionContext) unmarshalOMetricScope2ᚕgithubᚗcomᚋClusterCockp return res, nil } -func (ec *executionContext) marshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.MetricScope) graphql.Marshaler { +func (ec *executionContext) marshalOMetricScope2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.MetricScope) graphql.Marshaler { if v == nil { return graphql.Null } ret := make(graphql.Array, len(v)) for i := range v { - ret[i] = ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricScope(ctx, sel, v[i]) + ret[i] = ec.marshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricScope(ctx, sel, v[i]) } for _, e := range ret { @@ -20310,7 +20318,7 @@ func (ec *executionContext) unmarshalOMetricStatItem2ᚕᚖgithubᚗcomᚋCluste return res, nil } -func (ec *executionContext) marshalOMetricStatistics2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐMetricStatistics(ctx context.Context, sel ast.SelectionSet, v schema.MetricStatistics) graphql.Marshaler { +func (ec *executionContext) marshalOMetricStatistics2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricStatistics(ctx context.Context, sel ast.SelectionSet, v schema.MetricStatistics) graphql.Marshaler { return ec._MetricStatistics(ctx, sel, &v) } @@ -20332,7 +20340,7 @@ func (ec *executionContext) marshalOMonitoringState2ᚖstring(ctx context.Contex return res } -func (ec *executionContext) marshalONode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐNode(ctx context.Context, sel ast.SelectionSet, v *schema.Node) graphql.Marshaler { +func (ec *executionContext) marshalONode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode(ctx context.Context, sel ast.SelectionSet, v *schema.Node) graphql.Marshaler { if v == nil { return graphql.Null } @@ -20373,7 +20381,7 @@ func (ec *executionContext) unmarshalOPageRequest2ᚖgithubᚗcomᚋClusterCockp return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) unmarshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState(ctx context.Context, v any) (*schema.SchedulerState, error) { +func (ec *executionContext) unmarshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState(ctx context.Context, v any) (*schema.SchedulerState, error) { if v == nil { return nil, nil } @@ -20382,7 +20390,7 @@ func (ec *executionContext) unmarshalOSchedulerState2ᚖgithubᚗcomᚋClusterCo return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSchedulerState(ctx context.Context, sel ast.SelectionSet, v *schema.SchedulerState) graphql.Marshaler { +func (ec *executionContext) marshalOSchedulerState2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState(ctx context.Context, sel ast.SelectionSet, v *schema.SchedulerState) graphql.Marshaler { if v == nil { return graphql.Null } @@ -20392,7 +20400,7 @@ func (ec *executionContext) marshalOSchedulerState2ᚖgithubᚗcomᚋClusterCock return res } -func (ec *executionContext) marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSeriesᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.Series) graphql.Marshaler { +func (ec *executionContext) marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeriesᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.Series) graphql.Marshaler { if v == nil { return graphql.Null } @@ -20419,7 +20427,7 @@ func (ec *executionContext) marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋcc if !isLen1 { defer wg.Done() } - ret[i] = ec.marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐSeries(ctx, sel, v[i]) + ret[i] = ec.marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeries(ctx, sel, v[i]) } if isLen1 { f(i) @@ -20455,7 +20463,7 @@ func (ec *executionContext) marshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCoc return v } -func (ec *executionContext) marshalOStatsSeries2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐStatsSeries(ctx context.Context, sel ast.SelectionSet, v *schema.StatsSeries) graphql.Marshaler { +func (ec *executionContext) marshalOStatsSeries2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐStatsSeries(ctx context.Context, sel ast.SelectionSet, v *schema.StatsSeries) graphql.Marshaler { if v == nil { return graphql.Null } @@ -20562,11 +20570,11 @@ func (ec *executionContext) unmarshalOTimeRange2ᚖgithubᚗcomᚋClusterCockpit return &res, graphql.ErrorOnPath(ctx, err) } -func (ec *executionContext) marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v schema.Unit) graphql.Marshaler { +func (ec *executionContext) marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v schema.Unit) graphql.Marshaler { return ec._Unit(ctx, sel, &v) } -func (ec *executionContext) marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v *schema.Unit) graphql.Marshaler { +func (ec *executionContext) marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v *schema.Unit) graphql.Marshaler { if v == nil { return graphql.Null } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 31ba03ab..06f0ffcf 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -82,6 +82,7 @@ type JobFilter struct { State []schema.JobState `json:"state,omitempty"` MetricStats []*MetricStatItem `json:"metricStats,omitempty"` Shared *string `json:"shared,omitempty"` + Schedule *string `json:"schedule,omitempty"` Node *StringInput `json:"node,omitempty"` } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 32499b8c..11168e80 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -972,12 +972,10 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } // SubCluster returns generated.SubClusterResolver implementation. func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} } -type ( - clusterResolver struct{ *Resolver } - jobResolver struct{ *Resolver } - metricValueResolver struct{ *Resolver } - mutationResolver struct{ *Resolver } - nodeResolver struct{ *Resolver } - queryResolver struct{ *Resolver } - subClusterResolver struct{ *Resolver } -) +type clusterResolver struct{ *Resolver } +type jobResolver struct{ *Resolver } +type metricValueResolver struct{ *Resolver } +type mutationResolver struct{ *Resolver } +type nodeResolver struct{ *Resolver } +type queryResolver struct{ *Resolver } +type subClusterResolver struct{ *Resolver } diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 8c341afb..4655614f 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -143,57 +143,35 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde // Build a sq.SelectBuilder out of a schema.JobFilter. func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder { - if filter.Tags != nil { - // This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query? - query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct() - } + // Primary Key if filter.DbID != nil { dbIDs := make([]string, len(filter.DbID)) copy(dbIDs, filter.DbID) query = query.Where(sq.Eq{"job.id": dbIDs}) } - if filter.JobID != nil { - query = buildStringCondition("job.job_id", filter.JobID, query) - } - if filter.ArrayJobID != nil { - query = query.Where("job.array_job_id = ?", *filter.ArrayJobID) - } - if filter.User != nil { - query = buildStringCondition("job.hpc_user", filter.User, query) - } - if filter.Project != nil { - query = buildStringCondition("job.project", filter.Project, query) - } - if filter.JobName != nil { - query = buildMetaJsonCondition("jobName", filter.JobName, query) - } + // Explicit indices if filter.Cluster != nil { query = buildStringCondition("job.cluster", filter.Cluster, query) } if filter.Partition != nil { query = buildStringCondition("job.cluster_partition", filter.Partition, query) } - if filter.StartTime != nil { - query = buildTimeCondition("job.start_time", filter.StartTime, query) - } - if filter.Duration != nil { - query = buildIntCondition("job.duration", filter.Duration, query) - } - if filter.MinRunningFor != nil { - now := time.Now().Unix() // There does not seam to be a portable way to get the current unix timestamp accross different DBs. - query = query.Where("(job.job_state != 'running' OR (? - job.start_time) > ?)", now, *filter.MinRunningFor) - } - if filter.Shared != nil { - query = query.Where("job.shared = ?", *filter.Shared) - } if filter.State != nil { states := make([]string, len(filter.State)) for i, val := range filter.State { states[i] = string(val) } - query = query.Where(sq.Eq{"job.job_state": states}) } + if filter.Shared != nil { + query = query.Where("job.shared = ?", *filter.Shared) + } + if filter.Project != nil { + query = buildStringCondition("job.project", filter.Project, query) + } + if filter.User != nil { + query = buildStringCondition("job.hpc_user", filter.User, query) + } if filter.NumNodes != nil { query = buildIntCondition("job.num_nodes", filter.NumNodes, query) } @@ -203,17 +181,57 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select if filter.NumHWThreads != nil { query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query) } - if filter.Node != nil { - query = buildResourceJsonCondition("hostname", filter.Node, query) + if filter.ArrayJobID != nil { + query = query.Where("job.array_job_id = ?", *filter.ArrayJobID) + } + if filter.StartTime != nil { + query = buildTimeCondition("job.start_time", filter.StartTime, query) + } + if filter.Duration != nil { + query = buildIntCondition("job.duration", filter.Duration, query) } if filter.Energy != nil { query = buildFloatCondition("job.energy", filter.Energy, query) } + // Indices on Tag Table + if filter.Tags != nil { + // This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query? + query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct() + } + // No explicit Indices + if filter.JobID != nil { + query = buildStringCondition("job.job_id", filter.JobID, query) + } + // Queries Within JSONs if filter.MetricStats != nil { for _, ms := range filter.MetricStats { query = buildFloatJsonCondition(ms.MetricName, ms.Range, query) } } + if filter.Node != nil { + query = buildResourceJsonCondition("hostname", filter.Node, query) + } + if filter.JobName != nil { + query = buildMetaJsonCondition("jobName", filter.JobName, query) + } + if filter.Schedule != nil { + interactiveJobname := "interactive" + if *filter.Schedule == "interactive" { + iFilter := model.StringInput{Eq: &interactiveJobname} + query = buildMetaJsonCondition("jobName", &iFilter, query) + } else if *filter.Schedule == "batch" { + sFilter := model.StringInput{Neq: &interactiveJobname} + query = buildMetaJsonCondition("jobName", &sFilter, query) + } + } + + // Configurable Filter to exclude recently started jobs, see config.go: ShortRunningJobsDuration + if filter.MinRunningFor != nil { + now := time.Now().Unix() + // Only jobs whose start timestamp is more than MinRunningFor seconds in the past + // If a job completed within the configured timeframe, it will still show up after the start_time matches the condition! + query = query.Where(sq.Lt{"job.start_time": (now - int64(*filter.MinRunningFor))}) + } return query } diff --git a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql index 863b50ea..bd465bcb 100644 --- a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql +++ b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql @@ -118,15 +118,13 @@ DROP TABLE lookup_exclusive; DROP TABLE job; -- Deletes All Existing 'job' Indices; Recreate after Renaming ALTER TABLE job_new RENAME TO job; --- Recreate Indices from 08_add-footprint, include new submit_time indices +-- Recreate Indices from 08_add-footprint; include new 'shared' column -- Cluster Filter -CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); -- Cluster Filter Sorting CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_submittime ON job (cluster, submit_time); CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads); @@ -134,44 +132,42 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy); -- Cluster+Partition Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared); + -- Cluster+Partition Filter Sorting CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_submittime ON job (cluster, cluster_partition, submit_time); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy); --- Cluster+Partition+Jobstate Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project); --- Cluster+Partition+Jobstate Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_submittime ON job (cluster, cluster_partition, job_state, submit_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, cluster_partition, job_state, num_hwthreads); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, cluster_partition, job_state, num_acc); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, cluster_partition, job_state, energy); - -- Cluster+JobState Filter -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); -- Cluster+JobState Filter Sorting CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_submittime ON job (cluster, job_state, submit_time); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy); +-- Cluster+Shared Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project); +-- Cluster+Shared Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime ON job (cluster, shared, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration ON job (cluster, shared, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy); + -- User Filter -CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user); -- User Filter Sorting CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); @@ -181,7 +177,6 @@ CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc); CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); -- Project Filter -CREATE INDEX IF NOT EXISTS jobs_project ON job (project); CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); -- Project Filter Sorting CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); @@ -192,10 +187,8 @@ CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc); CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy); -- JobState Filter -CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); -CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); -- JobState Filter Sorting CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration); @@ -204,18 +197,21 @@ CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwt CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc); CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy); +-- Shared Filter +CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project); +-- Shared Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_shared_starttime ON job (shared, start_time); +CREATE INDEX IF NOT EXISTS jobs_shared_duration ON job (shared, duration); +CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc); +CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy); + -- ArrayJob Filter CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); --- Sorting without active filters -CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); -CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); -CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes); -CREATE INDEX IF NOT EXISTS jobs_numhwthreads ON job (num_hwthreads); -CREATE INDEX IF NOT EXISTS jobs_numacc ON job (num_acc); -CREATE INDEX IF NOT EXISTS jobs_energy ON job (energy); - -- Single filters with default starttime sorting CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time); CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time); @@ -223,6 +219,18 @@ CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, st CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time); CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time); +-- Single filters with duration sorting +CREATE INDEX IF NOT EXISTS jobs_starttime_duration ON job (start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_numnodes_duration ON job (num_nodes, duration); +CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, duration); +CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration); +CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration); + +-- Notes: +-- Cluster+Partition+Jobstate Filter: Tested -> Full Array Of Combinations non-required +-- Cluster+JobState+Shared Filter: Tested -> No further timing improvement +-- JobState+Shared Filter: Tested -> No further timing improvement + -- Optimize DB index usage PRAGMA optimize; diff --git a/internal/repository/migrations/sqlite3/10_node-table.up.sql b/internal/repository/migrations/sqlite3/10_node-table.up.sql index 247bceab..7b5b5ac7 100644 --- a/internal/repository/migrations/sqlite3/10_node-table.up.sql +++ b/internal/repository/migrations/sqlite3/10_node-table.up.sql @@ -33,8 +33,6 @@ CREATE INDEX IF NOT EXISTS nodes_cluster_subcluster ON node (cluster, subcluster -- Add NEW Indices For New Node_State Table Fields CREATE INDEX IF NOT EXISTS nodestates_timestamp ON node_state (time_stamp); -CREATE INDEX IF NOT EXISTS nodestates_state ON node_state (node_state); -CREATE INDEX IF NOT EXISTS nodestates_health ON node_state (health_state); CREATE INDEX IF NOT EXISTS nodestates_state_timestamp ON node_state (node_state, time_stamp); CREATE INDEX IF NOT EXISTS nodestates_health_timestamp ON node_state (health_state, time_stamp); CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state); diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 436031ef..e6a79095 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -257,6 +257,12 @@ func buildFilterPresets(query url.Values) map[string]interface{} { if len(query["state"]) != 0 { filterPresets["state"] = query["state"] } + if query.Get("shared") != "" { + filterPresets["shared"] = query.Get("shared") + } + if query.Get("schedule") != "" { + filterPresets["schedule"] = query.Get("schedule") + } if rawtags, ok := query["tag"]; ok { tags := make([]int, len(rawtags)) for i, tid := range rawtags { diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index 7bc877f0..74417015 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -28,7 +28,7 @@ } from "@sveltestrap/sveltestrap"; import Info from "./filters/InfoBox.svelte"; import Cluster from "./filters/Cluster.svelte"; - import JobStates, { allJobStates } from "./filters/JobStates.svelte"; + import JobStates, { allJobStates, mapSharedStates } from "./filters/JobStates.svelte"; import StartTime, { startTimeSelectOptions } from "./filters/StartTime.svelte"; import Duration from "./filters/Duration.svelte"; import Tags from "./filters/Tags.svelte"; @@ -69,6 +69,8 @@ cluster: null, partition: null, states: allJobStates, + shared: "", + schedule: "", startTime: { from: null, to: null, range: ""}, duration: { lessThan: null, @@ -103,6 +105,8 @@ filterPresets.states || filterPresets.state ? [filterPresets.state].flat() : allJobStates, + shared: filterPresets.shared || "", + schedule: filterPresets.schedule || "", startTime: filterPresets.startTime || { from: null, to: null, range: ""}, duration: filterPresets.duration || { lessThan: null, @@ -146,19 +150,39 @@ let items = []; if (filters.dbId.length != 0) items.push({ dbId: filters.dbId }); - if (filters.jobId) - items.push({ jobId: { [filters.jobIdMatch]: filters.jobId } }); - if (filters.arrayJobId != null) - items.push({ arrayJobId: filters.arrayJobId }); - if (filters.jobName) items.push({ jobName: { contains: filters.jobName } }); - if (filters.project) - items.push({ project: { [filters.projectMatch]: filters.project } }); - if (filters.user) - items.push({ user: { [filters.userMatch]: filters.user } }); if (filters.cluster) items.push({ cluster: { eq: filters.cluster } }); if (filters.partition) items.push({ partition: { eq: filters.partition } }); if (filters.states.length != allJobStates?.length) items.push({ state: filters.states }); + if (filters.shared) items.push({ shared: filters.shared }); + if (filters.project) + items.push({ project: { [filters.projectMatch]: filters.project } }); + if (filters.user) + items.push({ user: { [filters.userMatch]: filters.user } }); + if (filters.numNodes.from != null || filters.numNodes.to != null) { + items.push({ + numNodes: { from: filters.numNodes.from, to: filters.numNodes.to }, + }); + } + if (filters.numAccelerators.from != null || filters.numAccelerators.to != null) { + items.push({ + numAccelerators: { + from: filters.numAccelerators.from, + to: filters.numAccelerators.to, + }, + }); + } + if (filters.numHWThreads.from != null || filters.numHWThreads.to != null) { + items.push({ + numHWThreads: { + from: filters.numHWThreads.from, + to: filters.numHWThreads.to, + }, + }); + } + if (filters.arrayJobId != null) + items.push({ arrayJobId: filters.arrayJobId }); + if (filters.tags.length != 0) items.push({ tags: filters.tags }); if (filters.startTime.from || filters.startTime.to) items.push({ startTime: { from: filters.startTime.from, to: filters.startTime.to }, @@ -175,36 +199,17 @@ items.push({ duration: { from: 0, to: filters.duration.lessThan } }); if (filters.duration.moreThan) items.push({ duration: { from: filters.duration.moreThan, to: 604800 } }); // 7 days to include special jobs with long runtimes - if (filters.tags.length != 0) items.push({ tags: filters.tags }); - if (filters.numNodes.from != null || filters.numNodes.to != null) { - items.push({ - numNodes: { from: filters.numNodes.from, to: filters.numNodes.to }, - }); - } - if (filters.numHWThreads.from != null || filters.numHWThreads.to != null) { - items.push({ - numHWThreads: { - from: filters.numHWThreads.from, - to: filters.numHWThreads.to, - }, - }); - } - if (filters.numAccelerators.from != null || filters.numAccelerators.to != null) { - items.push({ - numAccelerators: { - from: filters.numAccelerators.from, - to: filters.numAccelerators.to, - }, - }); - } - if (filters.node) items.push({ node: { [filters.nodeMatch]: filters.node } }); if (filters.energy.from || filters.energy.to) items.push({ energy: { from: filters.energy.from, to: filters.energy.to }, }); + if (filters.jobId) + items.push({ jobId: { [filters.jobIdMatch]: filters.jobId } }); if (filters.stats.length != 0) items.push({ metricStats: filters.stats.map((st) => { return { metricName: st.field, range: { from: st.from, to: st.to }} }) }); - + if (filters.node) items.push({ node: { [filters.nodeMatch]: filters.node } }); + if (filters.jobName) items.push({ jobName: { contains: filters.jobName } }); + if (filters.schedule) items.push({ schedule: filters.schedule }); applyFilters({ filters: items }); changeURL(); return items; @@ -248,6 +253,8 @@ if (filters.partition) opts.push(`partition=${filters.partition}`); if (filters.states.length != allJobStates?.length) for (let state of filters.states) opts.push(`state=${state}`); + if (filters.shared) opts.push(`shared=${filters.shared}`); + if (filters.schedule) opts.push(`schedule=${filters.schedule}`); if (filters.startTime.from && filters.startTime.to) opts.push( `startTime=${dateToUnixEpoch(filters.startTime.from)}-${dateToUnixEpoch(filters.startTime.to)}`, @@ -366,6 +373,23 @@ {#if filters.states.length != allJobStates?.length} (isJobStatesOpen = true)}> {filters.states.join(", ")} + {#if filters.shared && !filters.schedule} + ({mapSharedStates[filters.shared]}) + {:else if filters.schedule && !filters.shared} + ({filters.schedule.charAt(0).toUpperCase() + filters.schedule?.slice(1)}) + {:else if (filters.shared && filters.schedule)} + ({[mapSharedStates[filters.shared], (filters.schedule.charAt(0).toUpperCase() + filters.schedule.slice(1))].join(", ")}) + {/if} + + {:else if (filters.shared || filters.schedule)} + (isJobStatesOpen = true)}> + {#if filters.shared && !filters.schedule} + {mapSharedStates[filters.shared]} + {:else if filters.schedule && !filters.shared} + {filters.schedule.charAt(0).toUpperCase() + filters.schedule?.slice(1)} + {:else if (filters.shared && filters.schedule)} + {[mapSharedStates[filters.shared], (filters.schedule.charAt(0).toUpperCase() + filters.schedule.slice(1))].join(", ")} + {/if} {/if} @@ -468,6 +492,8 @@ updateFilters(filter)} /> diff --git a/web/frontend/src/generic/filters/JobStates.svelte b/web/frontend/src/generic/filters/JobStates.svelte index ba4168f7..dc622a20 100644 --- a/web/frontend/src/generic/filters/JobStates.svelte +++ b/web/frontend/src/generic/filters/JobStates.svelte @@ -4,23 +4,35 @@ Properties: - `isOpen Bool?`: Is this filter component opened [Bindable, Default: false] - `presetStates [String]?`: The latest selected filter state [Default: [...allJobStates]] + - `presetShared String?`: The latest selected filter shared [Default: ""] + - `presetShedule String?`: The latest selected filter schedule [Default: ""] - `setFilter Func`: The callback function to apply current filter selection Exported: - `const allJobStates [String]`: List of all available job states used in cc-backend + - `const mapSharedStates {String:String}`: Object of all available shared states used in cc-backend with label --> @@ -60,10 +86,26 @@ name="flavours" value={state} /> - {state} + {state.charAt(0).toUpperCase() + state.slice(1)} {/each} +
    + + +
    Resource Sharing
    + + {#each allSharedStates as shared} + + {/each} + + +
    Processing Type
    + + + + +     - + {#if pendingStates.length != 0} + + {:else} + + {/if} From fb8db3c3aed9527e01c526f3943b5d5abdafed01 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 07:37:31 +0100 Subject: [PATCH 076/341] Add query which node metric data needs to be retained --- go.mod | 1 + go.sum | 2 ++ internal/repository/job.go | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/go.mod b/go.mod index 9cb82fbc..808b2e7a 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/urfave/cli/v3 v3.6.1 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect + github.com/xtgo/set v1.0.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect diff --git a/go.sum b/go.sum index 8d3904ae..39571309 100644 --- a/go.sum +++ b/go.sum @@ -318,6 +318,8 @@ github.com/vektah/gqlparser/v2 v2.5.31 h1:YhWGA1mfTjID7qJhd1+Vxhpk5HTgydrGU9IgkW github.com/vektah/gqlparser/v2 v2.5.31/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/xtgo/set v1.0.0 h1:6BCNBRv3ORNDQ7fyoJXRv+tstJz3m1JVFQErfeZz2pY= +github.com/xtgo/set v1.0.0/go.mod h1:d3NHzGzSa0NmB2NhFyECA+QdRp29oEn2xbT+TpeFoM8= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/internal/repository/job.go b/internal/repository/job.go index 99970ce1..1ff8047e 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -76,6 +76,7 @@ import ( "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" + "github.com/xtgo/set" ) var ( @@ -772,3 +773,63 @@ func (r *JobRepository) UpdateFootprint( return stmt.Set("footprint", string(rawFootprint)), nil } + +func (r *JobRepository) GetUsedNodes(ts uint64) map[string][]string { + q := sq.Select("job.cluster", "job.resources").From("job"). + Where("job.start_time < ?", ts). + Where(sq.Eq{"job.job_state": "running"}) + + rows, err := q.RunWith(r.stmtCache).Query() + if err != nil { + queryString, queryVars, _ := q.ToSql() + cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err) + return nil + } + defer rows.Close() + + // Use a map of sets for efficient deduplication + nodeSet := make(map[string]map[string]struct{}) + + var ( + cluster string + rawResources []byte + resources []*schema.Resource + ) + + for rows.Next() { + if err := rows.Scan(&cluster, &rawResources); err != nil { + cclog.Warnf("Error scanning job row in GetUsedNodes: %v", err) + continue + } + + if err := json.Unmarshal(rawResources, &resources); err != nil { + cclog.Warnf("Error unmarshaling resources for cluster %s: %v", cluster, err) + continue + } + + if _, ok := nodeSet[cluster]; !ok { + nodeSet[cluster] = make(map[string]struct{}) + } + + for _, res := range resources { + nodeSet[cluster][res.Hostname] = struct{}{} + } + } + + if err := rows.Err(); err != nil { + cclog.Errorf("Error iterating rows in GetUsedNodes: %v", err) + } + + nodeList := make(map[string][]string) + for cluster, nodes := range nodeSet { + // Convert map keys to slice + list := make([]string, 0, len(nodes)) + for node := range nodes { + list = append(list, node) + } + // set.Strings sorts the slice and ensures uniqueness + nodeList[cluster] = set.Strings(list) + } + + return nodeList +} From 71b75eea0e53b99b27b18ed78f5600ce4e199b6f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 08:49:55 +0100 Subject: [PATCH 077/341] Improve GetUsedNodes function --- internal/repository/job.go | 41 ++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 1ff8047e..78e1f3fe 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -66,6 +66,7 @@ import ( "fmt" "maps" "math" + "sort" "strconv" "sync" "time" @@ -76,7 +77,6 @@ import ( "github.com/ClusterCockpit/cc-lib/v2/schema" sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" - "github.com/xtgo/set" ) var ( @@ -774,7 +774,16 @@ func (r *JobRepository) UpdateFootprint( return stmt.Set("footprint", string(rawFootprint)), nil } -func (r *JobRepository) GetUsedNodes(ts uint64) map[string][]string { +// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames +// that are currently in use by jobs that started before the given timestamp and +// are still in running state. +// +// The timestamp parameter (ts) is compared against job.start_time to find +// relevant jobs. Returns an error if the database query fails or row iteration +// encounters errors. Individual row parsing errors are logged but don't fail +// the entire operation. +func (r *JobRepository) GetUsedNodes(ts uint64) (map[string][]string, error) { + // Note: Query expects index on (job_state, start_time) for optimal performance q := sq.Select("job.cluster", "job.resources").From("job"). Where("job.start_time < ?", ts). Where(sq.Eq{"job.job_state": "running"}) @@ -782,8 +791,7 @@ func (r *JobRepository) GetUsedNodes(ts uint64) map[string][]string { rows, err := q.RunWith(r.stmtCache).Query() if err != nil { queryString, queryVars, _ := q.ToSql() - cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err) - return nil + return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err) } defer rows.Close() @@ -794,16 +802,25 @@ func (r *JobRepository) GetUsedNodes(ts uint64) map[string][]string { cluster string rawResources []byte resources []*schema.Resource + skippedRows int ) for rows.Next() { if err := rows.Scan(&cluster, &rawResources); err != nil { cclog.Warnf("Error scanning job row in GetUsedNodes: %v", err) + skippedRows++ continue } + resources = resources[:0] // Clear slice, keep capacity if err := json.Unmarshal(rawResources, &resources); err != nil { cclog.Warnf("Error unmarshaling resources for cluster %s: %v", cluster, err) + skippedRows++ + continue + } + + if len(resources) == 0 { + cclog.Debugf("Job in cluster %s has no resources", cluster) continue } @@ -817,19 +834,23 @@ func (r *JobRepository) GetUsedNodes(ts uint64) map[string][]string { } if err := rows.Err(); err != nil { - cclog.Errorf("Error iterating rows in GetUsedNodes: %v", err) + return nil, fmt.Errorf("error iterating rows: %w", err) } - nodeList := make(map[string][]string) + if skippedRows > 0 { + cclog.Warnf("GetUsedNodes: Skipped %d rows due to parsing errors", skippedRows) + } + + // Convert sets to sorted slices + nodeList := make(map[string][]string, len(nodeSet)) for cluster, nodes := range nodeSet { - // Convert map keys to slice list := make([]string, 0, len(nodes)) for node := range nodes { list = append(list, node) } - // set.Strings sorts the slice and ensures uniqueness - nodeList[cluster] = set.Strings(list) + sort.Strings(list) + nodeList[cluster] = list } - return nodeList + return nodeList, nil } From 6cf59043a38ce70963e5a8b24046cd0c9820708c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 08:59:27 +0100 Subject: [PATCH 078/341] Review and improve, add documentation --- internal/repository/jobQuery.go | 128 +++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 42 deletions(-) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 4655614f..745fa32d 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -2,6 +2,10 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + +// Package repository provides job query functionality with filtering, pagination, +// and security controls. This file contains the main query builders and security +// checks for job retrieval operations. package repository import ( @@ -19,6 +23,22 @@ import ( sq "github.com/Masterminds/squirrel" ) +const ( + // Default initial capacity for job result slices + defaultJobsCapacity = 50 +) + +// QueryJobs retrieves jobs from the database with optional filtering, pagination, +// and sorting. Security controls are automatically applied based on the user context. +// +// Parameters: +// - ctx: Context containing user authentication information +// - filters: Optional job filters (cluster, state, user, time ranges, etc.) +// - page: Optional pagination parameters (page number and items per page) +// - order: Optional sorting specification (column or footprint field) +// +// Returns a slice of jobs matching the criteria, or an error if the query fails. +// The function enforces role-based access control through SecurityCheck. func (r *JobRepository) QueryJobs( ctx context.Context, filters []*model.JobFilter, @@ -33,18 +53,16 @@ func (r *JobRepository) QueryJobs( if order != nil { field := toSnakeCase(order.Field) if order.Type == "col" { - // "col": Fixed column name query switch order.Order { case model.SortDirectionEnumAsc: query = query.OrderBy(fmt.Sprintf("job.%s ASC", field)) case model.SortDirectionEnumDesc: query = query.OrderBy(fmt.Sprintf("job.%s DESC", field)) default: - return nil, errors.New("REPOSITORY/QUERY > invalid sorting order for column") + return nil, errors.New("invalid sorting order for column") } } else { - // "foot": Order by footprint JSON field values - // Verify and Search Only in Valid Jsons + // Order by footprint JSON field values query = query.Where("JSON_VALID(meta_data)") switch order.Order { case model.SortDirectionEnumAsc: @@ -52,7 +70,7 @@ func (r *JobRepository) QueryJobs( case model.SortDirectionEnumDesc: query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field)) default: - return nil, errors.New("REPOSITORY/QUERY > invalid sorting order for footprint") + return nil, errors.New("invalid sorting order for footprint") } } } @@ -69,29 +87,35 @@ func (r *JobRepository) QueryJobs( rows, err := query.RunWith(r.stmtCache).Query() if err != nil { queryString, queryVars, _ := query.ToSql() - cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err) - return nil, err + return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err) } + defer rows.Close() - jobs := make([]*schema.Job, 0, 50) + jobs := make([]*schema.Job, 0, defaultJobsCapacity) for rows.Next() { job, err := scanJob(rows) if err != nil { - rows.Close() - cclog.Warn("Error while scanning rows (Jobs)") - return nil, err + cclog.Warnf("Error scanning job row: %v", err) + return nil, fmt.Errorf("failed to scan job row: %w", err) } jobs = append(jobs, job) } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating job rows: %w", err) + } + return jobs, nil } +// CountJobs returns the total number of jobs matching the given filters. +// Security controls are automatically applied based on the user context. +// Uses DISTINCT count to handle tag filters correctly (jobs may appear multiple +// times when joined with the tag table). func (r *JobRepository) CountJobs( ctx context.Context, filters []*model.JobFilter, ) (int, error) { - // DISTICT count for tags filters, does not affect other queries query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job")) if qerr != nil { return 0, qerr @@ -103,12 +127,22 @@ func (r *JobRepository) CountJobs( var count int if err := query.RunWith(r.DB).Scan(&count); err != nil { - return 0, err + return 0, fmt.Errorf("failed to count jobs: %w", err) } return count, nil } +// SecurityCheckWithUser applies role-based access control filters to a job query +// based on the provided user's roles and permissions. +// +// Access rules by role: +// - API role (exclusive): Full access to all jobs +// - Admin/Support roles: Full access to all jobs +// - Manager role: Access to jobs in managed projects plus own jobs +// - User role: Access only to own jobs +// +// Returns an error if the user is nil or has no recognized roles. func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) { if user == nil { var qnil sq.SelectBuilder @@ -116,32 +150,35 @@ func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.Select } switch { - case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : All jobs + case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): return query, nil - case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : All jobs + case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): return query, nil - case user.HasRole(schema.RoleManager): // Manager : Add filter for managed projects' jobs only + personal jobs + case user.HasRole(schema.RoleManager): if len(user.Projects) != 0 { return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil - } else { - cclog.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username) - return query.Where("job.hpc_user = ?", user.Username), nil } - case user.HasRole(schema.RoleUser): // User : Only personal jobs + cclog.Debugf("Manager '%s' has no assigned projects, restricting to personal jobs", user.Username) return query.Where("job.hpc_user = ?", user.Username), nil - default: // No known Role, return error + case user.HasRole(schema.RoleUser): + return query.Where("job.hpc_user = ?", user.Username), nil + default: var qnil sq.SelectBuilder return qnil, fmt.Errorf("user has no or unknown roles") } } +// SecurityCheck extracts the user from the context and applies role-based access +// control filters to the query. This is a convenience wrapper around SecurityCheckWithUser. func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { user := GetUserFromContext(ctx) - return SecurityCheckWithUser(user, query) } -// Build a sq.SelectBuilder out of a schema.JobFilter. +// BuildWhereClause constructs SQL WHERE conditions from a JobFilter and applies +// them to the query. Supports filtering by job properties (cluster, state, user), +// time ranges, resource usage, tags, and JSON field searches in meta_data, +// footprint, and resources columns. func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder { // Primary Key if filter.DbID != nil { @@ -205,23 +242,24 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select // Queries Within JSONs if filter.MetricStats != nil { for _, ms := range filter.MetricStats { - query = buildFloatJsonCondition(ms.MetricName, ms.Range, query) + query = buildFloatJSONCondition(ms.MetricName, ms.Range, query) } } if filter.Node != nil { - query = buildResourceJsonCondition("hostname", filter.Node, query) + query = buildResourceJSONCondition("hostname", filter.Node, query) } if filter.JobName != nil { - query = buildMetaJsonCondition("jobName", filter.JobName, query) + query = buildMetaJSONCondition("jobName", filter.JobName, query) } if filter.Schedule != nil { interactiveJobname := "interactive" - if *filter.Schedule == "interactive" { + switch *filter.Schedule { + case "interactive": iFilter := model.StringInput{Eq: &interactiveJobname} - query = buildMetaJsonCondition("jobName", &iFilter, query) - } else if *filter.Schedule == "batch" { + query = buildMetaJSONCondition("jobName", &iFilter, query) + case "batch": sFilter := model.StringInput{Neq: &interactiveJobname} - query = buildMetaJsonCondition("jobName", &sFilter, query) + query = buildMetaJSONCondition("jobName", &sFilter, query) } } @@ -235,14 +273,18 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select return query } +// buildIntCondition creates a BETWEEN clause for integer range filters. func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) } +// buildFloatCondition creates a BETWEEN clause for float range filters. func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) } +// buildTimeCondition creates time range filters supporting absolute timestamps, +// relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges. func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder { if cond.From != nil && cond.To != nil { return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix()) @@ -272,12 +314,14 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui } } -func buildFloatJsonCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { - // Verify and Search Only in Valid Jsons +// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column. +func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(footprint)") return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To) } +// buildStringCondition creates filters for string fields supporting equality, +// inequality, prefix, suffix, substring, and IN list matching. func buildStringCondition(field string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { if cond.Eq != nil { return query.Where(field+" = ?", *cond.Eq) @@ -302,10 +346,9 @@ func buildStringCondition(field string, cond *model.StringInput, query sq.Select return query } -func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { - // Verify and Search Only in Valid Jsons +// buildMetaJSONCondition creates filters on fields within the meta_data JSON column. +func buildMetaJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(meta_data)") - // add "AND" Sql query Block for field match if cond.Eq != nil { return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq) } @@ -324,10 +367,10 @@ func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq. return query } -func buildResourceJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { - // Verify and Search Only in Valid Jsons +// buildResourceJSONCondition creates filters on fields within the resources JSON array column. +// Uses json_each to search within array elements. +func buildResourceJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(resources)") - // add "AND" Sql query Block for field match if cond.Eq != nil { return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") = ?)", *cond.Eq) } @@ -351,15 +394,16 @@ var ( matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") ) +// toSnakeCase converts camelCase strings to snake_case for SQL column names. +// Includes security checks to prevent SQL injection attempts. +// Panics if potentially dangerous characters are detected. func toSnakeCase(str string) string { for _, c := range str { - if c == '\'' || c == '\\' { - cclog.Panic("toSnakeCase() attack vector!") + if c == '\'' || c == '\\' || c == '"' || c == ';' || c == '-' || c == ' ' { + cclog.Panicf("toSnakeCase: potentially dangerous character detected in input: %q", str) } } - str = strings.ReplaceAll(str, "'", "") - str = strings.ReplaceAll(str, "\\", "") snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}") snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}") return strings.ToLower(snake) From 9e542dc2006c2dbe3fc4c7206254d2590affdfd8 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 09:26:03 +0100 Subject: [PATCH 079/341] Review and improve, add documentation --- internal/repository/jobFind.go | 83 +++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index ff2c27aa..8f6daeb4 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -27,6 +27,10 @@ func (r *JobRepository) Find( cluster *string, startTime *int64, ) (*schema.Job, error) { + if jobID == nil { + return nil, fmt.Errorf("jobID cannot be nil") + } + start := time.Now() q := sq.Select(jobColumns...).From("job"). Where("job.job_id = ?", *jobID) @@ -38,17 +42,27 @@ func (r *JobRepository) Find( q = q.Where("job.start_time = ?", *startTime) } - q = q.OrderBy("job.id DESC") // always use newest matching job by db id if more than one match + q = q.OrderBy("job.id DESC").Limit(1) // always use newest matching job by db id if more than one match cclog.Debugf("Timer Find %s", time.Since(start)) return scanJob(q.RunWith(r.stmtCache).QueryRow()) } +// FindCached executes a SQL query to find a specific batch job from the job_cache table. +// The job is queried using the batch job id, and optionally filtered by cluster name +// and start time (UNIX epoch time seconds). This method uses cached job data which +// may be stale but provides faster access than Find(). +// It returns a pointer to a schema.Job data structure and an error variable. +// To check if no job was found test err == sql.ErrNoRows func (r *JobRepository) FindCached( jobID *int64, cluster *string, startTime *int64, ) (*schema.Job, error) { + if jobID == nil { + return nil, fmt.Errorf("jobID cannot be nil") + } + q := sq.Select(jobCacheColumns...).From("job_cache"). Where("job_cache.job_id = ?", *jobID) @@ -59,7 +73,7 @@ func (r *JobRepository) FindCached( q = q.Where("job_cache.start_time = ?", *startTime) } - q = q.OrderBy("job_cache.id DESC") // always use newest matching job by db id if more than one match + q = q.OrderBy("job_cache.id DESC").Limit(1) // always use newest matching job by db id if more than one match return scanJob(q.RunWith(r.stmtCache).QueryRow()) } @@ -74,6 +88,10 @@ func (r *JobRepository) FindAll( cluster *string, startTime *int64, ) ([]*schema.Job, error) { + if jobID == nil { + return nil, fmt.Errorf("jobID cannot be nil") + } + start := time.Now() q := sq.Select(jobColumns...).From("job"). Where("job.job_id = ?", *jobID) @@ -87,8 +105,8 @@ func (r *JobRepository) FindAll( rows, err := q.RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running FindAll query for jobID=%d: %v", *jobID, err) + return nil, fmt.Errorf("failed to execute FindAll query: %w", err) } defer rows.Close() @@ -96,8 +114,8 @@ func (r *JobRepository) FindAll( for rows.Next() { job, err := scanJob(rows) if err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in FindAll: %v", err) + return nil, fmt.Errorf("failed to scan job row: %w", err) } jobs = append(jobs, job) } @@ -120,8 +138,8 @@ func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) { rows, err := query.RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running GetJobList query (limit=%d, offset=%d): %v", limit, offset, err) + return nil, fmt.Errorf("failed to execute GetJobList query: %w", err) } defer rows.Close() @@ -130,8 +148,8 @@ func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) { var id int64 err := rows.Scan(&id) if err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in GetJobList: %v", err) + return nil, fmt.Errorf("failed to scan job ID: %w", err) } jl = append(jl, id) } @@ -202,10 +220,10 @@ func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// IsJobOwner executes a SQL query to find a specific batch job. -// The job is queried using the slurm id,a username and the cluster. -// It returns a bool. -// If job was found, user is owner: test err != sql.ErrNoRows +// IsJobOwner checks if the specified user owns the batch job identified by jobID, +// startTime, and cluster. Returns true if the user is the owner, false otherwise. +// This method does not return errors; it returns false for both non-existent jobs +// and jobs owned by other users. func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool { q := sq.Select("id"). From("job"). @@ -215,6 +233,9 @@ func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cl Where("job.start_time = ?", startTime) _, err := scanJob(q.RunWith(r.stmtCache).QueryRow()) + if err != nil && err != sql.ErrNoRows { + cclog.Warnf("IsJobOwner: unexpected error for jobID=%d, user=%s, cluster=%s: %v", jobID, user, cluster, err) + } return err != sql.ErrNoRows } @@ -232,6 +253,11 @@ func (r *JobRepository) FindConcurrentJobs( } query = query.Where("cluster = ?", job.Cluster) + + if len(job.Resources) == 0 { + return nil, fmt.Errorf("job has no resources defined") + } + var startTime int64 var stopTime int64 @@ -244,10 +270,15 @@ func (r *JobRepository) FindConcurrentJobs( stopTime = startTime + int64(job.Duration) } - // Add 200s overlap for jobs start time at the end - startTimeTail := startTime + 10 - stopTimeTail := stopTime - 200 - startTimeFront := startTime + 200 + // Time buffer constants for finding overlapping jobs + // overlapBufferStart: 10s grace period at job start to catch jobs starting just after + // overlapBufferEnd: 200s buffer at job end to account for scheduling/cleanup overlap + const overlapBufferStart = 10 + const overlapBufferEnd = 200 + + startTimeTail := startTime + overlapBufferStart + stopTimeTail := stopTime - overlapBufferEnd + startTimeFront := startTime + overlapBufferEnd queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)", "running", startTimeTail, stopTimeTail, startTime) @@ -261,8 +292,8 @@ func (r *JobRepository) FindConcurrentJobs( rows, err := query.RunWith(r.stmtCache).Query() if err != nil { - cclog.Errorf("Error while running query: %v", err) - return nil, err + cclog.Errorf("Error while running concurrent jobs query: %v", err) + return nil, fmt.Errorf("failed to execute concurrent jobs query: %w", err) } defer rows.Close() @@ -273,8 +304,8 @@ func (r *JobRepository) FindConcurrentJobs( var id, jobID, startTime sql.NullInt64 if err = rows.Scan(&id, &jobID, &startTime); err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning concurrent job rows: %v", err) + return nil, fmt.Errorf("failed to scan concurrent job row: %w", err) } if id.Valid { @@ -289,8 +320,8 @@ func (r *JobRepository) FindConcurrentJobs( rows, err = queryRunning.RunWith(r.stmtCache).Query() if err != nil { - cclog.Errorf("Error while running query: %v", err) - return nil, err + cclog.Errorf("Error while running concurrent running jobs query: %v", err) + return nil, fmt.Errorf("failed to execute concurrent running jobs query: %w", err) } defer rows.Close() @@ -298,8 +329,8 @@ func (r *JobRepository) FindConcurrentJobs( var id, jobID, startTime sql.NullInt64 if err := rows.Scan(&id, &jobID, &startTime); err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning running concurrent job rows: %v", err) + return nil, fmt.Errorf("failed to scan running concurrent job row: %w", err) } if id.Valid { From b2f870e3c04a41b4fcca658afa59677d36a23146 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 10:08:06 +0100 Subject: [PATCH 080/341] Convert nodestate nats API to influx line protocol payload. Review and add doc comments. Improve and extend tests --- internal/api/nats.go | 143 +++++++++++++++++++++++++++----- internal/api/nats_test.go | 170 +++++++++++++++++++++++++++----------- 2 files changed, 243 insertions(+), 70 deletions(-) diff --git a/internal/api/nats.go b/internal/api/nats.go index efd04406..48c6449b 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -6,7 +6,6 @@ package api import ( - "bytes" "database/sql" "encoding/json" "strings" @@ -26,7 +25,40 @@ import ( ) // NatsAPI provides NATS subscription-based handlers for Job and Node operations. -// It mirrors the functionality of the REST API but uses NATS messaging. +// It mirrors the functionality of the REST API but uses NATS messaging with +// InfluxDB line protocol as the message format. +// +// # Message Format +// +// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) +// with the following structure: +// +// measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp +// +// # Job Events +// +// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations: +// +// job,function=start_job event="{...JSON payload...}" +// job,function=stop_job event="{...JSON payload...}" +// +// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure. +// +// Example job start message: +// +// job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000 +// +// # Node State Events +// +// Node state updates use the "nodestate" measurement with cluster information: +// +// nodestate event="{...JSON payload...}" +// +// The JSON payload follows the UpdateNodeStatesRequest structure. +// +// Example node state message: +// +// nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000 type NatsAPI struct { JobRepository *repository.JobRepository // RepositoryMutex protects job creation operations from race conditions @@ -67,10 +99,12 @@ func (api *NatsAPI) StartSubscriptions() error { return nil } +// processJobEvent routes job event messages to the appropriate handler based on the "function" tag. +// Validates that required tags and fields are present before processing. func (api *NatsAPI) processJobEvent(msg lp.CCMessage) { function, ok := msg.GetTag("function") if !ok { - cclog.Errorf("Job event is missing tag 'function': %+v", msg) + cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name()) return } @@ -78,43 +112,66 @@ func (api *NatsAPI) processJobEvent(msg lp.CCMessage) { case "start_job": v, ok := msg.GetEventValue() if !ok { - cclog.Errorf("Job event is missing event value: %+v", msg) + cclog.Errorf("Job start event is missing event field with JSON payload") + return } api.handleStartJob(v) case "stop_job": v, ok := msg.GetEventValue() if !ok { - cclog.Errorf("Job event is missing event value: %+v", msg) + cclog.Errorf("Job stop event is missing event field with JSON payload") + return } api.handleStopJob(v) + default: - cclog.Warnf("Unimplemented job event: %+v", msg) + cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function) } } +// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol. +// The message must be in line protocol format with measurement="job" and include: +// - tag "function" with value "start_job" or "stop_job" +// - field "event" containing JSON payload (schema.Job or StopJobAPIRequest) +// +// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000 func (api *NatsAPI) handleJobEvent(subject string, data []byte) { + if len(data) == 0 { + cclog.Warnf("NATS %s: received empty message", subject) + return + } + d := influx.NewDecoderWithBytes(data) for d.Next() { m, err := receivers.DecodeInfluxMessage(d) if err != nil { - cclog.Errorf("NATS %s: Failed to decode message: %v", subject, err) + cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err) return } - if m.IsEvent() { - if m.Name() == "job" { - api.processJobEvent(m) - } + if !m.IsEvent() { + cclog.Warnf("NATS %s: received non-event message, skipping", subject) + continue } + if m.Name() == "job" { + api.processJobEvent(m) + } else { + cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name()) + } } } // handleStartJob processes job start messages received via NATS. -// Expected JSON payload follows the schema.Job structure. +// The payload parameter contains JSON following the schema.Job structure. +// Jobs are validated, checked for duplicates, and inserted into the database. func (api *NatsAPI) handleStartJob(payload string) { + if payload == "" { + cclog.Error("NATS start job: payload is empty") + return + } req := schema.Job{ Shared: "none", MonitoringStatus: schema.MonitoringStatusRunningOrArchiving, @@ -173,8 +230,13 @@ func (api *NatsAPI) handleStartJob(payload string) { } // handleStopJob processes job stop messages received via NATS. -// Expected JSON payload follows the StopJobAPIRequest structure. +// The payload parameter contains JSON following the StopJobAPIRequest structure. +// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled. func (api *NatsAPI) handleStopJob(payload string) { + if payload == "" { + cclog.Error("NATS stop job: payload is empty") + return + } var req StopJobAPIRequest dec := json.NewDecoder(strings.NewReader(payload)) @@ -243,15 +305,21 @@ func (api *NatsAPI) handleStopJob(payload string) { archiver.TriggerArchiving(job) } -// handleNodeState processes node state update messages received via NATS. -// Expected JSON payload follows the UpdateNodeStatesRequest structure. -func (api *NatsAPI) handleNodeState(subject string, data []byte) { +// processNodestateEvent extracts and processes node state data from the InfluxDB message. +// Updates node states in the repository for all nodes in the payload. +func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) { + v, ok := msg.GetEventValue() + if !ok { + cclog.Errorf("Nodestate event is missing event field with JSON payload") + return + } + var req UpdateNodeStatesRequest - dec := json.NewDecoder(bytes.NewReader(data)) + dec := json.NewDecoder(strings.NewReader(v)) dec.DisallowUnknownFields() if err := dec.Decode(&req); err != nil { - cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + cclog.Errorf("NATS nodestate: parsing request failed: %v", err) return } @@ -270,10 +338,43 @@ func (api *NatsAPI) handleNodeState(subject string, data []byte) { } if err := repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState); err != nil { - cclog.Errorf("NATS %s: updating node state for %s on %s failed: %v", - subject, node.Hostname, req.Cluster, err) + cclog.Errorf("NATS nodestate: updating node state for %s on %s failed: %v", + node.Hostname, req.Cluster, err) } } - cclog.Debugf("NATS %s: updated %d node states for cluster %s", subject, len(req.Nodes), req.Cluster) + cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster) +} + +// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol. +// The message must be in line protocol format with measurement="nodestate" and include: +// - field "event" containing JSON payload (UpdateNodeStatesRequest) +// +// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000 +func (api *NatsAPI) handleNodeState(subject string, data []byte) { + if len(data) == 0 { + cclog.Warnf("NATS %s: received empty message", subject) + return + } + + d := influx.NewDecoderWithBytes(data) + + for d.Next() { + m, err := receivers.DecodeInfluxMessage(d) + if err != nil { + cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err) + return + } + + if !m.IsEvent() { + cclog.Warnf("NATS %s: received non-event message, skipping", subject) + continue + } + + if m.Name() == "nodestate" { + api.processNodestateEvent(m) + } else { + cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name()) + } + } } diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index 319668bb..4b1431cb 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -603,25 +603,13 @@ func TestNatsHandleNodeState(t *testing.T) { tests := []struct { name string - payload string + data []byte expectError bool validateFn func(t *testing.T) }{ { - name: "valid node state update", - payload: `{ - "cluster": "testcluster", - "nodes": [ - { - "hostname": "host123", - "states": ["allocated"], - "cpusAllocated": 8, - "memoryAllocated": 16384, - "gpusAllocated": 0, - "jobsRunning": 1 - } - ] - }`, + name: "valid node state update", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"allocated\"],\"cpusAllocated\":8,\"memoryAllocated\":16384,\"gpusAllocated\":0,\"jobsRunning\":1}]}" 1234567890000000000`), expectError: false, validateFn: func(t *testing.T) { // In a full test, we would verify the node state was updated in the database @@ -629,51 +617,35 @@ func TestNatsHandleNodeState(t *testing.T) { }, }, { - name: "multiple nodes", - payload: `{ - "cluster": "testcluster", - "nodes": [ - { - "hostname": "host123", - "states": ["idle"], - "cpusAllocated": 0, - "memoryAllocated": 0, - "gpusAllocated": 0, - "jobsRunning": 0 - }, - { - "hostname": "host124", - "states": ["allocated"], - "cpusAllocated": 4, - "memoryAllocated": 8192, - "gpusAllocated": 1, - "jobsRunning": 1 - } - ] - }`, + name: "multiple nodes", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"host124\",\"states\":[\"allocated\"],\"cpusAllocated\":4,\"memoryAllocated\":8192,\"gpusAllocated\":1,\"jobsRunning\":1}]}" 1234567890000000000`), expectError: false, }, { - name: "invalid JSON", - payload: `{ - "cluster": "testcluster", - "nodes": "not an array" - }`, + name: "invalid JSON in event field", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":\"not an array\"}" 1234567890000000000`), expectError: true, }, { - name: "empty nodes array", - payload: `{ - "cluster": "testcluster", - "nodes": [] - }`, + name: "empty nodes array", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1234567890000000000`), expectError: false, // Empty array should not cause error }, + { + name: "invalid line protocol format", + data: []byte(`invalid line protocol format`), + expectError: true, + }, + { + name: "empty data", + data: []byte(``), + expectError: false, // Should be handled gracefully with warning + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - natsAPI.handleNodeState("test.subject", []byte(tt.payload)) + natsAPI.handleNodeState("test.subject", tt.data) // Allow some time for async operations time.Sleep(50 * time.Millisecond) @@ -789,7 +761,7 @@ func TestNatsHandleJobEvent(t *testing.T) { }{ { name: "valid influx line protocol", - data: []byte(`job,function=start_job event="{\"jobId\":4001,\"user\":\"testuser\",\"project\":\"testproj\",\"cluster\":\"testcluster\",\"partition\":\"main\",\"walltime\":3600,\"numNodes\":1,\"numHwthreads\":8,\"numAcc\":0,\"shared\":\"none\",\"monitoringStatus\":1,\"smt\":1,\"resources\":[{\"hostname\":\"host123\",\"hwthreads\":[0,1,2,3]}],\"startTime\":1234567890}"`), + data: []byte(`job,function=start_job event="{\"jobId\":4001,\"user\":\"testuser\",\"project\":\"testproj\",\"cluster\":\"testcluster\",\"partition\":\"main\",\"walltime\":3600,\"numNodes\":1,\"numHwthreads\":8,\"numAcc\":0,\"shared\":\"none\",\"monitoringStatus\":1,\"smt\":1,\"resources\":[{\"hostname\":\"host123\",\"hwthreads\":[0,1,2,3]}],\"startTime\":1234567890}" 1234567890000000000`), expectError: false, }, { @@ -814,6 +786,106 @@ func TestNatsHandleJobEvent(t *testing.T) { } } +func TestNatsHandleJobEventEdgeCases(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + tests := []struct { + name string + data []byte + expectError bool + description string + }{ + { + name: "non-event message (metric data)", + data: []byte(`job,function=start_job value=123.45 1234567890000000000`), + expectError: false, + description: "Should skip non-event messages gracefully", + }, + { + name: "wrong measurement name", + data: []byte(`wrongmeasurement,function=start_job event="{}" 1234567890000000000`), + expectError: false, + description: "Should warn about unexpected measurement but not fail", + }, + { + name: "missing event field", + data: []byte(`job,function=start_job other_field="value" 1234567890000000000`), + expectError: true, + description: "Should error when event field is missing", + }, + { + name: "multiple measurements in one message", + data: []byte("job,function=start_job event=\"{}\" 1234567890000000000\njob,function=stop_job event=\"{}\" 1234567890000000000"), + expectError: false, + description: "Should process multiple lines", + }, + { + name: "escaped quotes in JSON payload", + data: []byte(`job,function=start_job event="{\"jobId\":6001,\"user\":\"test\\\"user\",\"cluster\":\"test\"}" 1234567890000000000`), + expectError: true, + description: "Should handle escaped quotes (though JSON parsing may fail)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + natsAPI.handleJobEvent("test.subject", tt.data) + time.Sleep(50 * time.Millisecond) + }) + } +} + +func TestNatsHandleNodeStateEdgeCases(t *testing.T) { + natsAPI := setupNatsTest(t) + t.Cleanup(cleanupNatsTest) + + tests := []struct { + name string + data []byte + expectError bool + description string + }{ + { + name: "missing cluster field in JSON", + data: []byte(`nodestate event="{\"nodes\":[]}" 1234567890000000000`), + expectError: true, + description: "Should fail when cluster is missing", + }, + { + name: "malformed JSON with unescaped quotes", + data: []byte(`nodestate event="{\"cluster\":\"test"cluster\",\"nodes\":[]}" 1234567890000000000`), + expectError: true, + description: "Should fail on malformed JSON", + }, + { + name: "unicode characters in hostname", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host-ñ123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`), + expectError: false, + description: "Should handle unicode characters", + }, + { + name: "very large node count", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node1\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node2\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node3\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`), + expectError: false, + description: "Should handle multiple nodes efficiently", + }, + { + name: "timestamp in past", + data: []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1000000000000000000`), + expectError: false, + description: "Should accept any valid timestamp", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + natsAPI.handleNodeState("test.subject", tt.data) + time.Sleep(50 * time.Millisecond) + }) + } +} + func TestNatsHandleStartJobDuplicatePrevention(t *testing.T) { natsAPI := setupNatsTest(t) t.Cleanup(cleanupNatsTest) From 19402d30af2c984e335099933206ecb7c8b6e8bc Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 10:09:19 +0100 Subject: [PATCH 081/341] Review and improve error messages and doc comments --- internal/repository/job.go | 149 +++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 55 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 78e1f3fe..293c28d4 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -156,27 +156,41 @@ func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) { return job, nil } +// Optimize performs database optimization by running VACUUM command. +// This reclaims unused space and defragments the database file. +// Should be run periodically during maintenance windows. func (r *JobRepository) Optimize() error { if _, err := r.DB.Exec(`VACUUM`); err != nil { - return err + cclog.Errorf("Error while executing VACUUM: %v", err) + return fmt.Errorf("failed to optimize database: %w", err) } return nil } +// Flush removes all data from job-related tables (jobtag, tag, job). +// WARNING: This is a destructive operation that deletes all job data. +// Use with extreme caution, typically only for testing or complete resets. func (r *JobRepository) Flush() error { if _, err := r.DB.Exec(`DELETE FROM jobtag`); err != nil { - return err + cclog.Errorf("Error while deleting from jobtag table: %v", err) + return fmt.Errorf("failed to flush jobtag table: %w", err) } if _, err := r.DB.Exec(`DELETE FROM tag`); err != nil { - return err + cclog.Errorf("Error while deleting from tag table: %v", err) + return fmt.Errorf("failed to flush tag table: %w", err) } if _, err := r.DB.Exec(`DELETE FROM job`); err != nil { - return err + cclog.Errorf("Error while deleting from job table: %v", err) + return fmt.Errorf("failed to flush job table: %w", err) } return nil } func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) { + if job == nil { + return nil, fmt.Errorf("job cannot be nil") + } + start := time.Now() cachekey := fmt.Sprintf("metadata:%d", job.ID) if cached := r.cache.Get(cachekey, nil); cached != nil { @@ -186,8 +200,8 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil { - cclog.Warn("Error while scanning for job metadata") - return nil, err + cclog.Warnf("Error while scanning for job metadata (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to fetch metadata for job %d: %w", job.ID, err) } if len(job.RawMetaData) == 0 { @@ -195,8 +209,8 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error } if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { - cclog.Warn("Error while unmarshaling raw metadata json") - return nil, err + cclog.Warnf("Error while unmarshaling raw metadata json (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to unmarshal metadata for job %d: %w", job.ID, err) } r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour) @@ -205,6 +219,10 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error } func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err error) { + if job == nil { + return fmt.Errorf("job cannot be nil") + } + cachekey := fmt.Sprintf("metadata:%d", job.ID) r.cache.Del(cachekey) if job.MetaData == nil { @@ -241,12 +259,16 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er } func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, error) { + if job == nil { + return nil, fmt.Errorf("job cannot be nil") + } + start := time.Now() if err := sq.Select("job.footprint").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawFootprint); err != nil { - cclog.Warn("Error while scanning for job footprint") - return nil, err + cclog.Warnf("Error while scanning for job footprint (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to fetch footprint for job %d: %w", job.ID, err) } if len(job.RawFootprint) == 0 { @@ -254,8 +276,8 @@ func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, err } if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil { - cclog.Warn("Error while unmarshaling raw footprint json") - return nil, err + cclog.Warnf("Error while unmarshaling raw footprint json (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to unmarshal footprint for job %d: %w", job.ID, err) } cclog.Debugf("Timer FetchFootprint %s", time.Since(start)) @@ -263,6 +285,10 @@ func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, err } func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float64, error) { + if job == nil { + return nil, fmt.Errorf("job cannot be nil") + } + start := time.Now() cachekey := fmt.Sprintf("energyFootprint:%d", job.ID) if cached := r.cache.Get(cachekey, nil); cached != nil { @@ -272,8 +298,8 @@ func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float6 if err := sq.Select("job.energy_footprint").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawEnergyFootprint); err != nil { - cclog.Warn("Error while scanning for job energy_footprint") - return nil, err + cclog.Warnf("Error while scanning for job energy_footprint (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to fetch energy footprint for job %d: %w", job.ID, err) } if len(job.RawEnergyFootprint) == 0 { @@ -281,8 +307,8 @@ func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float6 } if err := json.Unmarshal(job.RawEnergyFootprint, &job.EnergyFootprint); err != nil { - cclog.Warn("Error while unmarshaling raw energy footprint json") - return nil, err + cclog.Warnf("Error while unmarshaling raw energy footprint json (ID=%d): %v", job.ID, err) + return nil, fmt.Errorf("failed to unmarshal energy footprint for job %d: %w", job.ID, err) } r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour) @@ -363,6 +389,10 @@ func (r *JobRepository) DeleteJobByID(id int64) error { } func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm string) (jobid string, username string, project string, jobname string) { + if searchterm == "" { + return "", "", "", "" + } + if _, err := strconv.Atoi(searchterm); err == nil { // Return empty on successful conversion: parent method will redirect for integer jobId return searchterm, "", "", "" } else { // Has to have letters and logged-in user for other guesses @@ -394,6 +424,10 @@ var ( ) func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) { + if user == nil { + return "", fmt.Errorf("user cannot be nil") + } + compareStr := " = ?" query := searchterm if isLike { @@ -404,17 +438,11 @@ func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, ta theQuery := sq.Select(table+"."+selectColumn).Distinct().From(table). Where(table+"."+whereColumn+compareStr, query) - // theSql, args, theErr := theQuery.ToSql() - // if theErr != nil { - // cclog.Warn("Error while converting query to sql") - // return "", err - // } - // cclog.Debugf("SQL query (FindColumnValue): `%s`, args: %#v", theSql, args) - err := theQuery.RunWith(r.stmtCache).QueryRow().Scan(&result) if err != nil && err != sql.ErrNoRows { - return "", err + cclog.Warnf("Error while querying FindColumnValue (table=%s, column=%s): %v", table, selectColumn, err) + return "", fmt.Errorf("failed to find column value: %w", err) } else if err == nil { return result, nil } @@ -426,21 +454,26 @@ func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, ta } func (r *JobRepository) FindColumnValues(user *schema.User, query string, table string, selectColumn string, whereColumn string) (results []string, err error) { + if user == nil { + return nil, fmt.Errorf("user cannot be nil") + } + emptyResult := make([]string, 0) if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) { rows, err := sq.Select(table+"."+selectColumn).Distinct().From(table). Where(table+"."+whereColumn+" LIKE ?", fmt.Sprint("%", query, "%")). RunWith(r.stmtCache).Query() if err != nil && err != sql.ErrNoRows { - return emptyResult, err + cclog.Errorf("Error while querying FindColumnValues (table=%s, column=%s): %v", table, selectColumn, err) + return emptyResult, fmt.Errorf("failed to find column values: %w", err) } else if err == nil { + defer rows.Close() for rows.Next() { var result string err := rows.Scan(&result) if err != nil { - rows.Close() - cclog.Warnf("Error while scanning rows: %v", err) - return emptyResult, err + cclog.Warnf("Error while scanning rows in FindColumnValues: %v", err) + return emptyResult, fmt.Errorf("failed to scan column value: %w", err) } results = append(results, result) } @@ -482,8 +515,8 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in Where("job.cluster = ?", cluster). RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running AllocatedNodes query for cluster=%s: %v", cluster, err) + return nil, fmt.Errorf("failed to query allocated nodes for cluster %s: %w", cluster, err) } var raw []byte @@ -493,12 +526,12 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in var resources []*schema.Resource var subcluster string if err := rows.Scan(&raw, &subcluster); err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in AllocatedNodes: %v", err) + return nil, fmt.Errorf("failed to scan allocated nodes row: %w", err) } if err := json.Unmarshal(raw, &resources); err != nil { - cclog.Warn("Error while unmarshaling raw resources json") - return nil, err + cclog.Warnf("Error while unmarshaling raw resources json in AllocatedNodes: %v", err) + return nil, fmt.Errorf("failed to unmarshal resources in AllocatedNodes: %w", err) } hosts, ok := subclusters[subcluster] @@ -529,14 +562,14 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { Where("(? - job.start_time) > (job.walltime + ?)", currentTime, seconds). RunWith(r.DB).Exec() if err != nil { - cclog.Warn("Error while stopping jobs exceeding walltime") - return err + cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err) + return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err) } rowsAffected, err := res.RowsAffected() if err != nil { - cclog.Warn("Error while fetching affected rows after stopping due to exceeded walltime") - return err + cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err) + return fmt.Errorf("failed to get rows affected count: %w", err) } if rowsAffected > 0 { @@ -552,18 +585,19 @@ func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) { Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct() rows, err := query.RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err) + return nil, fmt.Errorf("failed to find job IDs by tag %d: %w", tagID, err) } + defer rows.Close() + jobIds := make([]int64, 0, 100) for rows.Next() { var jobID int64 if err := rows.Scan(&jobID); err != nil { - rows.Close() - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in FindJobIdsByTag: %v", err) + return nil, fmt.Errorf("failed to scan job ID in FindJobIdsByTag: %w", err) } jobIds = append(jobIds, jobID) @@ -581,8 +615,8 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) { rows, err := query.RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running FindRunningJobs query for cluster=%s: %v", cluster, err) + return nil, fmt.Errorf("failed to find running jobs for cluster %s: %w", cluster, err) } defer rows.Close() @@ -590,8 +624,8 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) { for rows.Next() { job, err := scanJob(rows) if err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in FindRunningJobs: %v", err) + return nil, fmt.Errorf("failed to scan job in FindRunningJobs: %w", err) } jobs = append(jobs, job) } @@ -607,7 +641,8 @@ func (r *JobRepository) UpdateDuration() error { _, err := stmnt.RunWith(r.stmtCache).Exec() if err != nil { - return err + cclog.Errorf("Error while updating duration for running jobs: %v", err) + return fmt.Errorf("failed to update duration for running jobs: %w", err) } return nil @@ -634,8 +669,8 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64 rows, err := query.RunWith(r.stmtCache).Query() if err != nil { - cclog.Error("Error while running query") - return nil, err + cclog.Errorf("Error while running FindJobsBetween query: %v", err) + return nil, fmt.Errorf("failed to find jobs between %d and %d: %w", startTimeBegin, startTimeEnd, err) } defer rows.Close() @@ -643,8 +678,8 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64 for rows.Next() { job, err := scanJob(rows) if err != nil { - cclog.Warn("Error while scanning rows") - return nil, err + cclog.Warnf("Error while scanning rows in FindJobsBetween: %v", err) + return nil, fmt.Errorf("failed to scan job in FindJobsBetween: %w", err) } jobs = append(jobs, job) } @@ -662,13 +697,17 @@ func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32 Set("monitoring_status", monitoringStatus). Where("job.id = ?", job) - _, err = stmt.RunWith(r.stmtCache).Exec() - return err + if _, err = stmt.RunWith(r.stmtCache).Exec(); err != nil { + cclog.Errorf("Error while updating monitoring status for job %d: %v", job, err) + return fmt.Errorf("failed to update monitoring status for job %d: %w", job, err) + } + return nil } func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error { if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil { - return err + cclog.Errorf("Error while executing statement: %v", err) + return fmt.Errorf("failed to execute update statement: %w", err) } return nil From 0ea0270fe17ceca3156a2f563df7987d6ab50063 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 10:37:07 +0100 Subject: [PATCH 082/341] Reintroduce Clusters as string list of cluster names --- web/web.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/web.go b/web/web.go index 37f1c2b2..6ff01403 100644 --- a/web/web.go +++ b/web/web.go @@ -245,6 +245,7 @@ type Page struct { User schema.User // Information about the currently logged in user (Full User Info) Roles map[string]schema.Role // Available roles for frontend render checks Build Build // Latest information about the application + Clusters []string // List of all cluster names SubClusters map[string][]string // Map per cluster of all subClusters for use in the Header FilterPresets map[string]any // For pages with the Filter component, this can be used to set initial filters. Infos map[string]any // For generic use (e.g. username for /monitoring/user/, job id for /monitoring/job/) @@ -259,9 +260,15 @@ func RenderTemplate(rw http.ResponseWriter, file string, page *Page) { cclog.Errorf("WEB/WEB > template '%s' not found", file) } + if page.Clusters == nil { + page.Clusters = make([]string, 2) + } + if page.SubClusters == nil { page.SubClusters = make(map[string][]string) for _, cluster := range archive.Clusters { + page.Clusters = append(page.Clusters, cluster.Name) + for _, sc := range cluster.SubClusters { page.SubClusters[cluster.Name] = append(page.SubClusters[cluster.Name], sc.Name) } From c8627a13f40011cd07d43772760c09a73078c0c4 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jan 2026 11:17:49 +0100 Subject: [PATCH 083/341] Remove obsolete slusters config section --- cmd/cc-backend/init.go | 28 ++------------- configs/config-demo.json | 47 -------------------------- configs/config.json | 34 ++----------------- internal/api/api_test.go | 35 +++++++------------ internal/api/nats_test.go | 11 ------ internal/importer/importer_test.go | 32 ++---------------- internal/repository/node_test.go | 11 ------ internal/repository/userConfig_test.go | 10 ------ 8 files changed, 19 insertions(+), 189 deletions(-) diff --git a/cmd/cc-backend/init.go b/cmd/cc-backend/init.go index 025396be..e30ae2e1 100644 --- a/cmd/cc-backend/init.go +++ b/cmd/cc-backend/init.go @@ -48,7 +48,7 @@ const configString = ` "emission-constant": 317 }, "cron": { - "commit-job-worker": "2m", + "commit-job-worker": "1m", "duration-worker": "5m", "footprint-worker": "10m" }, @@ -60,31 +60,7 @@ const configString = ` "jwts": { "max-age": "2000h" } - }, - "clusters": [ - { - "name": "name", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2023-01-01T00:00:00Z", - "to": null - } - } - } - ] + } } ` diff --git a/configs/config-demo.json b/configs/config-demo.json index aa388316..bd492e31 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -29,52 +29,6 @@ "username": "root", "password": "root" }, - "clusters": [ - { - "name": "fritz", - "metricDataRepository": { - "kind": "cc-metric-store-internal", - "url": "http://localhost:8082", - "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - }, - { - "name": "alex", - "metricDataRepository": { - "kind": "cc-metric-store-internal", - "url": "http://localhost:8082", - "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - } - ], "metric-store": { "checkpoints": { "file-format": "avro", @@ -99,4 +53,3 @@ ] } } - diff --git a/configs/config.json b/configs/config.json index 41d8ecac..44961c85 100644 --- a/configs/config.json +++ b/configs/config.json @@ -11,10 +11,7 @@ "resampling": { "minimumPoints": 600, "trigger": 180, - "resolutions": [ - 240, - 60 - ] + "resolutions": [240, 60] }, "apiSubjects": { "subjectJobEvent": "cc.job.event", @@ -22,37 +19,12 @@ } }, "cron": { - "commit-job-worker": "2m", + "commit-job-worker": "1m", "duration-worker": "5m", "footprint-worker": "10m" }, "archive": { "kind": "file", "path": "./var/job-archive" - }, - "clusters": [ - { - "name": "test", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "eyJhbGciOiJF-E-pQBQ" - }, - "filterRanges": { - "numNodes": { - "from": 1, - "to": 64 - }, - "duration": { - "from": 0, - "to": 86400 - }, - "startTime": { - "from": "2022-01-01T00:00:00Z", - "to": null - } - } - } - ] + } } - diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 7aa935ff..025983c1 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -39,33 +39,22 @@ func setup(t *testing.T) *api.RestAPI { repository.ResetConnection() const testconfig = `{ - "main": { - "addr": "0.0.0.0:8080", - "validate": false, - "apiAllowedIPs": [ - "*" - ] - }, + "main": { + "addr": "0.0.0.0:8080", + "validate": false, + "apiAllowedIPs": [ + "*" + ] + }, "archive": { - "kind": "file", - "path": "./var/job-archive" + "kind": "file", + "path": "./var/job-archive" }, "auth": { - "jwts": { - "max-age": "2m" + "jwts": { + "max-age": "2m" + } } - }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } - } - ] }` const testclusterJSON = `{ "name": "testcluster", diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go index 4b1431cb..c6a9bcd9 100644 --- a/internal/api/nats_test.go +++ b/internal/api/nats_test.go @@ -48,18 +48,7 @@ func setupNatsTest(t *testing.T) *NatsAPI { "jwts": { "max-age": "2m" } - }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } } - ] }` const testclusterJSON = `{ "name": "testcluster", diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 2d00fc84..ebc500b7 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -56,36 +56,8 @@ func setup(t *testing.T) *repository.JobRepository { "archive": { "kind": "file", "path": "./var/job-archive" - }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } - }, - { - "name": "fritz", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 944 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } - }, - { - "name": "taurus", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 4000 }, - "duration": { "from": 0, "to": 604800 }, - "startTime": { "from": "2010-01-01T00:00:00Z", "to": null } - } - } - ]}` + } + }` cclog.Init("info", true) tmpdir := t.TempDir() diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index fd935b53..990de924 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -38,18 +38,7 @@ func nodeTestSetup(t *testing.T) { "jwts": { "max-age": "2m" } - }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } - } } - ] }` const testclusterJSON = `{ "name": "testcluster", diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index ae3adaf2..f66fccfb 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -27,17 +27,7 @@ func setupUserTest(t *testing.T) *UserCfgRepo { "archive": { "kind": "file", "path": "./var/job-archive" - }, - "clusters": [ - { - "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, - "filterRanges": { - "numNodes": { "from": 1, "to": 64 }, - "duration": { "from": 0, "to": 86400 }, - "startTime": { "from": "2022-01-01T00:00:00Z", "to": null } } - }] }` cclog.Init("info", true) From 4cb8d648cbb5bd40e40462d6f0d2d4d8fa78178c Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 14 Jan 2026 11:25:40 +0100 Subject: [PATCH 084/341] adapt frontend to backend config changes, clarify variable names --- web/frontend/src/Config.root.svelte | 6 +-- web/frontend/src/Header.svelte | 36 +++++++-------- web/frontend/src/config.entrypoint.js | 2 +- web/frontend/src/config/AdminSettings.svelte | 6 +-- web/frontend/src/config/admin/Options.svelte | 12 ++--- .../src/generic/filters/Resources.svelte | 20 +-------- web/frontend/src/header.entrypoint.js | 4 +- web/frontend/src/header/NavbarLinks.svelte | 44 +++++++++---------- web/templates/base.tmpl | 4 +- 9 files changed, 58 insertions(+), 76 deletions(-) diff --git a/web/frontend/src/Config.root.svelte b/web/frontend/src/Config.root.svelte index 171b2a08..0e1daec3 100644 --- a/web/frontend/src/Config.root.svelte +++ b/web/frontend/src/Config.root.svelte @@ -7,7 +7,7 @@ - `isApi Bool!`: Is currently logged in user api authority - `username String!`: Empty string if auth. is disabled, otherwise the username as string - `ncontent String!`: The currently displayed message on the homescreen - - `clusters [String]`: The available clusternames + - `clusterNames [String]`: The available clusternames --> @@ -32,7 +32,7 @@ Admin Options - +     {/if} diff --git a/web/frontend/src/Header.svelte b/web/frontend/src/Header.svelte index f7ceac2e..ea818c62 100644 --- a/web/frontend/src/Header.svelte +++ b/web/frontend/src/Header.svelte @@ -4,8 +4,8 @@ Properties: - `username String`: Empty string if auth. is disabled, otherwise the username as string - `authlevel Number`: The current users authentication level - - `clusters [String]`: List of cluster names - - `subClusters [String]`: List of subCluster names + - `clusterNames [String]`: List of cluster names + - `subclusterMap map[String][]string`: Map of subclusters by cluster names - `roles [Number]`: Enum containing available roles --> @@ -28,8 +28,8 @@ let { username, authlevel, - clusters, - subClusters, + clusterNames, + subclusterMap, roles } = $props(); @@ -152,15 +152,15 @@
    • - {#if item?.data} - {#if item.disabled === true} - - Metric disabled for subcluster {selectedMetric}:{item.subCluster} - {:else if item.disabled === false} - - - {#key item.data[0].metric.series[0].data.length} - - {/key} - {:else} - - Global Metric List Not Initialized - Can not determine {selectedMetric} availability: Please Reload Page - - {/if} + {#if item?.disabled} + + + Disabled Metric + + +

    No dataset(s) returned for {selectedMetric}

    +

    Metric has been disabled for subcluster {item.subCluster}.

    +     +     + {:else if item?.data} + + + {#key item.data[0].metric.series[0].data.length} + + {/key} {:else} + Missing Metric @@ -205,10 +201,22 @@ {/each} {/key} +{:else if hostnameFilter || hoststateFilter != 'all'} + + + + Empty Filter Return + + +

    No datasets returned for {selectedMetric}.

    +

    Hostname filter and/or host state filter returned no matches.

    +     +     +     {:else} - - - + + + Missing Metric diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 2abe0b41..e091769b 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -72,10 +72,30 @@ ); const extendedLegendData = $derived($nodeJobsData?.data ? buildExtendedLegend() : null); - const refinedData = $derived(nodeData?.metrics ? sortAndSelectScope(nodeData.metrics) : []); + const refinedData = $derived(nodeData?.metrics ? sortAndSelectScope(selectedMetrics, nodeData.metrics) : []); const dataHealth = $derived(refinedData.filter((rd) => rd.disabled === false).map((enabled) => (enabled?.data?.metric?.series?.length > 0))); /* Functions */ + function sortAndSelectScope(metricList = [], nodeMetrics = []) { + const pendingData = []; + metricList.forEach((metricName) => { + const pendingMetric = { + name: metricName, + disabled: checkMetricDisabled( + globalMetrics, + metricName, + cluster, + nodeData.subCluster, + ), + data: null + }; + const scopesData = nodeMetrics.filter((nodeMetric) => nodeMetric.name == metricName) + if (scopesData.length > 0) pendingMetric.data = selectScope(scopesData) + pendingData.push(pendingMetric) + }); + return pendingData; + }; + const selectScope = (nodeMetrics) => nodeMetrics.reduce( (a, b) => @@ -83,29 +103,6 @@ nodeMetrics[0], ); - const sortAndSelectScope = (allNodeMetrics) => - selectedMetrics - .map((selectedName) => allNodeMetrics.filter((nodeMetric) => nodeMetric.name == selectedName)) - .map((matchedNodeMetrics) => ({ - disabled: false, - data: matchedNodeMetrics.length > 0 ? selectScope(matchedNodeMetrics) : null, - })) - .map((scopedNodeMetric) => { - if (scopedNodeMetric?.data) { - return { - disabled: checkMetricDisabled( - globalMetrics, - scopedNodeMetric.data.name, - cluster, - nodeData.subCluster, - ), - data: scopedNodeMetric.data, - }; - } else { - return scopedNodeMetric; - } - }); - function buildExtendedLegend() { let pendingExtendedLegendData = null // Build Extended for allocated nodes [Commented: Only Build extended Legend For Shared Nodes] @@ -171,68 +168,59 @@ {/if} {#each refinedData as metricData, i (metricData?.data?.name || i)} - {#key metricData} - - {#if metricData?.disabled} - Metric {selectedMetrics[i]} disabled for subcluster {nodeData.subCluster} - {:else if !metricData?.data} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric was not found in metric store for cluster {cluster}.

    -     - {:else if !metricData?.data?.name} - Metric without name for subcluster {`Metric Index ${i}`}:{nodeData.subCluster} - {:else if !!metricData.data?.metric.statisticsSeries} - - -
    - {#key extendedLegendData} - - {/key} - {:else} - - {/if} - - {/key} + + {#if metricData?.disabled} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric has been disabled for subcluster {nodeData.subCluster}.

    +     + {:else if !metricData?.data} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric was not found in metric store for cluster {cluster}.

    +     + {:else if !!metricData.data?.metric.statisticsSeries} + + +
    + {#key extendedLegendData} + + {/key} + {:else} + + {/if} + {/each} From 51e9d33f9f6abfc73344c07aa9708df187ee8f36 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 9 Feb 2026 17:21:49 +0100 Subject: [PATCH 228/341] fix empty availability print case --- .../src/generic/select/MetricSelection.svelte | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/web/frontend/src/generic/select/MetricSelection.svelte b/web/frontend/src/generic/select/MetricSelection.svelte index dcefa56d..8234b32c 100644 --- a/web/frontend/src/generic/select/MetricSelection.svelte +++ b/web/frontend/src/generic/select/MetricSelection.svelte @@ -88,16 +88,19 @@ function printAvailability(metric, cluster) { const avail = globalMetrics.find((gm) => gm.name === metric)?.availability - if (!cluster) { - return avail.map((av) => av.cluster).join(', ') - } else { - const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters - if (subAvail) { - return subAvail.join(', ') + if (avail) { + if (!cluster) { + return avail.map((av) => av.cluster).join(', ') } else { - return `Not available for ${cluster}` + const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters + if (subAvail) { + return subAvail.join(', ') + } else { + return `Not available for ${cluster}` + } } } + return "" } function columnsDragOver(event) { From ac7eb93141d081ca083e8e576e8bc61268f2671e Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 9 Feb 2026 19:57:46 +0100 Subject: [PATCH 229/341] fix: Transfer always to main job table before archiving --- internal/api/job.go | 22 +++++++--- internal/api/nats.go | 22 +++++++--- internal/repository/jobCreate.go | 43 +++++++++++-------- internal/repository/jobCreate_test.go | 62 ++++++++++++++------------- 4 files changed, 91 insertions(+), 58 deletions(-) diff --git a/internal/api/job.go b/internal/api/job.go index d67dbb93..9bd93b1c 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -754,6 +754,7 @@ func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { return } + isCached := false job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime) if err != nil { // Try cached jobs if not found in main repository @@ -764,9 +765,10 @@ func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { return } job = cachedJob + isCached = true } - api.checkAndHandleStopJob(rw, job, req) + api.checkAndHandleStopJob(rw, job, req, isCached) } // deleteJobByID godoc @@ -923,7 +925,7 @@ func (api *RestAPI) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { } } -func (api *RestAPI) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobAPIRequest) { +func (api *RestAPI) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobAPIRequest, isCached bool) { // Sanity checks if job.State != schema.JobStateRunning { handleError(fmt.Errorf("jobId %d (id %d) on %s : job has already been stopped (state is: %s)", job.JobID, *job.ID, job.Cluster, job.State), http.StatusUnprocessableEntity, rw) @@ -948,11 +950,21 @@ func (api *RestAPI) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo api.JobRepository.Mutex.Lock() defer api.JobRepository.Mutex.Unlock() - if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - handleError(fmt.Errorf("jobId %d (id %d) on %s : marking job as '%s' (duration: %d) in DB failed: %w", job.JobID, *job.ID, job.Cluster, job.State, job.Duration, err), http.StatusInternalServerError, rw) + // If the job is still in job_cache, transfer it to the job table first + // so that job.ID always points to the job table for downstream code + if isCached { + newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID) + if err != nil { + handleError(fmt.Errorf("jobId %d (id %d) on %s : transferring cached job failed: %w", job.JobID, *job.ID, job.Cluster, err), http.StatusInternalServerError, rw) return } + cclog.Infof("transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID) + job.ID = &newID + } + + if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + handleError(fmt.Errorf("jobId %d (id %d) on %s : marking job as '%s' (duration: %d) in DB failed: %w", job.JobID, *job.ID, job.Cluster, job.State, job.Duration, err), http.StatusInternalServerError, rw) + return } cclog.Infof("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s", *job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) diff --git a/internal/api/nats.go b/internal/api/nats.go index c0a8c174..0e929426 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -251,6 +251,7 @@ func (api *NatsAPI) handleStopJob(payload string) { return } + isCached := false job, err := api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime) if err != nil { cachedJob, cachedErr := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime) @@ -260,6 +261,7 @@ func (api *NatsAPI) handleStopJob(payload string) { return } job = cachedJob + isCached = true } if job.State != schema.JobStateRunning { @@ -287,16 +289,26 @@ func (api *NatsAPI) handleStopJob(payload string) { api.JobRepository.Mutex.Lock() defer api.JobRepository.Mutex.Unlock() - if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v", - job.JobID, job.ID, job.Cluster, job.State, err) + // If the job is still in job_cache, transfer it to the job table first + if isCached { + newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID) + if err != nil { + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v", + job.JobID, *job.ID, job.Cluster, err) return } + cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID) + job.ID = &newID + } + + if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v", + job.JobID, *job.ID, job.Cluster, job.State, err) + return } cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s", - job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) + *job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) if job.MonitoringStatus == schema.MonitoringStatusDisabled { return diff --git a/internal/repository/jobCreate.go b/internal/repository/jobCreate.go index 6114ae5e..9f4f366d 100644 --- a/internal/repository/jobCreate.go +++ b/internal/repository/jobCreate.go @@ -71,8 +71,9 @@ func (r *JobRepository) SyncJobs() ([]*schema.Job, error) { jobs = append(jobs, job) } + // Use INSERT OR IGNORE to skip jobs already transferred by the stop path _, err = r.DB.Exec( - "INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache") + "INSERT OR IGNORE INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache") if err != nil { cclog.Warnf("Error while Job sync: %v", err) return nil, err @@ -87,6 +88,29 @@ func (r *JobRepository) SyncJobs() ([]*schema.Job, error) { return jobs, nil } +// TransferCachedJobToMain moves a job from job_cache to the job table. +// Caller must hold r.Mutex. Returns the new job table ID. +func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) { + res, err := r.DB.Exec( + "INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?", + cacheID) + if err != nil { + return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err) + } + + newID, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err) + } + + _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID) + if err != nil { + return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err) + } + + return newID, nil +} + // Start inserts a new job in the table, returning the unique job ID. // Statistics are not transfered! func (r *JobRepository) Start(job *schema.Job) (id int64, err error) { @@ -129,20 +153,3 @@ func (r *JobRepository) Stop( return err } -func (r *JobRepository) StopCached( - jobID int64, - duration int32, - state schema.JobState, - monitoringStatus int32, -) (err error) { - // Note: StopCached updates job_cache table, not the main job table - // Cache invalidation happens when job is synced to main table - stmt := sq.Update("job_cache"). - Set("job_state", state). - Set("duration", duration). - Set("monitoring_status", monitoringStatus). - Where("job_cache.id = ?", jobID) - - _, err = stmt.RunWith(r.stmtCache).Exec() - return err -} diff --git a/internal/repository/jobCreate_test.go b/internal/repository/jobCreate_test.go index 3a586482..9e72555f 100644 --- a/internal/repository/jobCreate_test.go +++ b/internal/repository/jobCreate_test.go @@ -331,58 +331,60 @@ func TestStop(t *testing.T) { }) } -func TestStopCached(t *testing.T) { +func TestTransferCachedJobToMain(t *testing.T) { r := setup(t) - t.Run("successful stop cached job", func(t *testing.T) { + t.Run("successful transfer from cache to main", func(t *testing.T) { // Insert a job in job_cache job := createTestJob(999009, "testcluster") - id, err := r.Start(job) + cacheID, err := r.Start(job) require.NoError(t, err) - // Stop the cached job - duration := int32(3600) - state := schema.JobStateCompleted - monitoringStatus := int32(schema.MonitoringStatusArchivingSuccessful) + // Transfer the cached job to the main table + r.Mutex.Lock() + newID, err := r.TransferCachedJobToMain(cacheID) + r.Mutex.Unlock() + require.NoError(t, err, "TransferCachedJobToMain should succeed") + assert.NotEqual(t, cacheID, newID, "New ID should differ from cache ID") - err = r.StopCached(id, duration, state, monitoringStatus) - require.NoError(t, err, "StopCached should succeed") - - // Verify job was updated in job_cache table - var retrievedDuration int32 - var retrievedState string - var retrievedMonStatus int32 - err = r.DB.QueryRow(`SELECT duration, job_state, monitoring_status FROM job_cache WHERE id = ?`, id).Scan( - &retrievedDuration, &retrievedState, &retrievedMonStatus) + // Verify job exists in job table + var count int + err = r.DB.QueryRow(`SELECT COUNT(*) FROM job WHERE id = ?`, newID).Scan(&count) require.NoError(t, err) - assert.Equal(t, duration, retrievedDuration) - assert.Equal(t, string(state), retrievedState) - assert.Equal(t, monitoringStatus, retrievedMonStatus) + assert.Equal(t, 1, count, "Job should exist in main table") + + // Verify job was removed from job_cache + err = r.DB.QueryRow(`SELECT COUNT(*) FROM job_cache WHERE id = ?`, cacheID).Scan(&count) + require.NoError(t, err) + assert.Equal(t, 0, count, "Job should be removed from cache") // Clean up - _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id) + _, err = r.DB.Exec("DELETE FROM job WHERE id = ?", newID) require.NoError(t, err) }) - t.Run("stop cached job does not affect job table", func(t *testing.T) { + t.Run("transfer preserves job data", func(t *testing.T) { // Insert a job in job_cache job := createTestJob(999010, "testcluster") - id, err := r.Start(job) + cacheID, err := r.Start(job) require.NoError(t, err) - // Stop the cached job - err = r.StopCached(id, 3600, schema.JobStateCompleted, int32(schema.MonitoringStatusArchivingSuccessful)) + // Transfer the cached job + r.Mutex.Lock() + newID, err := r.TransferCachedJobToMain(cacheID) + r.Mutex.Unlock() require.NoError(t, err) - // Verify job table was not affected - var count int - err = r.DB.QueryRow(`SELECT COUNT(*) FROM job WHERE job_id = ? AND cluster = ?`, - job.JobID, job.Cluster).Scan(&count) + // Verify the transferred job has the correct data + var jobID int64 + var cluster string + err = r.DB.QueryRow(`SELECT job_id, cluster FROM job WHERE id = ?`, newID).Scan(&jobID, &cluster) require.NoError(t, err) - assert.Equal(t, 0, count, "Job table should not be affected by StopCached") + assert.Equal(t, job.JobID, jobID) + assert.Equal(t, job.Cluster, cluster) // Clean up - _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id) + _, err = r.DB.Exec("DELETE FROM job WHERE id = ?", newID) require.NoError(t, err) }) } From 035ac2384eb4be136b18308379c4b15ad194540c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 9 Feb 2026 21:56:41 +0100 Subject: [PATCH 230/341] Refactor GlobalMetricLists --- pkg/archive/clusterConfig.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 272eeb35..64851365 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -25,6 +25,7 @@ func initClusterConfig() error { GlobalUserMetricList = []*schema.GlobalMetricListItem{} NodeLists = map[string]map[string]NodeList{} metricLookup := make(map[string]schema.GlobalMetricListItem) + userMetricLookup := make(map[string]schema.GlobalMetricListItem) for _, c := range ar.GetClusters() { @@ -62,11 +63,12 @@ func initClusterConfig() error { if _, ok := metricLookup[mc.Name]; !ok { metricLookup[mc.Name] = schema.GlobalMetricListItem{ - Name: mc.Name, Scope: mc.Scope, Restrict: mc.Restrict, Unit: mc.Unit, Footprint: mc.Footprint, + Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint, } } availability := schema.ClusterSupport{Cluster: cluster.Name} + userAvailability := schema.ClusterSupport{Cluster: cluster.Name} scLookup := make(map[string]*schema.SubClusterConfig) for _, scc := range mc.SubClusters { @@ -94,6 +96,7 @@ func initClusterConfig() error { newMetric.Footprint = mc.Footprint } + isRestricted := mc.Restrict if cfg, ok := scLookup[sc.Name]; ok { if cfg.Remove { continue @@ -105,9 +108,13 @@ func initClusterConfig() error { newMetric.Footprint = cfg.Footprint newMetric.Energy = cfg.Energy newMetric.LowerIsBetter = cfg.LowerIsBetter + isRestricted = cfg.Restrict } availability.SubClusters = append(availability.SubClusters, sc.Name) + if !isRestricted { + userAvailability.SubClusters = append(userAvailability.SubClusters, sc.Name) + } sc.MetricConfig = append(sc.MetricConfig, newMetric) if newMetric.Footprint != "" { @@ -124,6 +131,17 @@ func initClusterConfig() error { item := metricLookup[mc.Name] item.Availability = append(item.Availability, availability) metricLookup[mc.Name] = item + + if len(userAvailability.SubClusters) > 0 { + userItem, ok := userMetricLookup[mc.Name] + if !ok { + userItem = schema.GlobalMetricListItem{ + Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint, + } + } + userItem.Availability = append(userItem.Availability, userAvailability) + userMetricLookup[mc.Name] = userItem + } } Clusters = append(Clusters, cluster) @@ -144,9 +162,9 @@ func initClusterConfig() error { for _, metric := range metricLookup { GlobalMetricList = append(GlobalMetricList, &metric) - if !metric.Restrict { - GlobalUserMetricList = append(GlobalUserMetricList, &metric) - } + } + for _, metric := range userMetricLookup { + GlobalUserMetricList = append(GlobalUserMetricList, &metric) } return nil From d21943a514124b39e3f4aa602b1d6a2feaa708e0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 10 Feb 2026 07:52:58 +0100 Subject: [PATCH 231/341] Upgrade cc-lib --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 77da0104..6bcc3b08 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ tool ( require ( github.com/99designs/gqlgen v0.17.85 - github.com/ClusterCockpit/cc-lib/v2 v2.2.1 + github.com/ClusterCockpit/cc-lib/v2 v2.2.2 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.6 diff --git a/go.sum b/go.sum index 40b90751..f4f41dfd 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/ClusterCockpit/cc-lib/v2 v2.2.1 h1:iCVas+Jc61zFH5S2VG3H1sc7tsn+U4lOJwUYjYZEims= github.com/ClusterCockpit/cc-lib/v2 v2.2.1/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= +github.com/ClusterCockpit/cc-lib/v2 v2.2.2 h1:ye4RY57I19c2cXr3XWZBS/QYYgQVeGFvsiu5HkyKq9E= +github.com/ClusterCockpit/cc-lib/v2 v2.2.2/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= From 1feb3baf68af9cc6fe5a222013e9d8d47ff54ed8 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 10 Feb 2026 07:53:30 +0100 Subject: [PATCH 232/341] Create copy of test db before unit tests --- internal/repository/node_test.go | 7 +++++++ internal/repository/repository_test.go | 20 ++++++++++++++++++-- internal/repository/stats_test.go | 10 ++-------- internal/repository/userConfig_test.go | 21 +++++++++++++++++++-- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index b863dc69..4286ab34 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -139,6 +139,13 @@ func nodeTestSetup(t *testing.T) { } archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive) + if err := ResetConnection(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ResetConnection() + }) + Connect(dbfilepath) if err := archive.Init(json.RawMessage(archiveCfg)); err != nil { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 34852830..b9496143 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -6,6 +6,8 @@ package repository import ( "context" + "os" + "path/filepath" "testing" "github.com/ClusterCockpit/cc-backend/internal/graph/model" @@ -148,8 +150,22 @@ func getContext(tb testing.TB) context.Context { func setup(tb testing.TB) *JobRepository { tb.Helper() cclog.Init("warn", true) - dbfile := "testdata/job.db" - err := MigrateDB(dbfile) + + // Copy test DB to a temp file for test isolation + srcData, err := os.ReadFile("testdata/job.db") + noErr(tb, err) + dbfile := filepath.Join(tb.TempDir(), "job.db") + err = os.WriteFile(dbfile, srcData, 0o644) + noErr(tb, err) + + // Reset singletons so Connect uses the new temp DB + err = ResetConnection() + noErr(tb, err) + tb.Cleanup(func() { + ResetConnection() + }) + + err = MigrateDB(dbfile) noErr(tb, err) Connect(dbfile) return GetJobRepository() diff --git a/internal/repository/stats_test.go b/internal/repository/stats_test.go index a8dfc818..a6c2da17 100644 --- a/internal/repository/stats_test.go +++ b/internal/repository/stats_test.go @@ -25,17 +25,11 @@ func TestBuildJobStatsQuery(t *testing.T) { func TestJobStats(t *testing.T) { r := setup(t) - // First, count the actual jobs in the database (excluding test jobs) var expectedCount int - err := r.DB.QueryRow(`SELECT COUNT(*) FROM job WHERE cluster != 'testcluster'`).Scan(&expectedCount) + err := r.DB.QueryRow(`SELECT COUNT(*) FROM job`).Scan(&expectedCount) noErr(t, err) - filter := &model.JobFilter{} - // Exclude test jobs created by other tests - testCluster := "testcluster" - filter.Cluster = &model.StringInput{Neq: &testCluster} - - stats, err := r.JobsStats(getContext(t), []*model.JobFilter{filter}) + stats, err := r.JobsStats(getContext(t), []*model.JobFilter{}) noErr(t, err) if stats[0].TotalJobs != expectedCount { diff --git a/internal/repository/userConfig_test.go b/internal/repository/userConfig_test.go index cee59304..17ccbf78 100644 --- a/internal/repository/userConfig_test.go +++ b/internal/repository/userConfig_test.go @@ -31,8 +31,25 @@ func setupUserTest(t *testing.T) *UserCfgRepo { }` cclog.Init("info", true) - dbfilepath := "testdata/job.db" - err := MigrateDB(dbfilepath) + + // Copy test DB to a temp file for test isolation + srcData, err := os.ReadFile("testdata/job.db") + if err != nil { + t.Fatal(err) + } + dbfilepath := filepath.Join(t.TempDir(), "job.db") + if err := os.WriteFile(dbfilepath, srcData, 0o644); err != nil { + t.Fatal(err) + } + + if err := ResetConnection(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ResetConnection() + }) + + err = MigrateDB(dbfilepath) if err != nil { t.Fatal(err) } From 0dff9fa07ff521c00311e7af5ca28d5db40f7ab4 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 10 Feb 2026 09:17:34 +0100 Subject: [PATCH 233/341] Update docs and agent files --- CLAUDE.md | 18 +++++++++--------- README.md | 6 +++--- internal/archiver/README.md | 1 - tools/convert-pem-pubkey/Readme.md | 2 +- web/frontend/README.md | 6 +++--- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 406f11ba..2148fdca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ make make frontend # Build only the backend (requires frontend to be built first) -go build -ldflags='-s -X main.date=$(date +"%Y-%m-%d:T%H:%M:%S") -X main.version=1.4.4 -X main.commit=$(git rev-parse --short HEAD)' ./cmd/cc-backend +go build -ldflags='-s -X main.date=$(date +"%Y-%m-%d:T%H:%M:%S") -X main.version=1.5.0 -X main.commit=$(git rev-parse --short HEAD)' ./cmd/cc-backend ``` ### Testing @@ -41,7 +41,7 @@ go test ./internal/repository ### Code Generation ```bash -# Regenerate GraphQL schema and resolvers (after modifying api/*.graphqls) +# Regenerate GraphQL schema and resolvers (after modifying api/schema.graphqls) make graphql # Regenerate Swagger/OpenAPI docs (after modifying API comments) @@ -90,7 +90,7 @@ The backend follows a layered architecture with clear separation of concerns: - Transaction support for batch operations - **internal/api**: REST API endpoints (Swagger/OpenAPI documented) - **internal/graph**: GraphQL API (uses gqlgen) - - Schema in `api/*.graphqls` + - Schema in `api/schema.graphqls` - Generated code in `internal/graph/generated/` - Resolvers in `internal/graph/schema.resolvers.go` - **internal/auth**: Authentication layer @@ -108,7 +108,7 @@ The backend follows a layered architecture with clear separation of concerns: - File system backend (default) - S3 backend - SQLite backend (experimental) -- **pkg/nats**: NATS client and message decoding utilities +- **internal/metricstoreclient**: Client for cc-metric-store queries ### Frontend Structure @@ -138,7 +138,7 @@ recommended). Configuration is per-cluster in `config.json`. 3. The first authenticator that returns true performs the actual `Login` 4. JWT tokens are used for API authentication -**Database Migrations**: SQL migrations in `internal/repository/migrations/` are +**Database Migrations**: SQL migrations in `internal/repository/migrations/sqlite3/` are applied automatically on startup. Version tracking in `version` table. **Scopes**: Metrics can be collected at different scopes: @@ -173,7 +173,7 @@ applied automatically on startup. Version tracking in `version` table. **GraphQL** (gqlgen): -- Schema: `api/*.graphqls` +- Schema: `api/schema.graphqls` - Config: `gqlgen.yml` - Generated code: `internal/graph/generated/` - Custom resolvers: `internal/graph/schema.resolvers.go` @@ -182,7 +182,7 @@ applied automatically on startup. Version tracking in `version` table. **Swagger/OpenAPI**: - Annotations in `internal/api/*.go` -- Generated docs: `api/docs.go`, `api/swagger.yaml` +- Generated docs: `internal/api/docs.go`, `api/swagger.yaml` - Run `make swagger` after API changes ## Testing Conventions @@ -196,7 +196,7 @@ applied automatically on startup. Version tracking in `version` table. ### Adding a new GraphQL field -1. Edit schema in `api/*.graphqls` +1. Edit schema in `api/schema.graphqls` 2. Run `make graphql` 3. Implement resolver in `internal/graph/schema.resolvers.go` @@ -215,7 +215,7 @@ applied automatically on startup. Version tracking in `version` table. ### Modifying database schema -1. Create new migration in `internal/repository/migrations/` +1. Create new migration in `internal/repository/migrations/sqlite3/` 2. Increment `repository.Version` 3. Test with fresh database and existing database diff --git a/README.md b/README.md index 475401f4..d01c7140 100644 --- a/README.md +++ b/README.md @@ -173,14 +173,14 @@ ln -s ./var/job-archive Job classification and application detection - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager) Background task management and scheduled jobs + - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstoreclient) + Client for cc-metric-store queries - [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg) contains Go packages that can be used by other projects. - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive) - Job archive backend implementations (filesystem, S3) + Job archive backend implementations (filesystem, S3, SQLite) - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/metricstore) In-memory metric data store with checkpointing and metric loading - - [`nats`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/nats) - NATS client and message handling - [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools) Additional command line helper tools. - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager) diff --git a/internal/archiver/README.md b/internal/archiver/README.md index 48aed797..53d00948 100644 --- a/internal/archiver/README.md +++ b/internal/archiver/README.md @@ -170,7 +170,6 @@ All exported functions are safe for concurrent use: - `Start()` - Safe to call once - `TriggerArchiving()` - Safe from multiple goroutines - `Shutdown()` - Safe to call once -- `WaitForArchiving()` - Deprecated, but safe Internal state is protected by: - Channel synchronization (`archiveChannel`) diff --git a/tools/convert-pem-pubkey/Readme.md b/tools/convert-pem-pubkey/Readme.md index 1429acc4..22fd0db2 100644 --- a/tools/convert-pem-pubkey/Readme.md +++ b/tools/convert-pem-pubkey/Readme.md @@ -16,7 +16,7 @@ CROSS_LOGIN_JWT_PUBLIC_KEY="+51iXX8BdLFocrppRxIw52xCOf8xFSH/eNilN5IHVGc=" Instructions -- `cd tools/convert-pem-pubkey-for-cc/` +- `cd tools/convert-pem-pubkey/` - Insert your public ed25519 PEM key into `dummy.pub` - `go run . dummy.pub` - Copy the result into ClusterCockpit's `.env` diff --git a/web/frontend/README.md b/web/frontend/README.md index d61d302e..4dff4405 100644 --- a/web/frontend/README.md +++ b/web/frontend/README.md @@ -1,11 +1,11 @@ # cc-frontend -[![Build](https://github.com/ClusterCockpit/cc-svelte-datatable/actions/workflows/build.yml/badge.svg)](https://github.com/ClusterCockpit/cc-svelte-datatable/actions/workflows/build.yml) +[![Build](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml) -A frontend for [ClusterCockpit](https://github.com/ClusterCockpit/ClusterCockpit) and [cc-backend](https://github.com/ClusterCockpit/cc-backend). Backend specific configuration can de done using the constants defined in the `intro` section in `./rollup.config.js`. +A frontend for [ClusterCockpit](https://github.com/ClusterCockpit/ClusterCockpit) and [cc-backend](https://github.com/ClusterCockpit/cc-backend). Backend specific configuration can be done using the constants defined in the `intro` section in `./rollup.config.mjs`. Builds on: -* [Svelte](https://svelte.dev/) +* [Svelte 5](https://svelte.dev/) * [SvelteStrap](https://sveltestrap.js.org/) * [Bootstrap 5](https://getbootstrap.com/) * [urql](https://github.com/FormidableLabs/urql) From 49a1748641e7b6875659e6ab44c89791c2df943d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 10 Feb 2026 13:49:23 +0100 Subject: [PATCH 234/341] add not configured info cards, show short job filter options if one active filter --- go.sum | 2 - web/frontend/src/Job.root.svelte | 22 +++++++--- web/frontend/src/Jobs.root.svelte | 3 +- web/frontend/src/Node.root.svelte | 36 +++++++++++------ web/frontend/src/User.root.svelte | 3 +- .../src/generic/joblist/JobListRow.svelte | 11 +++-- web/frontend/src/generic/utils.js | 40 +++++++++++++------ web/frontend/src/systems/NodeOverview.svelte | 19 +++++++-- .../src/systems/nodelist/NodeListRow.svelte | 32 +++++---------- 9 files changed, 105 insertions(+), 63 deletions(-) diff --git a/go.sum b/go.sum index f4f41dfd..a407436f 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/99designs/gqlgen v0.17.85 h1:EkGx3U2FDcxQm8YDLQSpXIAVmpDyZ3IcBMOJi2nH github.com/99designs/gqlgen v0.17.85/go.mod h1:yvs8s0bkQlRfqg03YXr3eR4OQUowVhODT/tHzCXnbOU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/ClusterCockpit/cc-lib/v2 v2.2.1 h1:iCVas+Jc61zFH5S2VG3H1sc7tsn+U4lOJwUYjYZEims= -github.com/ClusterCockpit/cc-lib/v2 v2.2.1/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/ClusterCockpit/cc-lib/v2 v2.2.2 h1:ye4RY57I19c2cXr3XWZBS/QYYgQVeGFvsiu5HkyKq9E= github.com/ClusterCockpit/cc-lib/v2 v2.2.2/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 99dfa7ac..3baed1c1 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -30,7 +30,7 @@ import { init, groupByScope, - checkMetricDisabled, + checkMetricAvailability, } from "./generic/utils.js"; import Metric from "./job/Metric.svelte"; import MetricSelection from "./generic/select/MetricSelection.svelte"; @@ -151,17 +151,17 @@ } return names; }, []); - + // return metricNames.filter( (metric) => !metrics.some((jm) => jm.name == metric) && selectedMetrics.includes(metric) && - !checkMetricDisabled( + (checkMetricAvailability( globalMetrics, metric, thisJob.cluster, thisJob.subCluster, - ), + ) == "configured") ); } else { return [] @@ -212,7 +212,7 @@ inputMetrics.map((metric) => ({ metric: metric, data: grouped.find((group) => group[0].name == metric), - disabled: checkMetricDisabled( + availability: checkMetricAvailability( globalMetrics, metric, thisJob.cluster, @@ -333,7 +333,17 @@ {:else if thisJob && $jobMetrics?.data?.scopedJobStats} {#snippet gridContent(item)} - {#if item?.disabled} + {#if item.availability == "none"} + + + Metric not configured + + +

    No datasets returned for {item.metric}.

    +

    Metric is not configured for cluster {thisJob.cluster}.

    +     +     + {:else if item.availability == "disabled"} Disabled Metric diff --git a/web/frontend/src/Jobs.root.svelte b/web/frontend/src/Jobs.root.svelte index 0d543fc8..a06aee3c 100644 --- a/web/frontend/src/Jobs.root.svelte +++ b/web/frontend/src/Jobs.root.svelte @@ -142,7 +142,8 @@ 0)} shortJobCutoff={ccconfig?.jobList_hideShortRunningJobs} showFilter={!showCompare} matchedJobs={showCompare? matchedCompareJobs: matchedListJobs} diff --git a/web/frontend/src/Node.root.svelte b/web/frontend/src/Node.root.svelte index d3364b49..06056466 100644 --- a/web/frontend/src/Node.root.svelte +++ b/web/frontend/src/Node.root.svelte @@ -32,7 +32,7 @@ } from "@urql/svelte"; import { init, - checkMetricDisabled, + checkMetricAvailability, } from "./generic/utils.js"; import PlotGrid from "./generic/PlotGrid.svelte"; import MetricPlot from "./generic/plots/MetricPlot.svelte"; @@ -242,17 +242,17 @@ {item.name} {systemUnits[item.name] ? "(" + systemUnits[item.name] + ")" : ""} - {#if item.disabled === false && item.metric} - c.name == cluster)} - subCluster={$nodeMetricsData.data.nodeMetrics[0].subCluster} - series={item.metric.series} - enableFlip - forNode - /> - {:else if item.disabled === true && item.metric} + {#if item.availability == "none"} + + + Metric not configured + + +

    No datasets returned for {item.name}.

    +

    Metric is not configured for cluster {cluster}.

    +     +     + {:else if item.availability == "disabled"} Disabled Metric @@ -262,6 +262,16 @@

    Metric has been disabled for subcluster {$nodeMetricsData.data.nodeMetrics[0].subCluster}.

         + {:else if item?.metric} + c.name == cluster)} + subCluster={$nodeMetricsData.data.nodeMetrics[0].subCluster} + series={item.metric.series} + enableFlip + forNode + /> {:else} @@ -279,7 +289,7 @@ items={$nodeMetricsData.data.nodeMetrics[0].metrics .map((m) => ({ ...m, - disabled: checkMetricDisabled( + availability: checkMetricAvailability( globalMetrics, m.name, cluster, diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 4ee3f892..d086df14 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -219,7 +219,8 @@ 0)} shortJobCutoff={ccconfig?.jobList_hideShortRunningJobs} showFilter={!showCompare} matchedJobs={showCompare? matchedCompareJobs: matchedListJobs} diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 5d129ad0..9db340d4 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -19,7 +19,7 @@ @@ -169,7 +152,12 @@ {#each refinedData as metricData, i (metricData?.data?.name || i)} - {#if metricData?.disabled} + {#if metricData?.availability == "none"} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric is not configured for cluster {cluster}.

    +     + {:else if metricData?.availability == "disabled"}

    No dataset(s) returned for {selectedMetrics[i]}

    Metric has been disabled for subcluster {nodeData.subCluster}.

    @@ -177,7 +165,7 @@ {:else if !metricData?.data}

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric was not found in metric store for cluster {cluster}.

    +

    Metric or host was not found in metric store for cluster {cluster}.

     {:else if !!metricData.data?.metric.statisticsSeries} From a5a1fd1a6a8da35bbc3316b4ed73fb8660baaef1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 10 Feb 2026 15:47:38 +0100 Subject: [PATCH 235/341] fix missing component argument --- web/frontend/src/status/dashdetails/StatisticsDash.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/frontend/src/status/dashdetails/StatisticsDash.svelte b/web/frontend/src/status/dashdetails/StatisticsDash.svelte index 42c6823f..2cf8621e 100644 --- a/web/frontend/src/status/dashdetails/StatisticsDash.svelte +++ b/web/frontend/src/status/dashdetails/StatisticsDash.svelte @@ -35,6 +35,7 @@ /* Const Init */ const ccconfig = getContext("cc-config"); + const globalMetrics = getContext("globalMetrics"); const client = getContextClient(); /* State Init */ @@ -139,6 +140,7 @@ Date: Tue, 10 Feb 2026 16:46:18 +0100 Subject: [PATCH 236/341] revert external config supply for nodeList component --- internal/graph/schema.resolvers.go | 3 ++- web/frontend/src/Systems.root.svelte | 4 ++-- web/frontend/src/systems/NodeList.svelte | 17 +++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 19d04eab..a233c3ba 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -867,7 +867,8 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub } nodeMetricsListResult := &model.NodesResultList{ - Items: nodeMetricsList, + Items: nodeMetricsList, + // TotalNodes depends on sum of nodes grouped on latest timestamp, see repo/node.go:357 TotalNodes: &countNodes, HasNextPage: &hasNextPage, } diff --git a/web/frontend/src/Systems.root.svelte b/web/frontend/src/Systems.root.svelte index fb5c4495..d89b5f06 100644 --- a/web/frontend/src/Systems.root.svelte +++ b/web/frontend/src/Systems.root.svelte @@ -272,8 +272,8 @@ {:else} - + {/if} {/if} diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index 4e8b45d9..da196b82 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -4,8 +4,6 @@ Properties: - `cluster String`: The nodes' cluster - `subCluster String`: The nodes' subCluster [Default: ""] - - `ccconfig Object?`: The ClusterCockpit Config Context [Default: null] - - `globalMetrics [Obj]`: Includes the backend supplied availabilities for cluster and subCluster - `pendingSelectedMetrics [String]`: The array of selected metrics [Default []] - `selectedResolution Number?`: The selected data resolution [Default: 0] - `hostnameFilter String?`: The active hostnamefilter [Default: ""] @@ -16,7 +14,7 @@ --> From 8d6c6b819b3f25b8a498930c5709a9f97734a3ac Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 11 Feb 2026 07:06:06 +0100 Subject: [PATCH 238/341] Update and port to cc-lib --- cmd/cc-backend/main.go | 2 +- go.mod | 2 +- go.sum | 2 ++ internal/api/cluster.go | 4 ++-- internal/api/job.go | 4 ++-- internal/api/user.go | 2 +- internal/auth/auth.go | 12 ++++++------ internal/graph/generated/generated.go | 2 +- internal/graph/schema.resolvers.go | 2 +- internal/metricdispatch/dataLoader.go | 2 +- internal/metricdispatch/dataLoader_test.go | 2 +- internal/metricstoreclient/cc-metric-store.go | 6 +++--- internal/repository/jobQuery.go | 2 +- internal/repository/tags.go | 4 ++-- pkg/archive/json.go | 2 +- pkg/archive/parquet/convert.go | 1 - pkg/archive/parquet/writer_test.go | 1 - pkg/metricstore/query.go | 6 +++--- 18 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 3c70a960..3ee05383 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -248,7 +248,7 @@ func generateJWT(authHandle *auth.Authentication, username string) error { return fmt.Errorf("getting user '%s': %w", username, err) } - if !user.HasRole(schema.RoleApi) { + if !user.HasRole(schema.RoleAPI) { cclog.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username) } diff --git a/go.mod b/go.mod index 6bcc3b08..fedc6a22 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ tool ( require ( github.com/99designs/gqlgen v0.17.85 - github.com/ClusterCockpit/cc-lib/v2 v2.2.2 + github.com/ClusterCockpit/cc-lib/v2 v2.3.0 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.6 diff --git a/go.sum b/go.sum index a407436f..5573c63a 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/ClusterCockpit/cc-lib/v2 v2.2.2 h1:ye4RY57I19c2cXr3XWZBS/QYYgQVeGFvsiu5HkyKq9E= github.com/ClusterCockpit/cc-lib/v2 v2.2.2/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= +github.com/ClusterCockpit/cc-lib/v2 v2.3.0 h1:69NqCAYCU1r2w6J5Yuxoe8jfR68VLqtWwsWXZ6KTOo4= +github.com/ClusterCockpit/cc-lib/v2 v2.3.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= diff --git a/internal/api/cluster.go b/internal/api/cluster.go index d1c3c898..5e6e3a27 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -36,9 +36,9 @@ type GetClustersAPIResponse struct { // @router /api/clusters/ [get] func (api *RestAPI) getClusters(rw http.ResponseWriter, r *http.Request) { if user := repository.GetUserFromContext(r.Context()); user != nil && - !user.HasRole(schema.RoleApi) { + !user.HasRole(schema.RoleAPI) { - handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleAPI)), http.StatusForbidden, rw) return } diff --git a/internal/api/job.go b/internal/api/job.go index 9bd93b1c..66258668 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -1054,8 +1054,8 @@ type GetUsedNodesAPIResponse struct { // @router /api/jobs/used_nodes [get] func (api *RestAPI) getUsedNodes(rw http.ResponseWriter, r *http.Request) { if user := repository.GetUserFromContext(r.Context()); user != nil && - !user.HasRole(schema.RoleApi) { - handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw) + !user.HasRole(schema.RoleAPI) { + handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleAPI)), http.StatusForbidden, rw) return } diff --git a/internal/api/user.go b/internal/api/user.go index 5eba0dfc..e2f78165 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -164,7 +164,7 @@ func (api *RestAPI) createUser(rw http.ResponseWriter, r *http.Request) { return } - if len(password) == 0 && role != schema.GetRoleString(schema.RoleApi) { + if len(password) == 0 && role != schema.GetRoleString(schema.RoleAPI) { handleError(fmt.Errorf("only API users are allowed to have a blank password (login will be impossible)"), http.StatusBadRequest, rw) return } diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 8a2073b5..9b1e2121 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -448,13 +448,13 @@ func (auth *Authentication) AuthAPI( if user != nil { switch { case len(user.Roles) == 1: - if user.HasRole(schema.RoleApi) { + if user.HasRole(schema.RoleAPI) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return } case len(user.Roles) >= 2: - if user.HasAllRoles([]schema.Role{schema.RoleAdmin, schema.RoleApi}) { + if user.HasAllRoles([]schema.Role{schema.RoleAdmin, schema.RoleAPI}) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return @@ -484,13 +484,13 @@ func (auth *Authentication) AuthUserAPI( if user != nil { switch { case len(user.Roles) == 1: - if user.HasRole(schema.RoleApi) { + if user.HasRole(schema.RoleAPI) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return } case len(user.Roles) >= 2: - if user.HasRole(schema.RoleApi) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleSupport, schema.RoleAdmin}) { + if user.HasRole(schema.RoleAPI) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleSupport, schema.RoleAdmin}) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return @@ -520,13 +520,13 @@ func (auth *Authentication) AuthMetricStoreAPI( if user != nil { switch { case len(user.Roles) == 1: - if user.HasRole(schema.RoleApi) { + if user.HasRole(schema.RoleAPI) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return } case len(user.Roles) >= 2: - if user.HasRole(schema.RoleApi) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleAdmin}) { + if user.HasRole(schema.RoleAPI) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleAdmin}) { ctx := context.WithValue(r.Context(), repository.ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index e1e5ea71..965fd860 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -10245,7 +10245,7 @@ func (ec *executionContext) _Series_id(ctx context.Context, field graphql.Collec field, ec.fieldContext_Series_id, func(ctx context.Context) (any, error) { - return obj.Id, nil + return obj.ID, nil }, nil, ec.marshalOString2ᚖstring, diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 19d04eab..af04d94d 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -552,7 +552,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [ for _, stat := range stats { mdlStats = append(mdlStats, &model.ScopedStats{ Hostname: stat.Hostname, - ID: stat.Id, + ID: stat.ID, Data: stat.Data, }) } diff --git a/internal/metricdispatch/dataLoader.go b/internal/metricdispatch/dataLoader.go index 78808a74..c420fee4 100644 --- a/internal/metricdispatch/dataLoader.go +++ b/internal/metricdispatch/dataLoader.go @@ -499,7 +499,7 @@ func copyJobMetric(src *schema.JobMetric) *schema.JobMetric { func copySeries(src *schema.Series) schema.Series { dst := schema.Series{ Hostname: src.Hostname, - Id: src.Id, + ID: src.ID, Statistics: src.Statistics, Data: make([]schema.Float, len(src.Data)), } diff --git a/internal/metricdispatch/dataLoader_test.go b/internal/metricdispatch/dataLoader_test.go index c4841f8d..65a366f9 100644 --- a/internal/metricdispatch/dataLoader_test.go +++ b/internal/metricdispatch/dataLoader_test.go @@ -21,7 +21,7 @@ func TestDeepCopy(t *testing.T) { Series: []schema.Series{ { Hostname: "node001", - Id: &nodeId, + ID: &nodeId, Data: []schema.Float{1.0, 2.0, 3.0}, Statistics: schema.MetricStatistics{ Min: 1.0, diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go index aadbe1b1..4472b825 100644 --- a/internal/metricstoreclient/cc-metric-store.go +++ b/internal/metricstoreclient/cc-metric-store.go @@ -267,7 +267,7 @@ func (ccms *CCMetricStore) LoadData( jobMetric.Series = append(jobMetric.Series, schema.Series{ Hostname: query.Hostname, - Id: id, + ID: id, Statistics: schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), @@ -419,7 +419,7 @@ func (ccms *CCMetricStore) LoadScopedStats( scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ Hostname: query.Hostname, - Id: id, + ID: id, Data: &schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), @@ -634,7 +634,7 @@ func (ccms *CCMetricStore) LoadNodeListData( scopeData.Series = append(scopeData.Series, schema.Series{ Hostname: query.Hostname, - Id: id, + ID: id, Statistics: schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 658413e8..81779583 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -150,7 +150,7 @@ func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.Select } switch { - case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): + case len(user.Roles) == 1 && user.HasRole(schema.RoleAPI): return query, nil case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): return query, nil diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 612666da..943dda66 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -644,12 +644,12 @@ func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scop if user != nil { switch { case operation == "write" && scope == "admin": - if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) { + if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleAPI)) { return true, nil } return false, nil case operation == "write" && scope == "global": - if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) { + if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) || (len(user.Roles) == 1 && user.HasRole(schema.RoleAPI)) { return true, nil } return false, nil diff --git a/pkg/archive/json.go b/pkg/archive/json.go index cf1b0a38..dd37075d 100644 --- a/pkg/archive/json.go +++ b/pkg/archive/json.go @@ -51,7 +51,7 @@ func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) { for _, series := range jobMetric.Series { scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ Hostname: series.Hostname, - Id: series.Id, + ID: series.ID, Data: &series.Statistics, }) } diff --git a/pkg/archive/parquet/convert.go b/pkg/archive/parquet/convert.go index ceaa3f2f..ba1e76eb 100644 --- a/pkg/archive/parquet/convert.go +++ b/pkg/archive/parquet/convert.go @@ -81,7 +81,6 @@ func JobToParquetRow(meta *schema.Job, data *schema.JobData) (*ParquetJobRow, er NumNodes: meta.NumNodes, NumHWThreads: meta.NumHWThreads, NumAcc: meta.NumAcc, - Exclusive: meta.Exclusive, Energy: meta.Energy, SMT: meta.SMT, ResourcesJSON: resourcesJSON, diff --git a/pkg/archive/parquet/writer_test.go b/pkg/archive/parquet/writer_test.go index 6baaa527..e532e472 100644 --- a/pkg/archive/parquet/writer_test.go +++ b/pkg/archive/parquet/writer_test.go @@ -47,7 +47,6 @@ func makeTestJob(jobID int64) (*schema.Job, *schema.JobData) { Walltime: 7200, NumNodes: 2, NumHWThreads: 16, - Exclusive: 1, SMT: 1, Resources: []*schema.Resource{ {Hostname: "node001"}, diff --git a/pkg/metricstore/query.go b/pkg/metricstore/query.go index e5a49af3..709a9710 100644 --- a/pkg/metricstore/query.go +++ b/pkg/metricstore/query.go @@ -149,7 +149,7 @@ func (ccms *InternalMetricStore) LoadData( jobMetric.Series = append(jobMetric.Series, schema.Series{ Hostname: query.Hostname, - Id: id, + ID: id, Statistics: schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), @@ -651,7 +651,7 @@ func (ccms *InternalMetricStore) LoadScopedStats( scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ Hostname: query.Hostname, - Id: id, + ID: id, Data: &schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), @@ -894,7 +894,7 @@ func (ccms *InternalMetricStore) LoadNodeListData( scopeData.Series = append(scopeData.Series, schema.Series{ Hostname: query.Hostname, - Id: id, + ID: id, Statistics: schema.MetricStatistics{ Avg: float64(res.Avg), Min: float64(res.Min), From 12e9f6700efbe20181eb4c9f653b0c2368e7de76 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 11 Feb 2026 16:16:09 +0100 Subject: [PATCH 239/341] fix nodeList resolver data handling, increase nodestate filter cutoff - add comment on cutoff --- internal/graph/schema.resolvers.go | 8 +++++--- internal/repository/node.go | 12 ++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index eb565b7b..059bd16d 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -824,6 +824,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub } nodeRepo := repository.GetNodeRepository() + // nodes -> array hostname nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page) if nerr != nil { return nil, errors.New("could not retrieve node list required for resolving NodeMetricsList") @@ -835,6 +836,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub } } + // data -> map hostname:jobdata data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx) if err != nil { cclog.Warn("error while loading node data (Resolver.NodeMetricsList") @@ -842,18 +844,18 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub } nodeMetricsList := make([]*model.NodeMetrics, 0, len(data)) - for hostname, metrics := range data { + for _, hostname := range nodes { host := &model.NodeMetrics{ Host: hostname, State: stateMap[hostname], - Metrics: make([]*model.JobMetricWithName, 0, len(metrics)*len(scopes)), + Metrics: make([]*model.JobMetricWithName, 0), } host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname) if err != nil { cclog.Warnf("error in nodeMetrics resolver: %s", err) } - for metric, scopedMetrics := range metrics { + for metric, scopedMetrics := range data[hostname] { for scope, scopedMetric := range scopedMetrics { host.Metrics = append(host.Metrics, &model.JobMetricWithName{ Name: metric, diff --git a/internal/repository/node.go b/internal/repository/node.go index df3aec8b..42e7b101 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -263,14 +263,16 @@ func (r *NodeRepository) QueryNodes( if f.SchedulerState != nil { query = query.Where("node_state = ?", f.SchedulerState) // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned + // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 60)}) + query = query.Where(sq.Gt{"time_stamp": (now - 300)}) } if f.HealthState != nil { query = query.Where("health_state = ?", f.HealthState) // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned + // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 60)}) + query = query.Where(sq.Gt{"time_stamp": (now - 300)}) } } @@ -331,14 +333,16 @@ func (r *NodeRepository) CountNodes( if f.SchedulerState != nil { query = query.Where("node_state = ?", f.SchedulerState) // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned + // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 60)}) + query = query.Where(sq.Gt{"time_stamp": (now - 300)}) } if f.HealthState != nil { query = query.Where("health_state = ?", f.HealthState) // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned + // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 60)}) + query = query.Where(sq.Gt{"time_stamp": (now - 300)}) } } From e75da7f8cc1376ed835f496fa85173e8f28e3ffc Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 11 Feb 2026 18:32:29 +0100 Subject: [PATCH 240/341] fix reactivity key placement in nodeList --- .../src/systems/nodelist/NodeListRow.svelte | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index b5bb9d77..46f8c4a4 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -151,24 +151,25 @@ {/if} {#each refinedData as metricData, i (metricData?.data?.name || i)} - - {#if metricData?.availability == "none"} + {#key metricData} + + {#if metricData?.availability == "none"}

    No dataset(s) returned for {selectedMetrics[i]}

    Metric is not configured for cluster {cluster}.

     - {:else if metricData?.availability == "disabled"} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric has been disabled for subcluster {nodeData.subCluster}.

    -     - {:else if !metricData?.data} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric or host was not found in metric store for cluster {cluster}.

    -     - {:else if !!metricData.data?.metric.statisticsSeries} - + {:else if metricData?.availability == "disabled"} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric has been disabled for subcluster {nodeData.subCluster}.

    +     + {:else if !metricData?.data} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric or host was not found in metric store for cluster {cluster}.

    +     + {:else if !!metricData.data?.metric.statisticsSeries} + -
    - {#key extendedLegendData} +
    - {/key} - {:else} + {:else} - {/if} - + {/if} + + {/key} {/each} From f4ee0d10424d7153f8f59ac880cabe176fba5d1c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 12 Feb 2026 07:34:24 +0100 Subject: [PATCH 241/341] Update cc-lib and extend nodestate sql schema --- go.mod | 2 +- go.sum | 2 ++ internal/repository/migrations/sqlite3/10_node-table.up.sql | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fedc6a22..f9bf7e42 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ tool ( require ( github.com/99designs/gqlgen v0.17.85 - github.com/ClusterCockpit/cc-lib/v2 v2.3.0 + github.com/ClusterCockpit/cc-lib/v2 v2.4.0 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.6 diff --git a/go.sum b/go.sum index 5573c63a..8db4d1a3 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/ClusterCockpit/cc-lib/v2 v2.2.2 h1:ye4RY57I19c2cXr3XWZBS/QYYgQVeGFvsi github.com/ClusterCockpit/cc-lib/v2 v2.2.2/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/ClusterCockpit/cc-lib/v2 v2.3.0 h1:69NqCAYCU1r2w6J5Yuxoe8jfR68VLqtWwsWXZ6KTOo4= github.com/ClusterCockpit/cc-lib/v2 v2.3.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= +github.com/ClusterCockpit/cc-lib/v2 v2.4.0 h1:OnZlvqSatg7yCQ2NtSR7AddpUVSiuSMZ8scF1a7nfOk= +github.com/ClusterCockpit/cc-lib/v2 v2.4.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= diff --git a/internal/repository/migrations/sqlite3/10_node-table.up.sql b/internal/repository/migrations/sqlite3/10_node-table.up.sql index 7b5b5ac7..fd118f5d 100644 --- a/internal/repository/migrations/sqlite3/10_node-table.up.sql +++ b/internal/repository/migrations/sqlite3/10_node-table.up.sql @@ -23,6 +23,7 @@ CREATE TABLE "node_state" ( CHECK (health_state IN ( 'full', 'partial', 'failed' )), + health_metrics TEXT, -- JSON array of strings node_id INTEGER, FOREIGN KEY (node_id) REFERENCES node (id) ); From 865cd3db54cbb0ee7713b2fcd7c0ef2dd98e46b4 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 12 Feb 2026 08:48:15 +0100 Subject: [PATCH 242/341] Prersist faulty nodestate metric lists to db --- go.mod | 2 ++ go.sum | 6 ------ internal/api/node.go | 18 +++++++++++------ internal/repository/node.go | 7 ++++--- pkg/metricstore/healthcheck.go | 31 +++++++++++++++++++++++------ pkg/metricstore/metricstore_test.go | 4 ++-- 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index f9bf7e42..b35eafe5 100644 --- a/go.mod +++ b/go.mod @@ -124,3 +124,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +replace github.com/ClusterCockpit/cc-lib/v2 => ../cc-lib diff --git a/go.sum b/go.sum index 8db4d1a3..d5bbe045 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,6 @@ github.com/99designs/gqlgen v0.17.85 h1:EkGx3U2FDcxQm8YDLQSpXIAVmpDyZ3IcBMOJi2nH github.com/99designs/gqlgen v0.17.85/go.mod h1:yvs8s0bkQlRfqg03YXr3eR4OQUowVhODT/tHzCXnbOU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/ClusterCockpit/cc-lib/v2 v2.2.2 h1:ye4RY57I19c2cXr3XWZBS/QYYgQVeGFvsiu5HkyKq9E= -github.com/ClusterCockpit/cc-lib/v2 v2.2.2/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= -github.com/ClusterCockpit/cc-lib/v2 v2.3.0 h1:69NqCAYCU1r2w6J5Yuxoe8jfR68VLqtWwsWXZ6KTOo4= -github.com/ClusterCockpit/cc-lib/v2 v2.3.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= -github.com/ClusterCockpit/cc-lib/v2 v2.4.0 h1:OnZlvqSatg7yCQ2NtSR7AddpUVSiuSMZ8scF1a7nfOk= -github.com/ClusterCockpit/cc-lib/v2 v2.4.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= diff --git a/internal/api/node.go b/internal/api/node.go index 27cde7f0..e6b19479 100644 --- a/internal/api/node.go +++ b/internal/api/node.go @@ -80,7 +80,7 @@ func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) { ms := metricstore.GetMemoryStore() m := make(map[string][]string) - healthStates := make(map[string]schema.MonitoringState) + healthResults := make(map[string]metricstore.HealthCheckResult) startMs := time.Now() @@ -94,8 +94,8 @@ func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) { if sc != "" { metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc) metricNames := metricListToNames(metricList) - if states, err := ms.HealthCheck(req.Cluster, nl, metricNames); err == nil { - maps.Copy(healthStates, states) + if results, err := ms.HealthCheck(req.Cluster, nl, metricNames); err == nil { + maps.Copy(healthResults, results) } } } @@ -106,8 +106,10 @@ func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) { for _, node := range req.Nodes { state := determineState(node.States) healthState := schema.MonitoringStateFailed - if hs, ok := healthStates[node.Hostname]; ok { - healthState = hs + var healthMetrics string + if result, ok := healthResults[node.Hostname]; ok { + healthState = result.State + healthMetrics = result.HealthMetrics } nodeState := schema.NodeStateDB{ TimeStamp: requestReceived, @@ -116,10 +118,14 @@ func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) { MemoryAllocated: node.MemoryAllocated, GpusAllocated: node.GpusAllocated, HealthState: healthState, + HealthMetrics: healthMetrics, JobsRunning: node.JobsRunning, } - repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState) + if err := repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState); err != nil { + cclog.Errorf("updateNodeStates: updating node state for %s on %s failed: %v", + node.Hostname, req.Cluster, err) + } } cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB)) diff --git a/internal/repository/node.go b/internal/repository/node.go index 42e7b101..82dcf067 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -169,9 +169,10 @@ func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) { } const NamedNodeStateInsert string = ` -INSERT INTO node_state (time_stamp, node_state, health_state, cpus_allocated, - memory_allocated, gpus_allocated, jobs_running, node_id) - VALUES (:time_stamp, :node_state, :health_state, :cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);` +INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics, + cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id) + VALUES (:time_stamp, :node_state, :health_state, :health_metrics, + :cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);` // TODO: Add real Monitoring Health State diff --git a/pkg/metricstore/healthcheck.go b/pkg/metricstore/healthcheck.go index ed1ff38e..d6def692 100644 --- a/pkg/metricstore/healthcheck.go +++ b/pkg/metricstore/healthcheck.go @@ -6,6 +6,7 @@ package metricstore import ( + "encoding/json" "fmt" "time" @@ -19,6 +20,13 @@ type HealthCheckResponse struct { Error error } +// HealthCheckResult holds the monitoring state and raw JSON health metrics +// for a single node as determined by HealthCheck. +type HealthCheckResult struct { + State schema.MonitoringState + HealthMetrics string // JSON: {"missing":[...],"degraded":[...]} +} + // MaxMissingDataPoints is the threshold for stale data detection. // A buffer is considered healthy if the gap between its last data point // and the current time is within MaxMissingDataPoints * frequency. @@ -134,15 +142,15 @@ func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []str // - MonitoringStateFailed: node not found, or no healthy metrics at all func (m *MemoryStore) HealthCheck(cluster string, nodes []string, expectedMetrics []string, -) (map[string]schema.MonitoringState, error) { - results := make(map[string]schema.MonitoringState, len(nodes)) +) (map[string]HealthCheckResult, error) { + results := make(map[string]HealthCheckResult, len(nodes)) for _, hostname := range nodes { selector := []string{cluster, hostname} degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics) if err != nil { - results[hostname] = schema.MonitoringStateFailed + results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed} continue } @@ -158,13 +166,24 @@ func (m *MemoryStore) HealthCheck(cluster string, cclog.ComponentInfo("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList) } + var state schema.MonitoringState switch { case degradedCount == 0 && missingCount == 0: - results[hostname] = schema.MonitoringStateFull + state = schema.MonitoringStateFull case healthyCount == 0: - results[hostname] = schema.MonitoringStateFailed + state = schema.MonitoringStateFailed default: - results[hostname] = schema.MonitoringStatePartial + state = schema.MonitoringStatePartial + } + + hm, _ := json.Marshal(map[string][]string{ + "missing": missingList, + "degraded": degradedList, + }) + + results[hostname] = HealthCheckResult{ + State: state, + HealthMetrics: string(hm), } } diff --git a/pkg/metricstore/metricstore_test.go b/pkg/metricstore/metricstore_test.go index 4d68d76c..a9ff0055 100644 --- a/pkg/metricstore/metricstore_test.go +++ b/pkg/metricstore/metricstore_test.go @@ -253,8 +253,8 @@ func TestHealthCheck(t *testing.T) { // Check status if wantStatus, ok := tt.wantStates[node]; ok { - if state != wantStatus { - t.Errorf("HealthCheck() node %s status = %v, want %v", node, state, wantStatus) + if state.State != wantStatus { + t.Errorf("HealthCheck() node %s status = %v, want %v", node, state.State, wantStatus) } } } From 54ea5d790054dc87b82d43b61b9ac6b180f36684 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 12 Feb 2026 09:21:44 +0100 Subject: [PATCH 243/341] Add nodestate retention and archiving --- internal/config/config.go | 17 +++ internal/config/schema.go | 53 ++++++++ internal/repository/node.go | 65 ++++++++++ .../taskmanager/nodestateRetentionService.go | 120 ++++++++++++++++++ internal/taskmanager/taskManager.go | 21 +++ pkg/archive/parquet/nodestate_schema.go | 20 +++ pkg/archive/parquet/nodestate_writer.go | 104 +++++++++++++++ 7 files changed, 400 insertions(+) create mode 100644 internal/taskmanager/nodestateRetentionService.go create mode 100644 pkg/archive/parquet/nodestate_schema.go create mode 100644 pkg/archive/parquet/nodestate_writer.go diff --git a/internal/config/config.go b/internal/config/config.go index 4e6fe975..2e601ed7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -71,6 +71,23 @@ type ProgramConfig struct { // If exists, will enable dynamic zoom in frontend metric plots using the configured values EnableResampling *ResampleConfig `json:"resampling"` + + // Node state retention configuration + NodeStateRetention *NodeStateRetention `json:"nodestate-retention"` +} + +type NodeStateRetention struct { + Policy string `json:"policy"` // "delete" or "parquet" + Age int `json:"age"` // hours, default 24 + TargetKind string `json:"target-kind"` // "file" or "s3" + TargetPath string `json:"target-path"` + TargetEndpoint string `json:"target-endpoint"` + TargetBucket string `json:"target-bucket"` + TargetAccessKey string `json:"target-access-key"` + TargetSecretKey string `json:"target-secret-key"` + TargetRegion string `json:"target-region"` + TargetUsePathStyle bool `json:"target-use-path-style"` + MaxFileSizeMB int `json:"max-file-size-mb"` } type ResampleConfig struct { diff --git a/internal/config/schema.go b/internal/config/schema.go index 0d575b3c..bd1b314e 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -130,6 +130,59 @@ var configSchema = ` } }, "required": ["subject-job-event", "subject-node-state"] + }, + "nodestate-retention": { + "description": "Node state retention configuration for cleaning up old node_state rows.", + "type": "object", + "properties": { + "policy": { + "description": "Retention policy: 'delete' to remove old rows, 'parquet' to archive then delete.", + "type": "string", + "enum": ["delete", "parquet"] + }, + "age": { + "description": "Retention age in hours (default: 24).", + "type": "integer" + }, + "target-kind": { + "description": "Target kind for parquet archiving: 'file' or 's3'.", + "type": "string", + "enum": ["file", "s3"] + }, + "target-path": { + "description": "Filesystem path for parquet file target.", + "type": "string" + }, + "target-endpoint": { + "description": "S3 endpoint URL.", + "type": "string" + }, + "target-bucket": { + "description": "S3 bucket name.", + "type": "string" + }, + "target-access-key": { + "description": "S3 access key.", + "type": "string" + }, + "target-secret-key": { + "description": "S3 secret key.", + "type": "string" + }, + "target-region": { + "description": "S3 region.", + "type": "string" + }, + "target-use-path-style": { + "description": "Use path-style S3 addressing.", + "type": "boolean" + }, + "max-file-size-mb": { + "description": "Maximum parquet file size in MB (default: 128).", + "type": "integer" + } + }, + "required": ["policy"] } } }` diff --git a/internal/repository/node.go b/internal/repository/node.go index 82dcf067..a746182b 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -225,6 +225,71 @@ func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeSt // return nil // } +// NodeStateWithNode combines a node state row with denormalized node info. +type NodeStateWithNode struct { + ID int64 `db:"id"` + TimeStamp int64 `db:"time_stamp"` + NodeState string `db:"node_state"` + HealthState string `db:"health_state"` + HealthMetrics string `db:"health_metrics"` + CpusAllocated int `db:"cpus_allocated"` + MemoryAllocated int64 `db:"memory_allocated"` + GpusAllocated int `db:"gpus_allocated"` + JobsRunning int `db:"jobs_running"` + Hostname string `db:"hostname"` + Cluster string `db:"cluster"` + SubCluster string `db:"subcluster"` +} + +// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff, +// joined with node info for denormalized archiving. +func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode, error) { + rows, err := sq.Select( + "node_state.id", "node_state.time_stamp", "node_state.node_state", + "node_state.health_state", "node_state.health_metrics", + "node_state.cpus_allocated", "node_state.memory_allocated", + "node_state.gpus_allocated", "node_state.jobs_running", + "node.hostname", "node.cluster", "node.subcluster", + ). + From("node_state"). + Join("node ON node_state.node_id = node.id"). + Where(sq.Lt{"node_state.time_stamp": cutoff}). + Where("node_state.id NOT IN (SELECT MAX(id) FROM node_state GROUP BY node_id)"). + OrderBy("node_state.time_stamp ASC"). + RunWith(r.DB).Query() + if err != nil { + return nil, err + } + defer rows.Close() + + var result []NodeStateWithNode + for rows.Next() { + var ns NodeStateWithNode + if err := rows.Scan(&ns.ID, &ns.TimeStamp, &ns.NodeState, + &ns.HealthState, &ns.HealthMetrics, + &ns.CpusAllocated, &ns.MemoryAllocated, + &ns.GpusAllocated, &ns.JobsRunning, + &ns.Hostname, &ns.Cluster, &ns.SubCluster); err != nil { + return nil, err + } + result = append(result, ns) + } + return result, nil +} + +// DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff, +// but always preserves the latest row per node_id. +func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) { + res, err := r.DB.Exec( + `DELETE FROM node_state WHERE time_stamp < ? AND id NOT IN (SELECT MAX(id) FROM node_state GROUP BY node_id)`, + cutoff, + ) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + func (r *NodeRepository) DeleteNode(id int64) error { _, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id) if err != nil { diff --git a/internal/taskmanager/nodestateRetentionService.go b/internal/taskmanager/nodestateRetentionService.go new file mode 100644 index 00000000..9a704502 --- /dev/null +++ b/internal/taskmanager/nodestateRetentionService.go @@ -0,0 +1,120 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package taskmanager + +import ( + "time" + + "github.com/ClusterCockpit/cc-backend/internal/config" + "github.com/ClusterCockpit/cc-backend/internal/repository" + pqarchive "github.com/ClusterCockpit/cc-backend/pkg/archive/parquet" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/go-co-op/gocron/v2" +) + +func RegisterNodeStateRetentionDeleteService(ageHours int) { + cclog.Info("Register node state retention delete service") + + s.NewJob(gocron.DurationJob(1*time.Hour), + gocron.NewTask( + func() { + cutoff := time.Now().Unix() - int64(ageHours*3600) + nodeRepo := repository.GetNodeRepository() + cnt, err := nodeRepo.DeleteNodeStatesBefore(cutoff) + if err != nil { + cclog.Errorf("NodeState retention: error deleting old rows: %v", err) + } else if cnt > 0 { + cclog.Infof("NodeState retention: deleted %d old rows", cnt) + } + })) +} + +func RegisterNodeStateRetentionParquetService(cfg *config.NodeStateRetention) { + cclog.Info("Register node state retention parquet service") + + maxFileSizeMB := cfg.MaxFileSizeMB + if maxFileSizeMB <= 0 { + maxFileSizeMB = 128 + } + + ageHours := cfg.Age + if ageHours <= 0 { + ageHours = 24 + } + + var target pqarchive.ParquetTarget + var err error + + switch cfg.TargetKind { + case "s3": + target, err = pqarchive.NewS3Target(pqarchive.S3TargetConfig{ + Endpoint: cfg.TargetEndpoint, + Bucket: cfg.TargetBucket, + AccessKey: cfg.TargetAccessKey, + SecretKey: cfg.TargetSecretKey, + Region: cfg.TargetRegion, + UsePathStyle: cfg.TargetUsePathStyle, + }) + default: + target, err = pqarchive.NewFileTarget(cfg.TargetPath) + } + + if err != nil { + cclog.Errorf("NodeState parquet retention: failed to create target: %v", err) + return + } + + s.NewJob(gocron.DurationJob(1*time.Hour), + gocron.NewTask( + func() { + cutoff := time.Now().Unix() - int64(ageHours*3600) + nodeRepo := repository.GetNodeRepository() + + rows, err := nodeRepo.FindNodeStatesBefore(cutoff) + if err != nil { + cclog.Errorf("NodeState parquet retention: error finding rows: %v", err) + return + } + if len(rows) == 0 { + return + } + + cclog.Infof("NodeState parquet retention: archiving %d rows", len(rows)) + pw := pqarchive.NewNodeStateParquetWriter(target, maxFileSizeMB) + + for _, ns := range rows { + row := pqarchive.ParquetNodeStateRow{ + TimeStamp: ns.TimeStamp, + NodeState: ns.NodeState, + HealthState: ns.HealthState, + HealthMetrics: ns.HealthMetrics, + CpusAllocated: int32(ns.CpusAllocated), + MemoryAllocated: ns.MemoryAllocated, + GpusAllocated: int32(ns.GpusAllocated), + JobsRunning: int32(ns.JobsRunning), + Hostname: ns.Hostname, + Cluster: ns.Cluster, + SubCluster: ns.SubCluster, + } + if err := pw.AddRow(row); err != nil { + cclog.Errorf("NodeState parquet retention: add row: %v", err) + continue + } + } + + if err := pw.Close(); err != nil { + cclog.Errorf("NodeState parquet retention: close writer: %v", err) + return + } + + cnt, err := nodeRepo.DeleteNodeStatesBefore(cutoff) + if err != nil { + cclog.Errorf("NodeState parquet retention: error deleting rows: %v", err) + } else { + cclog.Infof("NodeState parquet retention: deleted %d rows from db", cnt) + } + })) +} diff --git a/internal/taskmanager/taskManager.go b/internal/taskmanager/taskManager.go index e323557b..8cf6b4e6 100644 --- a/internal/taskmanager/taskManager.go +++ b/internal/taskmanager/taskManager.go @@ -144,9 +144,30 @@ func Start(cronCfg, archiveConfig json.RawMessage) { RegisterUpdateDurationWorker() RegisterCommitJobService() + if config.Keys.NodeStateRetention != nil && config.Keys.NodeStateRetention.Policy != "" { + initNodeStateRetention() + } + s.Start() } +func initNodeStateRetention() { + cfg := config.Keys.NodeStateRetention + age := cfg.Age + if age <= 0 { + age = 24 + } + + switch cfg.Policy { + case "delete": + RegisterNodeStateRetentionDeleteService(age) + case "parquet": + RegisterNodeStateRetentionParquetService(cfg) + default: + cclog.Warnf("Unknown nodestate-retention policy: %s", cfg.Policy) + } +} + // Shutdown stops the task manager and its scheduler. func Shutdown() { if s != nil { diff --git a/pkg/archive/parquet/nodestate_schema.go b/pkg/archive/parquet/nodestate_schema.go new file mode 100644 index 00000000..c9dfe363 --- /dev/null +++ b/pkg/archive/parquet/nodestate_schema.go @@ -0,0 +1,20 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package parquet + +type ParquetNodeStateRow struct { + TimeStamp int64 `parquet:"time_stamp"` + NodeState string `parquet:"node_state"` + HealthState string `parquet:"health_state"` + HealthMetrics string `parquet:"health_metrics,optional"` + CpusAllocated int32 `parquet:"cpus_allocated"` + MemoryAllocated int64 `parquet:"memory_allocated"` + GpusAllocated int32 `parquet:"gpus_allocated"` + JobsRunning int32 `parquet:"jobs_running"` + Hostname string `parquet:"hostname"` + Cluster string `parquet:"cluster"` + SubCluster string `parquet:"subcluster"` +} diff --git a/pkg/archive/parquet/nodestate_writer.go b/pkg/archive/parquet/nodestate_writer.go new file mode 100644 index 00000000..053417d6 --- /dev/null +++ b/pkg/archive/parquet/nodestate_writer.go @@ -0,0 +1,104 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package parquet + +import ( + "bytes" + "fmt" + "time" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + pq "github.com/parquet-go/parquet-go" +) + +// NodeStateParquetWriter batches ParquetNodeStateRows and flushes them to a target +// when the estimated size exceeds maxSizeBytes. +type NodeStateParquetWriter struct { + target ParquetTarget + maxSizeBytes int64 + rows []ParquetNodeStateRow + currentSize int64 + fileCounter int + datePrefix string +} + +// NewNodeStateParquetWriter creates a new writer for node state parquet files. +func NewNodeStateParquetWriter(target ParquetTarget, maxSizeMB int) *NodeStateParquetWriter { + return &NodeStateParquetWriter{ + target: target, + maxSizeBytes: int64(maxSizeMB) * 1024 * 1024, + datePrefix: time.Now().Format("2006-01-02"), + } +} + +// AddRow adds a row to the current batch. If the estimated batch size +// exceeds the configured maximum, the batch is flushed first. +func (pw *NodeStateParquetWriter) AddRow(row ParquetNodeStateRow) error { + rowSize := estimateNodeStateRowSize(&row) + + if pw.currentSize+rowSize > pw.maxSizeBytes && len(pw.rows) > 0 { + if err := pw.Flush(); err != nil { + return err + } + } + + pw.rows = append(pw.rows, row) + pw.currentSize += rowSize + return nil +} + +// Flush writes the current batch to a parquet file on the target. +func (pw *NodeStateParquetWriter) Flush() error { + if len(pw.rows) == 0 { + return nil + } + + pw.fileCounter++ + fileName := fmt.Sprintf("cc-nodestate-%s-%03d.parquet", pw.datePrefix, pw.fileCounter) + + data, err := writeNodeStateParquetBytes(pw.rows) + if err != nil { + return fmt.Errorf("write parquet buffer: %w", err) + } + + if err := pw.target.WriteFile(fileName, data); err != nil { + return fmt.Errorf("write parquet file %q: %w", fileName, err) + } + + cclog.Infof("NodeState retention: wrote %s (%d rows, %d bytes)", fileName, len(pw.rows), len(data)) + pw.rows = pw.rows[:0] + pw.currentSize = 0 + return nil +} + +// Close flushes any remaining rows and finalizes the writer. +func (pw *NodeStateParquetWriter) Close() error { + return pw.Flush() +} + +func writeNodeStateParquetBytes(rows []ParquetNodeStateRow) ([]byte, error) { + var buf bytes.Buffer + + writer := pq.NewGenericWriter[ParquetNodeStateRow](&buf, + pq.Compression(&pq.Snappy), + ) + + if _, err := writer.Write(rows); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func estimateNodeStateRowSize(row *ParquetNodeStateRow) int64 { + size := int64(100) // fixed numeric fields + size += int64(len(row.NodeState) + len(row.HealthState) + len(row.HealthMetrics)) + size += int64(len(row.Hostname) + len(row.Cluster) + len(row.SubCluster)) + return size +} From f016bd42325911e90122eee46ac77ac6cdd56908 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 12 Feb 2026 09:30:14 +0100 Subject: [PATCH 244/341] Extend node repository unit tests --- go.mod | 2 - go.sum | 2 + internal/repository/node.go | 10 ++- internal/repository/node_test.go | 146 ++++++++++++++++++++++++++++++- 4 files changed, 151 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b35eafe5..f9bf7e42 100644 --- a/go.mod +++ b/go.mod @@ -124,5 +124,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) - -replace github.com/ClusterCockpit/cc-lib/v2 => ../cc-lib diff --git a/go.sum b/go.sum index d5bbe045..509c659c 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/99designs/gqlgen v0.17.85 h1:EkGx3U2FDcxQm8YDLQSpXIAVmpDyZ3IcBMOJi2nH github.com/99designs/gqlgen v0.17.85/go.mod h1:yvs8s0bkQlRfqg03YXr3eR4OQUowVhODT/tHzCXnbOU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/ClusterCockpit/cc-lib/v2 v2.4.0 h1:OnZlvqSatg7yCQ2NtSR7AddpUVSiuSMZ8scF1a7nfOk= +github.com/ClusterCockpit/cc-lib/v2 v2.4.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= diff --git a/internal/repository/node.go b/internal/repository/node.go index a746182b..08a694c6 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -254,7 +254,7 @@ func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode From("node_state"). Join("node ON node_state.node_id = node.id"). Where(sq.Lt{"node_state.time_stamp": cutoff}). - Where("node_state.id NOT IN (SELECT MAX(id) FROM node_state GROUP BY node_id)"). + Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))"). OrderBy("node_state.time_stamp ASC"). RunWith(r.DB).Query() if err != nil { @@ -278,10 +278,14 @@ func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode } // DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff, -// but always preserves the latest row per node_id. +// but always preserves the row with the latest timestamp per node_id. func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) { res, err := r.DB.Exec( - `DELETE FROM node_state WHERE time_stamp < ? AND id NOT IN (SELECT MAX(id) FROM node_state GROUP BY node_id)`, + `DELETE FROM node_state WHERE time_stamp < ? + AND id NOT IN ( + SELECT id FROM node_state ns2 + WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id) + )`, cutoff, ) if err != nil { diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index 4286ab34..d1e86b9a 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -156,8 +156,12 @@ func nodeTestSetup(t *testing.T) { func TestUpdateNodeState(t *testing.T) { nodeTestSetup(t) + repo := GetNodeRepository() + now := time.Now().Unix() + nodeState := schema.NodeStateDB{ - TimeStamp: time.Now().Unix(), NodeState: "allocated", + TimeStamp: now, + NodeState: "allocated", CpusAllocated: 72, MemoryAllocated: 480, GpusAllocated: 0, @@ -165,18 +169,152 @@ func TestUpdateNodeState(t *testing.T) { JobsRunning: 1, } - repo := GetNodeRepository() err := repo.UpdateNodeState("host124", "testcluster", &nodeState) if err != nil { - return + t.Fatal(err) } node, err := repo.GetNode("host124", "testcluster", false) if err != nil { - return + t.Fatal(err) } if node.NodeState != "allocated" { t.Errorf("wrong node state\ngot: %s \nwant: allocated ", node.NodeState) } + + t.Run("FindBeforeEmpty", func(t *testing.T) { + // Only the current-timestamp row exists, so nothing should be found before now + rows, err := repo.FindNodeStatesBefore(now) + if err != nil { + t.Fatal(err) + } + if len(rows) != 0 { + t.Errorf("expected 0 rows, got %d", len(rows)) + } + }) + + t.Run("DeleteOldRows", func(t *testing.T) { + // Insert 2 more old rows for host124 + for i, ts := range []int64{now - 7200, now - 3600} { + ns := schema.NodeStateDB{ + TimeStamp: ts, + NodeState: "allocated", + HealthState: schema.MonitoringStateFull, + CpusAllocated: 72, + MemoryAllocated: 480, + JobsRunning: i, + } + if err := repo.UpdateNodeState("host124", "testcluster", &ns); err != nil { + t.Fatal(err) + } + } + + // Delete rows older than 30 minutes + cutoff := now - 1800 + cnt, err := repo.DeleteNodeStatesBefore(cutoff) + if err != nil { + t.Fatal(err) + } + + // Should delete the 2 old rows + if cnt != 2 { + t.Errorf("expected 2 deleted rows, got %d", cnt) + } + + // Latest row should still exist + node, err := repo.GetNode("host124", "testcluster", false) + if err != nil { + t.Fatal(err) + } + if node.NodeState != "allocated" { + t.Errorf("expected node state 'allocated', got %s", node.NodeState) + } + }) + + t.Run("PreservesLatestPerNode", func(t *testing.T) { + // Insert a single old row for host125 — it's the latest per node so it must survive + ns := schema.NodeStateDB{ + TimeStamp: now - 7200, + NodeState: "idle", + HealthState: schema.MonitoringStateFull, + CpusAllocated: 0, + MemoryAllocated: 0, + JobsRunning: 0, + } + if err := repo.UpdateNodeState("host125", "testcluster", &ns); err != nil { + t.Fatal(err) + } + + // Delete everything older than now — the latest per node should be preserved + _, err := repo.DeleteNodeStatesBefore(now) + if err != nil { + t.Fatal(err) + } + + // The latest row for host125 must still exist + node, err := repo.GetNode("host125", "testcluster", false) + if err != nil { + t.Fatal(err) + } + if node.NodeState != "idle" { + t.Errorf("expected node state 'idle', got %s", node.NodeState) + } + + // Verify exactly 1 row remains for host125 + var countAfter int + if err := repo.DB.QueryRow( + "SELECT COUNT(*) FROM node_state WHERE node_id = (SELECT id FROM node WHERE hostname = 'host125')"). + Scan(&countAfter); err != nil { + t.Fatal(err) + } + if countAfter != 1 { + t.Errorf("expected 1 row remaining for host125, got %d", countAfter) + } + }) + + t.Run("FindBeforeWithJoin", func(t *testing.T) { + // Insert old and current rows for host123 + for _, ts := range []int64{now - 7200, now} { + ns := schema.NodeStateDB{ + TimeStamp: ts, + NodeState: "allocated", + HealthState: schema.MonitoringStateFull, + CpusAllocated: 8, + MemoryAllocated: 1024, + GpusAllocated: 1, + JobsRunning: 1, + } + if err := repo.UpdateNodeState("host123", "testcluster", &ns); err != nil { + t.Fatal(err) + } + } + + // Find rows older than 30 minutes, excluding latest per node + cutoff := now - 1800 + rows, err := repo.FindNodeStatesBefore(cutoff) + if err != nil { + t.Fatal(err) + } + + // Should find the old host123 row + found := false + for _, row := range rows { + if row.Hostname == "host123" && row.TimeStamp == now-7200 { + found = true + if row.Cluster != "testcluster" { + t.Errorf("expected cluster 'testcluster', got %s", row.Cluster) + } + if row.SubCluster != "sc1" { + t.Errorf("expected subcluster 'sc1', got %s", row.SubCluster) + } + if row.CpusAllocated != 8 { + t.Errorf("expected cpus_allocated 8, got %d", row.CpusAllocated) + } + } + } + if !found { + t.Errorf("expected to find old host123 row among %d results", len(rows)) + } + }) } From 48729b172df47e56aff753cf4084a7fe792ad36e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 12 Feb 2026 14:27:41 +0100 Subject: [PATCH 245/341] improve nodeList loading indicator, streamlining --- web/frontend/src/generic/JobList.svelte | 2 +- web/frontend/src/systems/NodeList.svelte | 20 +-- .../src/systems/nodelist/NodeListRow.svelte | 134 ++++++++++-------- 3 files changed, 83 insertions(+), 73 deletions(-) diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index 278f189e..3ccbb560 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -305,7 +305,7 @@ {#if $jobsStore.fetching || !$jobsStore.data} -
    +
    diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index da196b82..2e342168 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -104,7 +104,7 @@ let itemsPerPage = $derived(usePaging ? (ccconfig?.nodeList_nodesPerPage || 10) : 10); let paging = $derived({ itemsPerPage, page }); - const nodesQuery = $derived(queryStore({ + const nodesStore = $derived(queryStore({ client: client, query: nodeListQuery, variables: { @@ -122,7 +122,7 @@ requestPolicy: "network-only", // Resolution queries are cached, but how to access them? For now: reload on every change })); - const matchedNodes = $derived($nodesQuery?.data?.nodeMetricsList?.totalNodes || 0); + const matchedNodes = $derived($nodesStore?.data?.nodeMetricsList?.totalNodes || 0); /* Effects */ $effect(() => { @@ -135,7 +135,7 @@ } = document.documentElement; // Add 100 px offset to trigger load earlier - if (scrollTop + clientHeight >= scrollHeight - 100 && $nodesQuery?.data?.nodeMetricsList?.hasNextPage) { + if (scrollTop + clientHeight >= scrollHeight - 100 && $nodesStore?.data?.nodeMetricsList?.hasNextPage) { page += 1 }; }); @@ -143,9 +143,9 @@ }); $effect(() => { - if ($nodesQuery?.data) { + if ($nodesStore?.data) { untrack(() => { - handleNodes($nodesQuery?.data?.nodeMetricsList?.items); + handleNodes($nodesStore?.data?.nodeMetricsList?.items); }); selectedMetrics = [...pendingSelectedMetrics]; // Trigger Rerender in NodeListRow Only After Data is Fetched }; @@ -228,7 +228,7 @@ style="padding-top: {headerPaddingTop}px;" > {cluster} Node Info - {#if $nodesQuery.fetching} + {#if $nodesStore.fetching} {/if} @@ -245,22 +245,22 @@ - {#if $nodesQuery.error} + {#if $nodesStore.error} - {$nodesQuery.error.message} + {$nodesStore.error.message} {:else} {#each nodes as nodeData (nodeData.host)} - + {:else} No nodes found {/each} {/if} - {#if $nodesQuery.fetching || !$nodesQuery.data} + {#if $nodesStore.fetching || !$nodesStore.data}
    diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 46f8c4a4..1fca83f2 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -4,6 +4,7 @@ Properties: - `cluster String`: The nodes' cluster - `nodeData Object`: The node data object including metric data + - `nodeDataFetching Bool`: Whether the metric query still runs - `selectedMetrics [String]`: The array of selected metrics - `globalMetrics [Obj]`: Includes the backend supplied availabilities for cluster and subCluster --> @@ -24,6 +25,7 @@ let { cluster, nodeData, + nodeDataFetching, selectedMetrics, globalMetrics } = $props(); @@ -72,7 +74,7 @@ ); const extendedLegendData = $derived($nodeJobsData?.data ? buildExtendedLegend() : null); - const refinedData = $derived(nodeData?.metrics ? sortAndSelectScope(selectedMetrics, nodeData.metrics) : []); + const refinedData = $derived(!nodeDataFetching ? sortAndSelectScope(selectedMetrics, nodeData.metrics) : []); const dataHealth = $derived(refinedData.filter((rd) => rd.availability == "configured").map((enabled) => (enabled?.data?.metric?.series?.length > 0))); /* Functions */ @@ -150,65 +152,73 @@ hoststate={nodeData?.state? nodeData.state: 'notindb'}/> {/if} - {#each refinedData as metricData, i (metricData?.data?.name || i)} - {#key metricData} - - {#if metricData?.availability == "none"} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric is not configured for cluster {cluster}.

    -     - {:else if metricData?.availability == "disabled"} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric has been disabled for subcluster {nodeData.subCluster}.

    -     - {:else if !metricData?.data} - -

    No dataset(s) returned for {selectedMetrics[i]}

    -

    Metric or host was not found in metric store for cluster {cluster}.

    -     - {:else if !!metricData.data?.metric.statisticsSeries} - - -
    - - {:else} - - {/if} - - {/key} - {/each} + {#if nodeDataFetching} + +
    + +
    + + {:else} + {#each refinedData as metricData, i (metricData?.data?.name || i)} + {#key metricData} + + {#if metricData?.availability == "none"} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric is not configured for cluster {cluster}.

    +     + {:else if metricData?.availability == "disabled"} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric has been disabled for subcluster {nodeData.subCluster}.

    +     + {:else if !metricData?.data} + +

    No dataset(s) returned for {selectedMetrics[i]}

    +

    Metric or host was not found in metric store for cluster {cluster}.

    +     + {:else if !!metricData.data?.metric.statisticsSeries} + + +
    + + {:else} + + {/if} + + {/key} + {/each} + {/if} From c15f1117f553010e6e9c55331f45e3dcd2ab1c71 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 12 Feb 2026 15:45:15 +0100 Subject: [PATCH 246/341] Review and improve node repo queries --- .../migrations/sqlite3/10_node-table.up.sql | 1 + internal/repository/node.go | 226 +++++++----------- 2 files changed, 83 insertions(+), 144 deletions(-) diff --git a/internal/repository/migrations/sqlite3/10_node-table.up.sql b/internal/repository/migrations/sqlite3/10_node-table.up.sql index fd118f5d..b788a8a9 100644 --- a/internal/repository/migrations/sqlite3/10_node-table.up.sql +++ b/internal/repository/migrations/sqlite3/10_node-table.up.sql @@ -38,6 +38,7 @@ CREATE INDEX IF NOT EXISTS nodestates_state_timestamp ON node_state (node_state, CREATE INDEX IF NOT EXISTS nodestates_health_timestamp ON node_state (health_state, time_stamp); CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state); CREATE INDEX IF NOT EXISTS nodestates_nodeid_health ON node_state (node_id, health_state); +CREATE INDEX IF NOT EXISTS nodestates_nodeid_timestamp ON node_state (node_id, time_stamp DESC); -- Add NEW Indices For Increased Amounts of Tags CREATE INDEX IF NOT EXISTS tags_jobid ON jobtag (job_id); diff --git a/internal/repository/node.go b/internal/repository/node.go index 08a694c6..2ffe6698 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -52,6 +52,38 @@ func GetNodeRepository() *NodeRepository { return nodeRepoInstance } +// latestStateCondition returns a squirrel expression that restricts node_state +// rows to the latest per node_id using a correlated subquery. +// Requires the query to join node and node_state tables. +func latestStateCondition() sq.Sqlizer { + return sq.Expr( + "node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)", + ) +} + +// applyNodeFilters applies common NodeFilter conditions to a query that joins +// the node and node_state tables with latestStateCondition. +func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder { + for _, f := range filters { + if f.Cluster != nil { + query = buildStringCondition("node.cluster", f.Cluster, query) + } + if f.SubCluster != nil { + query = buildStringCondition("node.subcluster", f.SubCluster, query) + } + if f.Hostname != nil { + query = buildStringCondition("node.hostname", f.Hostname, query) + } + if f.SchedulerState != nil { + query = query.Where("node_state.node_state = ?", f.SchedulerState) + } + if f.HealthState != nil { + query = query.Where("node_state.health_state = ?", f.HealthState) + } + } + return query +} + func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) { start := time.Now() @@ -82,17 +114,16 @@ func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[str func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) { node := &schema.Node{} - var timestamp int - if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state", - "node_state.health_state", "MAX(node_state.time_stamp) as time"). - From("node_state"). - Join("node ON node_state.node_id = node.id"). + if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", + "node_state.node_state", "node_state.health_state"). + From("node"). + Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition()). Where("node.hostname = ?", hostname). Where("node.cluster = ?", cluster). - GroupBy("node_state.node_id"). RunWith(r.DB). - QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState, ×tamp); err != nil { - cclog.Warnf("Error while querying node '%s' at time '%d' from database: %v", hostname, timestamp, err) + QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil { + cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err) return nil, err } @@ -111,16 +142,15 @@ func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) { node := &schema.Node{} - var timestamp int - if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state", - "node_state.health_state", "MAX(node_state.time_stamp) as time"). - From("node_state"). - Join("node ON node_state.node_id = node.id"). + if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", + "node_state.node_state", "node_state.health_state"). + From("node"). + Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition()). Where("node.id = ?", id). - GroupBy("node_state.node_id"). RunWith(r.DB). - QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState, ×tamp); err != nil { - cclog.Warnf("Error while querying node ID '%d' at time '%d' from database: %v", id, timestamp, err) + QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil { + cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err) return nil, err } @@ -313,40 +343,17 @@ func (r *NodeRepository) QueryNodes( order *model.OrderByInput, // Currently unused! ) ([]*schema.Node, error) { query, qerr := AccessCheck(ctx, - sq.Select("hostname", "cluster", "subcluster", "node_state", "health_state", "MAX(time_stamp) as time"). + sq.Select("node.hostname", "node.cluster", "node.subcluster", + "node_state.node_state", "node_state.health_state"). From("node"). - Join("node_state ON node_state.node_id = node.id")) + Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition())) if qerr != nil { return nil, qerr } - for _, f := range filters { - if f.Cluster != nil { - query = buildStringCondition("cluster", f.Cluster, query) - } - if f.SubCluster != nil { - query = buildStringCondition("subcluster", f.SubCluster, query) - } - if f.Hostname != nil { - query = buildStringCondition("hostname", f.Hostname, query) - } - if f.SchedulerState != nil { - query = query.Where("node_state = ?", f.SchedulerState) - // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned - // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? - now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 300)}) - } - if f.HealthState != nil { - query = query.Where("health_state = ?", f.HealthState) - // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned - // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? - now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 300)}) - } - } - - query = query.GroupBy("node_id").OrderBy("hostname ASC") + query = applyNodeFilters(query, filters) + query = query.OrderBy("node.hostname ASC") if page != nil && page.ItemsPerPage != -1 { limit := uint64(page.ItemsPerPage) @@ -363,11 +370,10 @@ func (r *NodeRepository) QueryNodes( nodes := make([]*schema.Node, 0) for rows.Next() { node := schema.Node{} - var timestamp int if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster, - &node.NodeState, &node.HealthState, ×tamp); err != nil { + &node.NodeState, &node.HealthState); err != nil { rows.Close() - cclog.Warnf("Error while scanning rows (QueryNodes) at time '%d'", timestamp) + cclog.Warn("Error while scanning rows (QueryNodes)") return nil, err } nodes = append(nodes, &node) @@ -377,74 +383,39 @@ func (r *NodeRepository) QueryNodes( } // CountNodes returns the total matched nodes based on a node filter. It always operates -// on the last state (largest timestamp). +// on the last state (largest timestamp) per node. func (r *NodeRepository) CountNodes( ctx context.Context, filters []*model.NodeFilter, ) (int, error) { query, qerr := AccessCheck(ctx, - sq.Select("time_stamp", "count(*) as countRes"). + sq.Select("COUNT(*)"). From("node"). - Join("node_state ON node_state.node_id = node.id")) + Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition())) if qerr != nil { return 0, qerr } - for _, f := range filters { - if f.Cluster != nil { - query = buildStringCondition("cluster", f.Cluster, query) - } - if f.SubCluster != nil { - query = buildStringCondition("subcluster", f.SubCluster, query) - } - if f.Hostname != nil { - query = buildStringCondition("hostname", f.Hostname, query) - } - if f.SchedulerState != nil { - query = query.Where("node_state = ?", f.SchedulerState) - // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned - // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? - now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 300)}) - } - if f.HealthState != nil { - query = query.Where("health_state = ?", f.HealthState) - // Requires Additional time_stamp Filter: Else the last (past!) time_stamp with queried state will be returned - // TODO: Hardcoded TimeDiff Suboptimal - Use Config Option? - now := time.Now().Unix() - query = query.Where(sq.Gt{"time_stamp": (now - 300)}) - } - } + query = applyNodeFilters(query, filters) - query = query.GroupBy("time_stamp").OrderBy("time_stamp DESC").Limit(1) - - rows, err := query.RunWith(r.stmtCache).Query() - if err != nil { + var count int + if err := query.RunWith(r.stmtCache).QueryRow().Scan(&count); err != nil { queryString, queryVars, _ := query.ToSql() cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err) return 0, err } - var totalNodes int - for rows.Next() { - var timestamp int - if err := rows.Scan(×tamp, &totalNodes); err != nil { - rows.Close() - cclog.Warnf("Error while scanning rows (CountNodes) at time '%d'", timestamp) - return 0, err - } - } - - return totalNodes, nil + return count, nil } func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) { - q := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state", - "node_state.health_state", "MAX(node_state.time_stamp) as time"). + q := sq.Select("node.hostname", "node.cluster", "node.subcluster", + "node_state.node_state", "node_state.health_state"). From("node"). Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition()). Where("node.cluster = ?", cluster). - GroupBy("node_state.node_id"). OrderBy("node.hostname ASC") rows, err := q.RunWith(r.DB).Query() @@ -456,10 +427,9 @@ func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) { defer rows.Close() for rows.Next() { node := &schema.Node{} - var timestamp int if err := rows.Scan(&node.Hostname, &node.Cluster, - &node.SubCluster, &node.NodeState, &node.HealthState, ×tamp); err != nil { - cclog.Warnf("Error while scanning node list (ListNodes) at time '%d'", timestamp) + &node.SubCluster, &node.NodeState, &node.HealthState); err != nil { + cclog.Warn("Error while scanning node list (ListNodes)") return nil, err } @@ -470,11 +440,11 @@ func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) { } func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) { - q := sq.Select("node.hostname", "node_state.node_state", "MAX(node_state.time_stamp) as time"). + q := sq.Select("node.hostname", "node_state.node_state"). From("node"). Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition()). Where("node.cluster = ?", cluster). - GroupBy("node_state.node_id"). OrderBy("node.hostname ASC") rows, err := q.RunWith(r.DB).Query() @@ -487,9 +457,8 @@ func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) { defer rows.Close() for rows.Next() { var hostname, nodestate string - var timestamp int - if err := rows.Scan(&hostname, &nodestate, ×tamp); err != nil { - cclog.Warnf("Error while scanning node list (MapNodes) at time '%d'", timestamp) + if err := rows.Scan(&hostname, &nodestate); err != nil { + cclog.Warn("Error while scanning node list (MapNodes)") return nil, err } @@ -500,33 +469,16 @@ func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) { } func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) { - query, qerr := AccessCheck(ctx, sq.Select("hostname", column, "MAX(time_stamp) as time").From("node")) + query, qerr := AccessCheck(ctx, + sq.Select(column). + From("node"). + Join("node_state ON node_state.node_id = node.id"). + Where(latestStateCondition())) if qerr != nil { return nil, qerr } - query = query.Join("node_state ON node_state.node_id = node.id") - - for _, f := range filters { - if f.Hostname != nil { - query = buildStringCondition("hostname", f.Hostname, query) - } - if f.Cluster != nil { - query = buildStringCondition("cluster", f.Cluster, query) - } - if f.SubCluster != nil { - query = buildStringCondition("subcluster", f.SubCluster, query) - } - if f.SchedulerState != nil { - query = query.Where("node_state = ?", f.SchedulerState) - } - if f.HealthState != nil { - query = query.Where("health_state = ?", f.HealthState) - } - } - - // Add Group and Order - query = query.GroupBy("hostname").OrderBy("hostname DESC") + query = applyNodeFilters(query, filters) rows, err := query.RunWith(r.stmtCache).Query() if err != nil { @@ -537,12 +489,10 @@ func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeF stateMap := map[string]int{} for rows.Next() { - var hostname, state string - var timestamp int - - if err := rows.Scan(&hostname, &state, ×tamp); err != nil { + var state string + if err := rows.Scan(&state); err != nil { rows.Close() - cclog.Warnf("Error while scanning rows (CountStates) at time '%d'", timestamp) + cclog.Warn("Error while scanning rows (CountStates)") return nil, err } @@ -735,26 +685,14 @@ func (r *NodeRepository) GetNodesForList( } } else { - // DB Nodes: Count and Find Next Page + // DB Nodes: Count and derive hasNextPage from count var cerr error countNodes, cerr = r.CountNodes(ctx, queryFilters) if cerr != nil { cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)") return nil, nil, 0, false, cerr } - - // Example Page 4 @ 10 IpP : Does item 41 exist? - // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists. - nextPage := &model.PageRequest{ - ItemsPerPage: 1, - Page: ((page.Page * page.ItemsPerPage) + 1), - } - nextNodes, err := r.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used - if err != nil { - cclog.Warn("Error while querying next nodes") - return nil, nil, 0, false, err - } - hasNextPage = len(nextNodes) == 1 + hasNextPage = page.Page*page.ItemsPerPage < countNodes } // Fallback for non-init'd node table in DB; Ignores stateFilter From 3215bc3de0a09888db800770da8d59a718a166e3 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 13 Feb 2026 11:58:52 +0100 Subject: [PATCH 247/341] review loading indicators in nodeList --- web/frontend/src/systems/NodeList.svelte | 21 ++- .../src/systems/nodelist/NodeInfo.svelte | 15 +- .../src/systems/nodelist/NodeListRow.svelte | 138 +++++++++--------- 3 files changed, 95 insertions(+), 79 deletions(-) diff --git a/web/frontend/src/systems/NodeList.svelte b/web/frontend/src/systems/NodeList.svelte index 2e342168..403a8030 100644 --- a/web/frontend/src/systems/NodeList.svelte +++ b/web/frontend/src/systems/NodeList.svelte @@ -152,12 +152,21 @@ }); $effect(() => { - // Triggers (Except Paging) + // Update NodeListRows metrics only: Keep ordered nodes on page 1 from, to pendingSelectedMetrics, selectedResolution + // Continous Scroll: Paging if parameters change: Existing entries will not match new selections + if (!usePaging) { + nodes = []; + page = 1; + } + }); + + $effect(() => { + // Update NodeListRows metrics only: Keep ordered nodes on page 1 hostnameFilter, hoststateFilter // Continous Scroll: Paging if parameters change: Existing entries will not match new selections - // Nodes Array Reset in HandleNodes func + nodes = []; if (!usePaging) { page = 1; } @@ -255,9 +264,11 @@ {#each nodes as nodeData (nodeData.host)} {:else} - - No nodes found - + {#if !$nodesStore.fetching} + + No nodes found + + {/if} {/each} {/if} {#if $nodesStore.fetching || !$nodesStore.data} diff --git a/web/frontend/src/systems/nodelist/NodeInfo.svelte b/web/frontend/src/systems/nodelist/NodeInfo.svelte index 39716ca2..4b616f10 100644 --- a/web/frontend/src/systems/nodelist/NodeInfo.svelte +++ b/web/frontend/src/systems/nodelist/NodeInfo.svelte @@ -51,6 +51,8 @@ /* Derived */ // Not at least one returned, selected metric: NodeHealth warning + const fetchInfo = $derived(dataHealth.includes('fetching')); + // Not at least one returned, selected metric: NodeHealth warning const healthWarn = $derived(!dataHealth.includes(true)); // At least one non-returned selected metric: Metric config error? const metricWarn = $derived(dataHealth.includes(false)); @@ -84,10 +86,17 @@ - {#if healthWarn} + {#if fetchInfo} + + + + + {:else if healthWarn} - Jobs + Info From cc21e0e62cde3c628cd4b7529ae28ad4e27612ec Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 25 Feb 2026 07:38:19 +0100 Subject: [PATCH 295/341] Make json the default checkpoint format --- pkg/metricstore/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metricstore/config.go b/pkg/metricstore/config.go index 69ee3563..1efee61a 100644 --- a/pkg/metricstore/config.go +++ b/pkg/metricstore/config.go @@ -144,7 +144,7 @@ type MetricStoreConfig struct { // Accessed by Init(), Checkpointing(), and other lifecycle functions. var Keys MetricStoreConfig = MetricStoreConfig{ Checkpoints: Checkpoints{ - FileFormat: "avro", + FileFormat: "json", RootDir: "./var/checkpoints", }, Cleanup: &Cleanup{ From df3bc111a47043b4413a07254037f315a62b4a21 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 25 Feb 2026 13:23:44 +0100 Subject: [PATCH 296/341] sort healthTable onMount --- web/frontend/src/status/dashdetails/HealthDash.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/frontend/src/status/dashdetails/HealthDash.svelte b/web/frontend/src/status/dashdetails/HealthDash.svelte index 2730642b..aa6539ae 100644 --- a/web/frontend/src/status/dashdetails/HealthDash.svelte +++ b/web/frontend/src/status/dashdetails/HealthDash.svelte @@ -6,6 +6,7 @@ --> From 0a0db36433ea18ea0d3c4ddcf3d88fda3b566aa1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 25 Feb 2026 19:12:18 +0100 Subject: [PATCH 297/341] load statusDetails GQL on tab change --- web/frontend/src/status/DashDetails.svelte | 15 +++++---- .../src/status/dashdetails/HealthDash.svelte | 19 ++++++------ .../status/dashdetails/StatisticsDash.svelte | 13 ++++---- .../src/status/dashdetails/StatusDash.svelte | 27 ++++++++-------- .../src/status/dashdetails/UsageDash.svelte | 31 ++++++++++--------- 5 files changed, 56 insertions(+), 49 deletions(-) diff --git a/web/frontend/src/status/DashDetails.svelte b/web/frontend/src/status/DashDetails.svelte index b46d0935..72c411b2 100644 --- a/web/frontend/src/status/DashDetails.svelte +++ b/web/frontend/src/status/DashDetails.svelte @@ -36,6 +36,9 @@ const { query: initq } = init(); const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false + /* State Init */ + let activeTab = $state(""); + /* Derived */ const subClusters = $derived($initq?.data?.clusters?.find((c) => c.name == presetCluster)?.subClusters || []); @@ -63,22 +66,22 @@ {:else} - + (activeTab = e.detail)}> - + - + - + @@ -86,7 +89,7 @@ {#each subClusters.map(sc => sc.name) as scn} - + {/each} @@ -94,7 +97,7 @@ - + diff --git a/web/frontend/src/status/dashdetails/HealthDash.svelte b/web/frontend/src/status/dashdetails/HealthDash.svelte index aa6539ae..a30552b1 100644 --- a/web/frontend/src/status/dashdetails/HealthDash.svelte +++ b/web/frontend/src/status/dashdetails/HealthDash.svelte @@ -29,6 +29,7 @@ /* Svelte 5 Props */ let { presetCluster, + loadMe = false, } = $props(); /* Const Init */ @@ -55,7 +56,7 @@ /* Derived */ let cluster = $derived(presetCluster); - const statusQuery = $derived(queryStore({ + const statusQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -85,7 +86,7 @@ sorting: querySorting, }, requestPolicy: "network-only" - })); + }) : null); let healthTableData = $derived.by(() => { if ($statusQuery?.data) { @@ -161,16 +162,16 @@
    -{#if $statusQuery.fetching} +{#if $statusQuery?.fetching} -{:else if $statusQuery.error} +{:else if $statusQuery?.error} - Status Query (States): {$statusQuery.error.message} + Status Query (States): {$statusQuery?.error?.message} {:else if $statusQuery?.data?.nodeStates} @@ -264,19 +265,19 @@
    -{#if $statusQuery.fetching} +{#if $statusQuery?.fetching} -{:else if $statusQuery.error} +{:else if $statusQuery?.error} - Status Query (Details): {$statusQuery.error.message} + Status Query (Details): {$statusQuery?.error?.message} -{:else if $statusQuery.data} +{:else if $statusQuery?.data} diff --git a/web/frontend/src/status/dashdetails/StatisticsDash.svelte b/web/frontend/src/status/dashdetails/StatisticsDash.svelte index 2cf8621e..d83adc15 100644 --- a/web/frontend/src/status/dashdetails/StatisticsDash.svelte +++ b/web/frontend/src/status/dashdetails/StatisticsDash.svelte @@ -30,7 +30,8 @@ /* Svelte 5 Props */ let { - presetCluster + presetCluster, + loadMe = false, } = $props(); /* Const Init */ @@ -49,7 +50,7 @@ : ccconfig['statusView_selectedHistograms'] || []); // Note: nodeMetrics are requested on configured $timestep resolution - const metricStatusQuery = $derived(queryStore({ + const metricStatusQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -75,7 +76,7 @@ selectedHistograms: selectedHistograms }, requestPolicy: "network-only" - })); + }) : null); @@ -100,18 +101,18 @@ - {#if $metricStatusQuery.fetching} + {#if $metricStatusQuery?.fetching} - {:else if $metricStatusQuery.error} + {:else if $metricStatusQuery?.error} {$metricStatusQuery.error.message} {/if} -{#if $metricStatusQuery.data} +{#if $metricStatusQuery?.data} {#if selectedHistograms} diff --git a/web/frontend/src/status/dashdetails/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte index 8d108964..0c2626d0 100644 --- a/web/frontend/src/status/dashdetails/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -32,6 +32,7 @@ let { clusters, presetCluster, + loadMe = false, } = $props(); /* Const Init */ @@ -59,7 +60,7 @@ /* Derived */ let cluster = $derived(presetCluster); // States for Stacked charts - const statesTimed = $derived(queryStore({ + const statesTimed = $derived(loadMe ? queryStore({ client: client, query: gql` query ($filter: [NodeFilter!], $typeNode: String!, $typeHealth: String!) { @@ -81,11 +82,11 @@ typeHealth: "health" }, requestPolicy: "network-only" - })); + }) : null); // Note: nodeMetrics are requested on configured $timestep resolution // Result: The latest 5 minutes (datapoints) for each node independent of job - const statusQuery = $derived(queryStore({ + const statusQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -184,11 +185,11 @@ sorting: { field: "startTime", type: "col", order: "DESC" } }, requestPolicy: "network-only" - })); + }) : null); /* Effects */ $effect(() => { - if ($statusQuery.data) { + if ($statusQuery?.data) { let subClusters = clusters.find( (c) => c.name == cluster, ).subClusters; @@ -374,19 +375,19 @@
    -{#if $statesTimed.fetching} +{#if $statesTimed?.fetching} -{:else if $statesTimed.error} +{:else if $statesTimed?.error} - States Timed: {$statesTimed.error.message} + States Timed: {$statesTimed?.error?.message} -{:else if $statesTimed.data} +{:else if $statesTimed?.data}
    @@ -427,19 +428,19 @@
    -{#if $statusQuery.fetching} +{#if $statusQuery?.fetching} -{:else if $statusQuery.error} +{:else if $statusQuery?.error} - Status Query (Details): {$statusQuery.error.message} + Status Query (Details): {$statusQuery?.error?.message} -{:else if $statusQuery.data} +{:else if $statusQuery?.data} {#each clusters.find((c) => c.name == cluster).subClusters as subCluster, i} diff --git a/web/frontend/src/status/dashdetails/UsageDash.svelte b/web/frontend/src/status/dashdetails/UsageDash.svelte index 3fa197ae..2a9b3037 100644 --- a/web/frontend/src/status/dashdetails/UsageDash.svelte +++ b/web/frontend/src/status/dashdetails/UsageDash.svelte @@ -40,7 +40,8 @@ presetCluster, presetSubCluster = null, useCbColors = false, - useAltColors = false + useAltColors = false, + loadMe = false, } = $props(); /* Const Init */ @@ -62,7 +63,7 @@ ? [{ state: ["running"] }, { cluster: { eq: presetCluster} }, { subCluster: { eq: presetSubCluster } }] : [{ state: ["running"] }, { cluster: { eq: presetCluster} }] ); - const topJobsQuery = $derived(queryStore({ + const topJobsQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -95,9 +96,9 @@ paging: pagingState // Top 10 }, requestPolicy: "network-only" - })); + }) : null); - const topNodesQuery = $derived(queryStore({ + const topNodesQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -130,9 +131,9 @@ paging: pagingState }, requestPolicy: "network-only" - })); + }) : null); - const topAccsQuery = $derived(queryStore({ + const topAccsQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -165,10 +166,10 @@ paging: pagingState }, requestPolicy: "network-only" - })); + }): null); // Note: nodeMetrics are requested on configured $timestep resolution - const nodeStatusQuery = $derived(queryStore({ + const nodeStatusQuery = $derived(loadMe ? queryStore({ client: client, query: gql` query ( @@ -198,7 +199,7 @@ numDurationBins: numDurationBins, }, requestPolicy: "network-only" - })); + }) : null); /* Functions */ function legendColors(targetIdx) { @@ -246,9 +247,9 @@
    -{#if $topJobsQuery.fetching || $nodeStatusQuery.fetching} +{#if $topJobsQuery?.fetching || $nodeStatusQuery?.fetching} -{:else if $topJobsQuery.data && $nodeStatusQuery.data} +{:else if $topJobsQuery?.data && $nodeStatusQuery?.data} {#key $nodeStatusQuery.data.jobsStatistics[0].histDuration} @@ -354,9 +355,9 @@
    -{#if $topNodesQuery.fetching || $nodeStatusQuery.fetching} +{#if $topNodesQuery?.fetching || $nodeStatusQuery?.fetching} -{:else if $topNodesQuery.data && $nodeStatusQuery.data} +{:else if $topNodesQuery?.data && $nodeStatusQuery?.data} -{#if $topAccsQuery.fetching || $nodeStatusQuery.fetching} +{#if $topAccsQuery?.fetching || $nodeStatusQuery?.fetching} -{:else if $topAccsQuery.data && $nodeStatusQuery.data} +{:else if $topAccsQuery?.data && $nodeStatusQuery?.data} Date: Thu, 26 Feb 2026 10:08:40 +0100 Subject: [PATCH 298/341] Introduce metric store binary checkpoints with write ahead log --- go.mod | 2 - go.sum | 11 - pkg/metricstore/avroCheckpoint.go | 481 ------------------ pkg/metricstore/avroHelper.go | 130 ----- pkg/metricstore/avroStruct.go | 167 ------- pkg/metricstore/checkpoint.go | 369 +++++--------- pkg/metricstore/config.go | 7 +- pkg/metricstore/configSchema.go | 2 +- pkg/metricstore/lineprotocol.go | 4 +- pkg/metricstore/metricstore.go | 22 +- pkg/metricstore/walCheckpoint.go | 787 ++++++++++++++++++++++++++++++ 11 files changed, 920 insertions(+), 1062 deletions(-) delete mode 100644 pkg/metricstore/avroCheckpoint.go delete mode 100644 pkg/metricstore/avroHelper.go delete mode 100644 pkg/metricstore/avroStruct.go create mode 100644 pkg/metricstore/walCheckpoint.go diff --git a/go.mod b/go.mod index e244062c..c561f627 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/influxdata/line-protocol/v2 v2.2.1 github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 - github.com/linkedin/goavro/v2 v2.15.0 github.com/mattn/go-sqlite3 v1.14.34 github.com/parquet-go/parquet-go v0.27.0 github.com/qustavo/sqlhooks/v2 v2.1.0 @@ -80,7 +79,6 @@ require ( github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect diff --git a/go.sum b/go.sum index f2929454..5586b9c5 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/99designs/gqlgen v0.17.86 h1:C8N3UTa5heXX6twl+b0AJyGkTwYL6dNmFrgZNLRc github.com/99designs/gqlgen v0.17.86/go.mod h1:KTrPl+vHA1IUzNlh4EYkl7+tcErL3MgKnhHrBcV74Fw= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= -github.com/ClusterCockpit/cc-lib/v2 v2.5.1 h1:s6M9tyPDty+4zTdQGJYKpGJM9Nz7N6ITMdjPvNSLX5g= -github.com/ClusterCockpit/cc-lib/v2 v2.5.1/go.mod h1:DZ8OIHPUZJpWqErLITt0B8P6/Q7CBW2IQSQ5YiFFaG0= github.com/ClusterCockpit/cc-lib/v2 v2.6.0 h1:Q7zvRAVhfYA9PDB18pfY9A/6Ws4oWpnv8+P9MBRUDzg= github.com/ClusterCockpit/cc-lib/v2 v2.6.0/go.mod h1:DZ8OIHPUZJpWqErLITt0B8P6/Q7CBW2IQSQ5YiFFaG0= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= @@ -151,9 +149,6 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -226,8 +221,6 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI= -github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= @@ -289,14 +282,11 @@ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKk github.com/stmcginnis/gofish v0.21.1 h1:sutDvBhmLh4RDOZ1DN8GUyYRu7f1ggvKMMnSaiqhwn4= github.com/stmcginnis/gofish v0.21.1/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -378,7 +368,6 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/metricstore/avroCheckpoint.go b/pkg/metricstore/avroCheckpoint.go deleted file mode 100644 index 14898186..00000000 --- a/pkg/metricstore/avroCheckpoint.go +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricstore - -import ( - "bufio" - "encoding/json" - "errors" - "fmt" - "os" - "path" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/ClusterCockpit/cc-lib/v2/schema" - "github.com/linkedin/goavro/v2" -) - -var ( - NumAvroWorkers int = DefaultAvroWorkers - startUp bool = true -) - -func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { - levels := make([]*AvroLevel, 0) - selectors := make([][]string, 0) - as.root.lock.RLock() - // Cluster - for sel1, l1 := range as.root.children { - l1.lock.RLock() - // Node - for sel2, l2 := range l1.children { - l2.lock.RLock() - // Frequency - for sel3, l3 := range l2.children { - levels = append(levels, l3) - selectors = append(selectors, []string{sel1, sel2, sel3}) - } - l2.lock.RUnlock() - } - l1.lock.RUnlock() - } - as.root.lock.RUnlock() - - type workItem struct { - level *AvroLevel - dir string - selector []string - } - - n, errs := int32(0), int32(0) - - var wg sync.WaitGroup - wg.Add(NumAvroWorkers) - work := make(chan workItem, NumAvroWorkers*2) - for range NumAvroWorkers { - go func() { - defer wg.Done() - - for workItem := range work { - from := getTimestamp(workItem.dir) - - if err := workItem.level.toCheckpoint(workItem.dir, from, dumpAll); err != nil { - if err == ErrNoNewArchiveData { - continue - } - - cclog.Errorf("error while checkpointing %#v: %s", workItem.selector, err.Error()) - atomic.AddInt32(&errs, 1) - } else { - atomic.AddInt32(&n, 1) - } - } - }() - } - - for i := range len(levels) { - dir := path.Join(dir, path.Join(selectors[i]...)) - work <- workItem{ - level: levels[i], - dir: dir, - selector: selectors[i], - } - } - - close(work) - wg.Wait() - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n) - } - - startUp = false - - return int(n), nil -} - -// getTimestamp returns the timestamp from the directory name -func getTimestamp(dir string) int64 { - // Extract the resolution and timestamp from the directory name - // The existing avro file will be in epoch timestamp format - // iterate over all the files in the directory and find the maximum timestamp - // and return it - - resolution := path.Base(dir) - dir = path.Dir(dir) - - files, err := os.ReadDir(dir) - if err != nil { - return 0 - } - var maxTS int64 = 0 - - if len(files) == 0 { - return 0 - } - - for _, file := range files { - if file.IsDir() { - continue - } - name := file.Name() - - if len(name) < 5 || !strings.HasSuffix(name, ".avro") || !strings.HasPrefix(name, resolution+"_") { - continue - } - - ts, err := strconv.ParseInt(name[strings.Index(name, "_")+1:len(name)-5], 10, 64) - if err != nil { - fmt.Printf("error while parsing timestamp: %s\n", err.Error()) - continue - } - - if ts > maxTS { - maxTS = ts - } - } - - interval, _ := time.ParseDuration(Keys.Checkpoints.Interval) - updateTime := time.Unix(maxTS, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix() - - if startUp { - return 0 - } - - if updateTime < time.Now().Unix() { - return 0 - } - - return maxTS -} - -func (l *AvroLevel) toCheckpoint(dir string, from int64, dumpAll bool) error { - l.lock.Lock() - defer l.lock.Unlock() - - // fmt.Printf("Checkpointing directory: %s\n", dir) - // filepath contains the resolution - intRes, _ := strconv.Atoi(path.Base(dir)) - - // find smallest overall timestamp in l.data map and delete it from l.data - minTS := int64(1<<63 - 1) - for ts, dat := range l.data { - if ts < minTS && len(dat) != 0 { - minTS = ts - } - } - - if from == 0 && minTS != int64(1<<63-1) { - from = minTS - } - - if from == 0 { - return ErrNoNewArchiveData - } - - var schema string - var codec *goavro.Codec - recordList := make([]map[string]any, 0) - - var f *os.File - - filePath := dir + fmt.Sprintf("_%d.avro", from) - - var err error - - fp_, err_ := os.Stat(filePath) - if errors.Is(err_, os.ErrNotExist) { - err = os.MkdirAll(path.Dir(dir), 0o755) - if err != nil { - return fmt.Errorf("failed to create directory: %v", err) - } - } else if fp_.Size() != 0 { - f, err = os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open existing avro file: %v", err) - } - defer f.Close() - - br := bufio.NewReader(f) - - reader, err := goavro.NewOCFReader(br) - if err != nil { - return fmt.Errorf("failed to create OCF reader: %v", err) - } - codec = reader.Codec() - schema = codec.Schema() - } - - timeRef := time.Now().Add(time.Duration(-CheckpointBufferMinutes+1) * time.Minute).Unix() - - if dumpAll { - timeRef = time.Now().Unix() - } - - // Empty values - if len(l.data) == 0 { - // we checkpoint avro files every 60 seconds - repeat := 60 / intRes - - for range repeat { - recordList = append(recordList, make(map[string]any)) - } - } - - readFlag := true - - for ts := range l.data { - flag := false - if ts < timeRef { - data := l.data[ts] - - schemaGen, err := generateSchema(data) - if err != nil { - return err - } - - flag, schema, err = compareSchema(schema, schemaGen) - if err != nil { - return fmt.Errorf("failed to compare read and generated schema: %v", err) - } - if flag && readFlag && !errors.Is(err_, os.ErrNotExist) { - // Use closure to ensure file is closed even on error - err := func() error { - f2, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open Avro file: %v", err) - } - defer f2.Close() - - br := bufio.NewReader(f2) - - ocfReader, err := goavro.NewOCFReader(br) - if err != nil { - return fmt.Errorf("failed to create OCF reader while changing schema: %v", err) - } - - for ocfReader.Scan() { - record, err := ocfReader.Read() - if err != nil { - return fmt.Errorf("failed to read record: %v", err) - } - - recordList = append(recordList, record.(map[string]any)) - } - - return nil - }() - if err != nil { - return err - } - - err = os.Remove(filePath) - if err != nil { - return fmt.Errorf("failed to delete file: %v", err) - } - - readFlag = false - } - codec, err = goavro.NewCodec(schema) - if err != nil { - return fmt.Errorf("failed to create codec after merged schema: %v", err) - } - - recordList = append(recordList, generateRecord(data)) - delete(l.data, ts) - } - } - - if len(recordList) == 0 { - return ErrNoNewArchiveData - } - - f, err = os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644) - if err != nil { - return fmt.Errorf("failed to append new avro file: %v", err) - } - defer f.Close() - - // fmt.Printf("Codec : %#v\n", codec) - - writer, err := goavro.NewOCFWriter(goavro.OCFConfig{ - W: f, - Codec: codec, - CompressionName: goavro.CompressionDeflateLabel, - }) - if err != nil { - return fmt.Errorf("failed to create OCF writer: %v", err) - } - - // Append the new record - if err := writer.Append(recordList); err != nil { - return fmt.Errorf("failed to append record: %v", err) - } - - return nil -} - -func compareSchema(schemaRead, schemaGen string) (bool, string, error) { - var genSchema, readSchema AvroSchema - - if schemaRead == "" { - return false, schemaGen, nil - } - - // Unmarshal the schema strings into AvroSchema structs - if err := json.Unmarshal([]byte(schemaGen), &genSchema); err != nil { - return false, "", fmt.Errorf("failed to parse generated schema: %v", err) - } - if err := json.Unmarshal([]byte(schemaRead), &readSchema); err != nil { - return false, "", fmt.Errorf("failed to parse read schema: %v", err) - } - - sort.Slice(genSchema.Fields, func(i, j int) bool { - return genSchema.Fields[i].Name < genSchema.Fields[j].Name - }) - - sort.Slice(readSchema.Fields, func(i, j int) bool { - return readSchema.Fields[i].Name < readSchema.Fields[j].Name - }) - - // Check if schemas are identical - schemasEqual := true - if len(genSchema.Fields) <= len(readSchema.Fields) { - - for i := range genSchema.Fields { - if genSchema.Fields[i].Name != readSchema.Fields[i].Name { - schemasEqual = false - break - } - } - - // If schemas are identical, return the read schema - if schemasEqual { - return false, schemaRead, nil - } - } - - // Create a map to hold unique fields from both schemas - fieldMap := make(map[string]AvroField) - - // Add fields from the read schema - for _, field := range readSchema.Fields { - fieldMap[field.Name] = field - } - - // Add or update fields from the generated schema - for _, field := range genSchema.Fields { - fieldMap[field.Name] = field - } - - // Create a union schema by collecting fields from the map - var mergedFields []AvroField - for _, field := range fieldMap { - mergedFields = append(mergedFields, field) - } - - // Sort fields by name for consistency - sort.Slice(mergedFields, func(i, j int) bool { - return mergedFields[i].Name < mergedFields[j].Name - }) - - // Create the merged schema - mergedSchema := AvroSchema{ - Type: "record", - Name: genSchema.Name, - Fields: mergedFields, - } - - // Check if schemas are identical - schemasEqual = len(mergedSchema.Fields) == len(readSchema.Fields) - if schemasEqual { - for i := range mergedSchema.Fields { - if mergedSchema.Fields[i].Name != readSchema.Fields[i].Name { - schemasEqual = false - break - } - } - - if schemasEqual { - return false, schemaRead, nil - } - } - - // Marshal the merged schema back to JSON - mergedSchemaJSON, err := json.Marshal(mergedSchema) - if err != nil { - return false, "", fmt.Errorf("failed to marshal merged schema: %v", err) - } - - return true, string(mergedSchemaJSON), nil -} - -func generateSchema(data map[string]schema.Float) (string, error) { - // Define the Avro schema structure - schema := map[string]any{ - "type": "record", - "name": "DataRecord", - "fields": []map[string]any{}, - } - - fieldTracker := make(map[string]struct{}) - - for key := range data { - if _, exists := fieldTracker[key]; !exists { - key = correctKey(key) - - field := map[string]any{ - "name": key, - "type": "double", - "default": -1.0, - } - schema["fields"] = append(schema["fields"].([]map[string]any), field) - fieldTracker[key] = struct{}{} - } - } - - schemaString, err := json.Marshal(schema) - if err != nil { - return "", fmt.Errorf("failed to marshal schema: %v", err) - } - - return string(schemaString), nil -} - -func generateRecord(data map[string]schema.Float) map[string]any { - record := make(map[string]any) - - // Iterate through each map in data - for key, value := range data { - key = correctKey(key) - - // Set the value in the record - // avro only accepts basic types - record[key] = value.Double() - } - - return record -} - -func correctKey(key string) string { - key = strings.ReplaceAll(key, "_", "_0x5F_") - key = strings.ReplaceAll(key, ":", "_0x3A_") - key = strings.ReplaceAll(key, ".", "_0x2E_") - return key -} - -func ReplaceKey(key string) string { - key = strings.ReplaceAll(key, "_0x2E_", ".") - key = strings.ReplaceAll(key, "_0x3A_", ":") - key = strings.ReplaceAll(key, "_0x5F_", "_") - return key -} diff --git a/pkg/metricstore/avroHelper.go b/pkg/metricstore/avroHelper.go deleted file mode 100644 index f6bef36e..00000000 --- a/pkg/metricstore/avroHelper.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricstore - -import ( - "context" - "slices" - "strconv" - "strings" - "sync" - - cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" -) - -func DataStaging(wg *sync.WaitGroup, ctx context.Context) { - wg.Add(1) - go func() { - defer wg.Done() - - if Keys.Checkpoints.FileFormat == "json" { - return - } - - ms := GetMemoryStore() - var avroLevel *AvroLevel - oldSelector := make([]string, 0) - - for { - select { - case <-ctx.Done(): - // Drain any remaining messages in channel before exiting - for { - select { - case val, ok := <-LineProtocolMessages: - if !ok { - // Channel closed - return - } - // Process remaining message - freq, err := ms.GetMetricFrequency(val.MetricName) - if err != nil { - continue - } - - var metricName strings.Builder - for _, selectorName := range val.Selector { - metricName.WriteString(selectorName + SelectorDelimiter) - } - metricName.WriteString(val.MetricName) - - var selector []string - selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10)) - - if !stringSlicesEqual(oldSelector, selector) { - avroLevel = avroStore.root.findAvroLevelOrCreate(selector) - if avroLevel == nil { - cclog.Errorf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName) - } - oldSelector = slices.Clone(selector) - } - - if avroLevel != nil { - avroLevel.addMetric(metricName.String(), val.Value, val.Timestamp, int(freq)) - } - default: - // No more messages, exit - return - } - } - case val, ok := <-LineProtocolMessages: - if !ok { - // Channel closed, exit gracefully - return - } - - // Fetch the frequency of the metric from the global configuration - freq, err := ms.GetMetricFrequency(val.MetricName) - if err != nil { - cclog.Errorf("Error fetching metric frequency: %s\n", err) - continue - } - - var metricName strings.Builder - - for _, selectorName := range val.Selector { - metricName.WriteString(selectorName + SelectorDelimiter) - } - - metricName.WriteString(val.MetricName) - - // Create a new selector for the Avro level - // The selector is a slice of strings that represents the path to the - // Avro level. It is created by appending the cluster, node, and metric - // name to the selector. - var selector []string - selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10)) - - if !stringSlicesEqual(oldSelector, selector) { - // Get the Avro level for the metric - avroLevel = avroStore.root.findAvroLevelOrCreate(selector) - - // If the Avro level is nil, create a new one - if avroLevel == nil { - cclog.Errorf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName) - } - oldSelector = slices.Clone(selector) - } - - if avroLevel != nil { - avroLevel.addMetric(metricName.String(), val.Value, val.Timestamp, int(freq)) - } - } - } - }() -} - -func stringSlicesEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/pkg/metricstore/avroStruct.go b/pkg/metricstore/avroStruct.go deleted file mode 100644 index 78a8d137..00000000 --- a/pkg/metricstore/avroStruct.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. This file is part of cc-backend. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package metricstore - -import ( - "sync" - - "github.com/ClusterCockpit/cc-lib/v2/schema" -) - -var ( - LineProtocolMessages = make(chan *AvroStruct) - // SelectorDelimiter separates hierarchical selector components in metric names for Avro encoding - SelectorDelimiter = "_SEL_" -) - -var CheckpointBufferMinutes = DefaultCheckpointBufferMin - -type AvroStruct struct { - MetricName string - Cluster string - Node string - Selector []string - Value schema.Float - Timestamp int64 -} - -type AvroStore struct { - root AvroLevel -} - -var avroStore AvroStore - -type AvroLevel struct { - children map[string]*AvroLevel - data map[int64]map[string]schema.Float - lock sync.RWMutex -} - -type AvroField struct { - Name string `json:"name"` - Type any `json:"type"` - Default any `json:"default,omitempty"` -} - -type AvroSchema struct { - Type string `json:"type"` - Name string `json:"name"` - Fields []AvroField `json:"fields"` -} - -func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel { - if len(selector) == 0 { - return l - } - - // Allow concurrent reads: - l.lock.RLock() - var child *AvroLevel - var ok bool - if l.children == nil { - // Children map needs to be created... - l.lock.RUnlock() - } else { - child, ok := l.children[selector[0]] - l.lock.RUnlock() - if ok { - return child.findAvroLevelOrCreate(selector[1:]) - } - } - - // The level does not exist, take write lock for unique access: - l.lock.Lock() - // While this thread waited for the write lock, another thread - // could have created the child node. - if l.children != nil { - child, ok = l.children[selector[0]] - if ok { - l.lock.Unlock() - return child.findAvroLevelOrCreate(selector[1:]) - } - } - - child = &AvroLevel{ - data: make(map[int64]map[string]schema.Float, 0), - children: nil, - } - - if l.children != nil { - l.children[selector[0]] = child - } else { - l.children = map[string]*AvroLevel{selector[0]: child} - } - l.lock.Unlock() - return child.findAvroLevelOrCreate(selector[1:]) -} - -func (l *AvroLevel) addMetric(metricName string, value schema.Float, timestamp int64, Freq int) { - l.lock.Lock() - defer l.lock.Unlock() - - KeyCounter := int(CheckpointBufferMinutes * 60 / Freq) - - // Create keys in advance for the given amount of time - if len(l.data) != KeyCounter { - if len(l.data) == 0 { - for i := range KeyCounter { - l.data[timestamp+int64(i*Freq)] = make(map[string]schema.Float, 0) - } - } else { - // Get the last timestamp - var lastTS int64 - for ts := range l.data { - if ts > lastTS { - lastTS = ts - } - } - // Create keys for the next KeyCounter timestamps - l.data[lastTS+int64(Freq)] = make(map[string]schema.Float, 0) - } - } - - closestTS := int64(0) - minDiff := int64(Freq) + 1 // Start with diff just outside the valid range - found := false - - // Iterate over timestamps and choose the one which is within range. - // Since its epoch time, we check if the difference is less than 60 seconds. - for ts, dat := range l.data { - // Check if timestamp is within range - diff := timestamp - ts - if diff < -int64(Freq) || diff > int64(Freq) { - continue - } - - // Metric already present at this timestamp — skip - if _, ok := dat[metricName]; ok { - continue - } - - // Check if this is the closest timestamp so far - if Abs(diff) < minDiff { - minDiff = Abs(diff) - closestTS = ts - found = true - } - } - - if found { - l.data[closestTS][metricName] = value - } -} - -func GetAvroStore() *AvroStore { - return &avroStore -} - -// Abs returns the absolute value of x. -func Abs(x int64) int64 { - if x < 0 { - return -x - } - return x -} diff --git a/pkg/metricstore/checkpoint.go b/pkg/metricstore/checkpoint.go index b4097ff2..590197e3 100644 --- a/pkg/metricstore/checkpoint.go +++ b/pkg/metricstore/checkpoint.go @@ -6,15 +6,15 @@ // This file implements checkpoint persistence for the in-memory metric store. // // Checkpoints enable graceful restarts by periodically saving in-memory metric -// data to disk in either JSON or Avro format. The checkpoint system: +// data to disk in JSON or binary format. The checkpoint system: // // Key Features: // - Periodic background checkpointing via the Checkpointing() worker -// - Two formats: JSON (human-readable) and Avro (compact, efficient) +// - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe) // - Parallel checkpoint creation and loading using worker pools -// - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|avro} +// - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin} +// - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry) // - Only saves unarchived data (archived data is already persisted elsewhere) -// - Automatic format detection and fallback during loading // - GC optimization during loading to prevent excessive heap growth // // Checkpoint Workflow: @@ -27,8 +27,9 @@ // checkpoints/ // cluster1/ // host001/ -// 1234567890.json (timestamp = checkpoint start time) -// 1234567950.json +// 1234567890.json (JSON format: full subtree snapshot) +// 1234567890.bin (binary format: full subtree snapshot) +// current.wal (WAL format: append-only per-entry log) // host002/ // ... package metricstore @@ -52,7 +53,6 @@ import ( cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" - "github.com/linkedin/goavro/v2" ) const ( @@ -86,47 +86,58 @@ var ( // Checkpointing starts a background worker that periodically saves metric data to disk. // -// The behavior depends on the configured file format: -// - JSON: Periodic checkpointing based on Keys.Checkpoints.Interval -// - Avro: Initial delay + periodic checkpointing at DefaultAvroCheckpointInterval -// -// The worker respects context cancellation and signals completion via the WaitGroup. +// Format behaviour: +// - "json": Periodic checkpointing based on Keys.Checkpoints.Interval +// - "wal": Periodic binary snapshots + WAL rotation at Keys.Checkpoints.Interval func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { lastCheckpointMu.Lock() lastCheckpoint = time.Now() lastCheckpointMu.Unlock() - if Keys.Checkpoints.FileFormat == "json" { - ms := GetMemoryStore() + ms := GetMemoryStore() - wg.Add(1) - go func() { - defer wg.Done() - d, err := time.ParseDuration(Keys.Checkpoints.Interval) - if err != nil { - cclog.Fatalf("[METRICSTORE]> invalid checkpoint interval '%s': %s", Keys.Checkpoints.Interval, err.Error()) - } - if d <= 0 { - cclog.Warnf("[METRICSTORE]> checkpoint interval is zero or negative (%s), checkpointing disabled", d) + wg.Add(1) + go func() { + defer wg.Done() + + d, err := time.ParseDuration(Keys.Checkpoints.Interval) + if err != nil { + cclog.Fatalf("[METRICSTORE]> invalid checkpoint interval '%s': %s", Keys.Checkpoints.Interval, err.Error()) + } + if d <= 0 { + cclog.Warnf("[METRICSTORE]> checkpoint interval is zero or negative (%s), checkpointing disabled", d) + return + } + + ticker := time.NewTicker(d) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): return - } + case <-ticker.C: + lastCheckpointMu.Lock() + from := lastCheckpoint + lastCheckpointMu.Unlock() - ticker := time.NewTicker(d) - defer ticker.Stop() + now := time.Now() + cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339)) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - lastCheckpointMu.Lock() - from := lastCheckpoint - lastCheckpointMu.Unlock() - - cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339)) - now := time.Now() - n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, - from.Unix(), now.Unix()) + if Keys.Checkpoints.FileFormat == "wal" { + n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix()) + if err != nil { + cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error()) + } else { + cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n) + lastCheckpointMu.Lock() + lastCheckpoint = now + lastCheckpointMu.Unlock() + // Rotate WAL files for successfully checkpointed hosts. + RotateWALFiles(hostDirs) + } + } else { + n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix()) if err != nil { cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error()) } else { @@ -137,32 +148,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { } } } - }() - } else { - wg.Add(1) - go func() { - defer wg.Done() - - select { - case <-ctx.Done(): - return - case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute): - GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - } - - ticker := time.NewTicker(DefaultAvroCheckpointInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - } - } - }() - } + } + }() } // MarshalJSON provides optimized JSON encoding for CheckpointMetrics. @@ -190,7 +177,7 @@ func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) { return buf, nil } -// ToCheckpoint writes metric data to checkpoint files in parallel. +// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format). // // Metrics at root and cluster levels are skipped. One file per host is created. // Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host @@ -378,7 +365,6 @@ func enqueueCheckpointHosts(dir string, work chan<- [2]string) error { return err } - gcCounter := 0 for _, clusterDir := range clustersDir { if !clusterDir.IsDir() { return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory") @@ -394,16 +380,6 @@ func enqueueCheckpointHosts(dir string, work chan<- [2]string) error { return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory") } - gcCounter++ - // if gcCounter%GCTriggerInterval == 0 { - // Forcing garbage collection runs here regulary during the loading of checkpoints - // will decrease the total heap size after loading everything back to memory is done. - // While loading data, the heap will grow fast, so the GC target size will double - // almost always. By forcing GCs here, we can keep it growing more slowly so that - // at the end, less memory is wasted. - // runtime.GC() - // } - work <- [2]string{clusterDir.Name(), hostDir.Name()} } } @@ -413,8 +389,8 @@ func enqueueCheckpointHosts(dir string, work chan<- [2]string) error { // FromCheckpoint loads checkpoint files from disk into memory in parallel. // -// Uses worker pool to load cluster/host combinations. Periodically triggers GC -// to prevent excessive heap growth. Returns number of files loaded and any errors. +// Uses worker pool to load cluster/host combinations. Returns number of files +// loaded and any errors. func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) { var wg sync.WaitGroup work := make(chan [2]string, Keys.NumWorkers*4) @@ -452,13 +428,11 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) { // FromCheckpointFiles is the main entry point for loading checkpoints at startup. // -// Automatically detects checkpoint format (JSON vs Avro) and falls back if needed. // Creates checkpoint directory if it doesn't exist. This function must be called // before any writes or reads, and can only be called once. func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) { if _, err := os.Stat(dir); os.IsNotExist(err) { - // The directory does not exist, so create it using os.MkdirAll() - err := os.MkdirAll(dir, CheckpointDirPerms) // CheckpointDirPerms sets the permissions for the directory + err := os.MkdirAll(dir, CheckpointDirPerms) if err != nil { cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err) } @@ -468,146 +442,6 @@ func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) { return m.FromCheckpoint(dir, from) } -func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error { - br := bufio.NewReader(f) - - fileName := f.Name()[strings.LastIndex(f.Name(), "/")+1:] - resolution, err := strconv.ParseInt(fileName[0:strings.Index(fileName, "_")], 10, 64) - if err != nil { - return fmt.Errorf("[METRICSTORE]> error while reading avro file (resolution parsing) : %s", err) - } - - fromTimestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64) - - // Same logic according to lineprotocol - fromTimestamp -= (resolution / 2) - - if err != nil { - return fmt.Errorf("[METRICSTORE]> error converting timestamp from the avro file : %s", err) - } - - // fmt.Printf("File : %s with resolution : %d\n", fileName, resolution) - - var recordCounter int64 = 0 - - // Create a new OCF reader from the buffered reader - ocfReader, err := goavro.NewOCFReader(br) - if err != nil { - return fmt.Errorf("[METRICSTORE]> error creating OCF reader: %w", err) - } - - metricsData := make(map[string]schema.FloatArray) - - for ocfReader.Scan() { - datum, err := ocfReader.Read() - if err != nil { - return fmt.Errorf("[METRICSTORE]> error while reading avro file : %s", err) - } - - record, ok := datum.(map[string]any) - if !ok { - return fmt.Errorf("[METRICSTORE]> failed to assert datum as map[string]interface{}") - } - - for key, value := range record { - metricsData[key] = append(metricsData[key], schema.ConvertToFloat(value.(float64))) - } - - recordCounter += 1 - } - - to := (fromTimestamp + (recordCounter / (60 / resolution) * 60)) - if to < from { - return nil - } - - for key, floatArray := range metricsData { - metricName := ReplaceKey(key) - - if strings.Contains(metricName, SelectorDelimiter) { - subString := strings.Split(metricName, SelectorDelimiter) - - lvl := l - - for i := 0; i < len(subString)-1; i++ { - - sel := subString[i] - - if lvl.children == nil { - lvl.children = make(map[string]*Level) - } - - child, ok := lvl.children[sel] - if !ok { - child = &Level{ - metrics: make([]*buffer, len(m.Metrics)), - children: nil, - } - lvl.children[sel] = child - } - lvl = child - } - - leafMetricName := subString[len(subString)-1] - err = lvl.createBuffer(m, leafMetricName, floatArray, fromTimestamp, resolution) - if err != nil { - return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err) - } - } else { - err = l.createBuffer(m, metricName, floatArray, fromTimestamp, resolution) - if err != nil { - return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err) - } - } - - } - - return nil -} - -func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schema.FloatArray, from int64, resolution int64) error { - n := len(floatArray) - b := &buffer{ - frequency: resolution, - start: from, - data: floatArray[0:n:n], - prev: nil, - next: nil, - archived: true, - } - - minfo, ok := m.Metrics[metricName] - if !ok { - return nil - } - - prev := l.metrics[minfo.offset] - if prev == nil { - l.metrics[minfo.offset] = b - } else { - if prev.start > b.start { - return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start) - } - - b.prev = prev - prev.next = b - - missingCount := ((int(b.start) - int(prev.start)) - len(prev.data)*int(b.frequency)) - if missingCount > 0 { - missingCount /= int(b.frequency) - - for range missingCount { - prev.data = append(prev.data, schema.NaN) - } - - prev.data = prev.data[0:len(prev.data):len(prev.data)] - } - } - l.metrics[minfo.offset] = b - - return nil -} - func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error { br := bufio.NewReader(f) cf := &CheckpointFile{} @@ -679,37 +513,37 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error { return nil } +// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a +// single host directory. Snapshot files are loaded first (sorted by timestamp), +// then current.wal is replayed on top. func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) { direntries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { return 0, nil } - return 0, err } allFiles := make([]fs.DirEntry, 0) + var walEntry fs.DirEntry filesLoaded := 0 + for _, e := range direntries { if e.IsDir() { - child := &Level{ - metrics: make([]*buffer, len(m.Metrics)), - children: make(map[string]*Level), - } - - files, err := child.fromCheckpoint(m, path.Join(dir, e.Name()), from) - filesLoaded += files - if err != nil { - return filesLoaded, err - } - - l.children[e.Name()] = child - } else if strings.HasSuffix(e.Name(), ".json") || strings.HasSuffix(e.Name(), ".avro") { - allFiles = append(allFiles, e) - } else { + // Legacy: skip subdirectories (only used by old Avro format). + // These are ignored; their data is not loaded. + cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir) continue } + + name := e.Name() + if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") { + allFiles = append(allFiles, e) + } else if name == "current.wal" { + walEntry = e + } + // Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes). } files, err := findFiles(allFiles, from, true) @@ -719,54 +553,81 @@ func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, err loaders := map[string]func(*MemoryStore, *os.File, int64) error{ ".json": l.loadJSONFile, - ".avro": l.loadAvroFile, + ".bin": l.loadBinaryFile, } for _, filename := range files { ext := filepath.Ext(filename) loader := loaders[ext] if loader == nil { - cclog.Warnf("Unknown extension for file %s", filename) + cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename) continue } - // Use a closure to ensure file is closed immediately after use err := func() error { f, err := os.Open(path.Join(dir, filename)) if err != nil { return err } defer f.Close() - return loader(m, f, from) }() if err != nil { return filesLoaded, err } + filesLoaded++ + } - filesLoaded += 1 + // Replay WAL after all snapshot files so it fills in data since the last snapshot. + if walEntry != nil { + err := func() error { + f, err := os.Open(path.Join(dir, walEntry.Name())) + if err != nil { + return err + } + defer f.Close() + return l.loadWALFile(m, f, from) + }() + if err != nil { + // WAL errors are non-fatal: the snapshot already loaded the bulk of data. + cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err) + } else { + filesLoaded++ + } } return filesLoaded, nil } -// This will probably get very slow over time! -// A solution could be some sort of an index file in which all other files -// and the timespan they contain is listed. -// NOTE: This now assumes that you have distinct timestamps for json and avro files -// Also, it assumes that the timestamps are not overlapping/self-modified. +// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename. +// Supports ".json" (format: ".json") and ".bin" (format: ".bin"). +func parseTimestampFromFilename(name string) (int64, error) { + switch { + case strings.HasSuffix(name, ".json"): + return strconv.ParseInt(name[:len(name)-5], 10, 64) + case strings.HasSuffix(name, ".bin"): + return strconv.ParseInt(name[:len(name)-4], 10, 64) + default: + return 0, fmt.Errorf("unknown checkpoint extension for file %q", name) + } +} + +// findFiles returns filenames from direntries whose timestamps satisfy the filter. +// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the +// last file before t if t falls between two files). func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) { nums := map[string]int64{} for _, e := range direntries { - if !strings.HasSuffix(e.Name(), ".json") && !strings.HasSuffix(e.Name(), ".avro") { + name := e.Name() + if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") { continue } - ts, err := strconv.ParseInt(e.Name()[strings.Index(e.Name(), "_")+1:len(e.Name())-5], 10, 64) + ts, err := parseTimestampFromFilename(name) if err != nil { return nil, err } - nums[e.Name()] = ts + nums[name] = ts } sort.Slice(direntries, func(i, j int) bool { @@ -783,16 +644,12 @@ func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]s for i, e := range direntries { ts1 := nums[e.Name()] - // Logic to look for files in forward or direction - // If logic: All files greater than or after - // the given timestamp will be selected - // Else If logic: All files less than or before - // the given timestamp will be selected if findMoreRecentFiles && t <= ts1 { filenames = append(filenames, e.Name()) } else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 { filenames = append(filenames, e.Name()) } + if i == len(direntries)-1 { continue } diff --git a/pkg/metricstore/config.go b/pkg/metricstore/config.go index 1efee61a..53716967 100644 --- a/pkg/metricstore/config.go +++ b/pkg/metricstore/config.go @@ -14,7 +14,7 @@ // ├─ RetentionInMemory: How long to keep data in RAM // ├─ MemoryCap: Memory limit in bytes (triggers forceFree) // ├─ Checkpoints: Persistence configuration -// │ ├─ FileFormat: "avro" or "json" +// │ ├─ FileFormat: "json" or "wal" // │ ├─ Interval: How often to save (e.g., "1h") // │ └─ RootDir: Checkpoint storage path // ├─ Cleanup: Long-term storage configuration @@ -55,16 +55,13 @@ const ( DefaultMaxWorkers = 10 DefaultBufferCapacity = 512 DefaultGCTriggerInterval = 100 - DefaultAvroWorkers = 4 - DefaultCheckpointBufferMin = 3 - DefaultAvroCheckpointInterval = time.Minute DefaultMemoryUsageTrackerInterval = 1 * time.Hour ) // Checkpoints configures periodic persistence of in-memory metric data. // // Fields: -// - FileFormat: "avro" (default, binary, compact) or "json" (human-readable, slower) +// - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe) // - Interval: Duration string (e.g., "1h", "30m") between checkpoint saves // - RootDir: Filesystem path for checkpoint files (created if missing) type Checkpoints struct { diff --git a/pkg/metricstore/configSchema.go b/pkg/metricstore/configSchema.go index 6a748be0..67f30976 100644 --- a/pkg/metricstore/configSchema.go +++ b/pkg/metricstore/configSchema.go @@ -18,7 +18,7 @@ const configSchema = `{ "type": "object", "properties": { "file-format": { - "description": "Specify the format for checkpoint files. There are 2 variants: 'avro' and 'json'. If nothing is specified, 'avro' is default.", + "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'json'.", "type": "string" }, "interval": { diff --git a/pkg/metricstore/lineprotocol.go b/pkg/metricstore/lineprotocol.go index bfbbef2d..1e04bba0 100644 --- a/pkg/metricstore/lineprotocol.go +++ b/pkg/metricstore/lineprotocol.go @@ -244,8 +244,8 @@ func DecodeLine(dec *lineprotocol.Decoder, time := t.Unix() - if Keys.Checkpoints.FileFormat != "json" { - LineProtocolMessages <- &AvroStruct{ + if Keys.Checkpoints.FileFormat == "wal" { + WALMessages <- &WALMessage{ MetricName: string(metricBuf), Cluster: cluster, Node: host, diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index 789c6d07..3fe64d55 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -172,7 +172,7 @@ func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.W Retention(wg, ctx) Checkpointing(wg, ctx) CleanUp(wg, ctx) - DataStaging(wg, ctx) + WALStaging(wg, ctx) MemoryUsageTracker(wg, ctx) // Note: Signal handling has been removed from this function. @@ -264,7 +264,7 @@ func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) { // // The function will: // 1. Cancel the context to stop all background workers -// 2. Close NATS message channels if using Avro format +// 2. Close the WAL messages channel if using WAL format // 3. Write a final checkpoint to preserve in-memory data // 4. Log any errors encountered during shutdown // @@ -276,8 +276,8 @@ func Shutdown() { shutdownFunc() } - if Keys.Checkpoints.FileFormat != "json" { - close(LineProtocolMessages) + if Keys.Checkpoints.FileFormat == "wal" { + close(WALMessages) } cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir) @@ -286,10 +286,18 @@ func Shutdown() { ms := GetMemoryStore() - if Keys.Checkpoints.FileFormat == "json" { - files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix()) + lastCheckpointMu.Lock() + from := lastCheckpoint + lastCheckpointMu.Unlock() + + if Keys.Checkpoints.FileFormat == "wal" { + var hostDirs []string + files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix()) + if err == nil { + RotateWALFiles(hostDirs) + } } else { - files, err = GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, true) + files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix()) } if err != nil { diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go new file mode 100644 index 00000000..e8a71ce2 --- /dev/null +++ b/pkg/metricstore/walCheckpoint.go @@ -0,0 +1,787 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation. +// +// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL) +// per host, eliminating the extra memory overhead of the AvroStore and providing +// truly continuous (per-write) crash safety. +// +// # Architecture +// +// Metric write (DecodeLine) +// │ +// ├─► WriteToLevel() → main MemoryStore (unchanged) +// │ +// └─► WALMessages channel +// │ +// ▼ +// WALStaging goroutine +// │ +// ▼ +// checkpoints/cluster/host/current.wal (append-only, binary) +// +// Periodic checkpoint (Checkpointing goroutine): +// 1. Write .bin snapshot (column-oriented, from main tree) +// 2. Signal WALStaging to truncate current.wal per host +// +// On restart (FromCheckpoint): +// 1. Load most recent .bin snapshot +// 2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps) +// +// # WAL Record Format +// +// [4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32] +// +// payload: +// [8B timestamp int64] +// [2B metric_name_len uint16][N metric name bytes] +// [1B selector_count uint8] +// per selector: [1B selector_len uint8][M selector bytes] +// [4B value float32 bits] +// +// # Binary Snapshot Format +// +// [4B magic 0xCC5B0001][8B from int64][8B to int64] +// Level tree (recursive): +// [4B num_metrics uint32] +// per metric: +// [2B name_len uint16][N name bytes] +// [8B frequency int64][8B start int64] +// [4B num_values uint32][num_values × 4B float32] +// [4B num_children uint32] +// per child: [2B name_len uint16][N name bytes] + Level (recursive) +package metricstore + +import ( + "bufio" + "context" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "math" + "os" + "path" + "sync" + "sync/atomic" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +// Magic numbers for binary formats. +const ( + walFileMagic = uint32(0xCC1DA701) // WAL file header magic + walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic + snapFileMagic = uint32(0xCC5B0001) // Binary snapshot magic +) + +// WALMessages is the channel for sending metric writes to the WAL staging goroutine. +// Buffered to allow burst writes without blocking the metric ingestion path. +var WALMessages = make(chan *WALMessage, 4096) + +// walRotateCh is used by the checkpoint goroutine to request WAL file rotation +// (close, delete, reopen) after a binary snapshot has been written. +var walRotateCh = make(chan walRotateReq, 256) + +// WALMessage represents a single metric write to be appended to the WAL. +// Cluster and Node are NOT stored in the WAL record (inferred from file path). +type WALMessage struct { + MetricName string + Cluster string + Node string + Selector []string + Value schema.Float + Timestamp int64 +} + +// walRotateReq requests WAL file rotation for a specific host directory. +// The done channel is closed by the WAL goroutine when rotation is complete. +type walRotateReq struct { + hostDir string + done chan struct{} +} + +// walFileState holds an open WAL file handle for one host directory. +type walFileState struct { + f *os.File +} + +// WALStaging starts a background goroutine that receives WALMessage items +// and appends binary WAL records to per-host current.wal files. +// Also handles WAL rotation requests from the checkpoint goroutine. +func WALStaging(wg *sync.WaitGroup, ctx context.Context) { + wg.Add(1) + go func() { + defer wg.Done() + + if Keys.Checkpoints.FileFormat == "json" { + return + } + + hostFiles := make(map[string]*walFileState) + + defer func() { + for _, ws := range hostFiles { + if ws.f != nil { + ws.f.Close() + } + } + }() + + getOrOpenWAL := func(hostDir string) *os.File { + ws, ok := hostFiles[hostDir] + if ok { + return ws.f + } + + if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil { + cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err) + return nil + } + + walPath := path.Join(hostDir, "current.wal") + f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms) + if err != nil { + cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err) + return nil + } + + // Write file header magic if file is new (empty). + info, err := f.Stat() + if err == nil && info.Size() == 0 { + var hdr [4]byte + binary.LittleEndian.PutUint32(hdr[:], walFileMagic) + if _, err := f.Write(hdr[:]); err != nil { + cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err) + f.Close() + return nil + } + } + + hostFiles[hostDir] = &walFileState{f: f} + return f + } + + processMsg := func(msg *WALMessage) { + hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node) + f := getOrOpenWAL(hostDir) + if f == nil { + return + } + if err := writeWALRecord(f, msg); err != nil { + cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err) + } + } + + processRotate := func(req walRotateReq) { + ws, ok := hostFiles[req.hostDir] + if ok && ws.f != nil { + ws.f.Close() + walPath := path.Join(req.hostDir, "current.wal") + if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) { + cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err) + } + delete(hostFiles, req.hostDir) + } + close(req.done) + } + + drain := func() { + for { + select { + case msg, ok := <-WALMessages: + if !ok { + return + } + processMsg(msg) + case req := <-walRotateCh: + processRotate(req) + default: + return + } + } + } + + for { + select { + case <-ctx.Done(): + drain() + return + case msg, ok := <-WALMessages: + if !ok { + return + } + processMsg(msg) + case req := <-walRotateCh: + processRotate(req) + } + } + }() +} + +// RotateWALFiles sends rotation requests for the given host directories +// and blocks until all rotations complete. +func RotateWALFiles(hostDirs []string) { + dones := make([]chan struct{}, len(hostDirs)) + for i, dir := range hostDirs { + dones[i] = make(chan struct{}) + walRotateCh <- walRotateReq{hostDir: dir, done: dones[i]} + } + for _, done := range dones { + <-done + } +} + +// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC). +func buildWALPayload(msg *WALMessage) []byte { + size := 8 + 2 + len(msg.MetricName) + 1 + 4 + for _, s := range msg.Selector { + size += 1 + len(s) + } + + buf := make([]byte, 0, size) + + // Timestamp (8 bytes, little-endian int64) + var ts [8]byte + binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp)) + buf = append(buf, ts[:]...) + + // Metric name (2-byte length prefix + bytes) + var mLen [2]byte + binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName))) + buf = append(buf, mLen[:]...) + buf = append(buf, msg.MetricName...) + + // Selector count (1 byte) + buf = append(buf, byte(len(msg.Selector))) + + // Selectors (1-byte length prefix + bytes each) + for _, sel := range msg.Selector { + buf = append(buf, byte(len(sel))) + buf = append(buf, sel...) + } + + // Value (4 bytes, float32 bit representation) + var val [4]byte + binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value))) + buf = append(buf, val[:]...) + + return buf +} + +// writeWALRecord appends a binary WAL record to the file. +// Format: [4B magic][4B payload_len][payload][4B CRC32] +func writeWALRecord(f *os.File, msg *WALMessage) error { + payload := buildWALPayload(msg) + crc := crc32.ChecksumIEEE(payload) + + record := make([]byte, 0, 4+4+len(payload)+4) + + var magic [4]byte + binary.LittleEndian.PutUint32(magic[:], walRecordMagic) + record = append(record, magic[:]...) + + var pLen [4]byte + binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload))) + record = append(record, pLen[:]...) + + record = append(record, payload...) + + var crcBytes [4]byte + binary.LittleEndian.PutUint32(crcBytes[:], crc) + record = append(record, crcBytes[:]...) + + _, err := f.Write(record) + return err +} + +// readWALRecord reads one WAL record from the reader. +// Returns (nil, nil) on clean EOF. Returns error on data corruption. +// A CRC mismatch indicates a truncated trailing record (expected on crash). +func readWALRecord(r io.Reader) (*WALMessage, error) { + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + if err == io.EOF { + return nil, nil // Clean EOF + } + return nil, fmt.Errorf("read record magic: %w", err) + } + + if magic != walRecordMagic { + return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic) + } + + var payloadLen uint32 + if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil { + return nil, fmt.Errorf("read payload length: %w", err) + } + + if payloadLen > 1<<20 { // 1 MB sanity limit + return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen) + } + + payload := make([]byte, payloadLen) + if _, err := io.ReadFull(r, payload); err != nil { + return nil, fmt.Errorf("read payload: %w", err) + } + + var storedCRC uint32 + if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil { + return nil, fmt.Errorf("read CRC: %w", err) + } + + if crc32.ChecksumIEEE(payload) != storedCRC { + return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)") + } + + return parseWALPayload(payload) +} + +// parseWALPayload decodes a binary payload into a WALMessage. +func parseWALPayload(payload []byte) (*WALMessage, error) { + if len(payload) < 8+2+1+4 { + return nil, fmt.Errorf("payload too short: %d bytes", len(payload)) + } + + offset := 0 + + // Timestamp (8 bytes) + ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8])) + offset += 8 + + // Metric name (2-byte length + bytes) + if offset+2 > len(payload) { + return nil, fmt.Errorf("metric name length overflows payload") + } + mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2])) + offset += 2 + + if offset+mLen > len(payload) { + return nil, fmt.Errorf("metric name overflows payload") + } + metricName := string(payload[offset : offset+mLen]) + offset += mLen + + // Selector count (1 byte) + if offset >= len(payload) { + return nil, fmt.Errorf("selector count overflows payload") + } + selCount := int(payload[offset]) + offset++ + + selectors := make([]string, selCount) + for i := range selCount { + if offset >= len(payload) { + return nil, fmt.Errorf("selector[%d] length overflows payload", i) + } + sLen := int(payload[offset]) + offset++ + + if offset+sLen > len(payload) { + return nil, fmt.Errorf("selector[%d] data overflows payload", i) + } + selectors[i] = string(payload[offset : offset+sLen]) + offset += sLen + } + + // Value (4 bytes, float32 bits) + if offset+4 > len(payload) { + return nil, fmt.Errorf("value overflows payload") + } + bits := binary.LittleEndian.Uint32(payload[offset : offset+4]) + value := schema.Float(math.Float32frombits(bits)) + + return &WALMessage{ + MetricName: metricName, + Timestamp: ts, + Selector: selectors, + Value: value, + }, nil +} + +// loadWALFile reads a WAL file and replays all valid records into the Level tree. +// l is the host-level node. Corrupt or partial trailing records are silently skipped +// (expected on crash). Records older than 'from' are skipped. +func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error { + br := bufio.NewReader(f) + + // Verify file header magic. + var fileMagic uint32 + if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil { + if err == io.EOF { + return nil // Empty file, no data + } + return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err) + } + + if fileMagic != walFileMagic { + return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic) + } + + // Cache level lookups to avoid repeated tree traversal. + lvlCache := make(map[string]*Level) + + for { + msg, err := readWALRecord(br) + if err != nil { + // Truncated trailing record is expected after a crash; stop replaying. + cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err) + break + } + if msg == nil { + break // Clean EOF + } + + if msg.Timestamp < from { + continue // Older than retention window + } + + minfo, ok := m.Metrics[msg.MetricName] + if !ok { + continue // Unknown metric (config may have changed) + } + + // Cache key is the null-separated selector path. + cacheKey := joinSelector(msg.Selector) + lvl, ok := lvlCache[cacheKey] + if !ok { + lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics)) + lvlCache[cacheKey] = lvl + } + + // Write directly to the buffer, same as WriteToLevel but without the + // global level lookup (we already have the right level). + lvl.lock.Lock() + b := lvl.metrics[minfo.offset] + if b == nil { + b = newBuffer(msg.Timestamp, minfo.Frequency) + lvl.metrics[minfo.offset] = b + } + nb, writeErr := b.write(msg.Timestamp, msg.Value) + if writeErr == nil && b != nb { + lvl.metrics[minfo.offset] = nb + } + // Ignore write errors for timestamps before buffer start (can happen when + // replaying WAL entries that predate a loaded snapshot's start time). + lvl.lock.Unlock() + } + + return nil +} + +// joinSelector builds a cache key from a selector slice using null bytes as separators. +func joinSelector(sel []string) string { + if len(sel) == 0 { + return "" + } + result := sel[0] + for i := 1; i < len(sel); i++ { + result += "\x00" + sel[i] + } + return result +} + +// ToCheckpointWAL writes binary snapshot files for all hosts in parallel. +// Returns the number of files written, the list of host directories that were +// successfully checkpointed (for WAL rotation), and any errors. +func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) { + // Collect all cluster/host pairs. + m.root.lock.RLock() + totalHosts := 0 + for _, l1 := range m.root.children { + l1.lock.RLock() + totalHosts += len(l1.children) + l1.lock.RUnlock() + } + m.root.lock.RUnlock() + + levels := make([]*Level, 0, totalHosts) + selectors := make([][]string, 0, totalHosts) + + m.root.lock.RLock() + for sel1, l1 := range m.root.children { + l1.lock.RLock() + for sel2, l2 := range l1.children { + levels = append(levels, l2) + selectors = append(selectors, []string{sel1, sel2}) + } + l1.lock.RUnlock() + } + m.root.lock.RUnlock() + + type workItem struct { + level *Level + hostDir string + selector []string + } + + n, errs := int32(0), int32(0) + var successDirs []string + var successMu sync.Mutex + + var wg sync.WaitGroup + wg.Add(Keys.NumWorkers) + work := make(chan workItem, Keys.NumWorkers*2) + + for range Keys.NumWorkers { + go func() { + defer wg.Done() + for wi := range work { + err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m) + if err != nil { + if err == ErrNoNewArchiveData { + continue + } + cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err) + atomic.AddInt32(&errs, 1) + } else { + atomic.AddInt32(&n, 1) + successMu.Lock() + successDirs = append(successDirs, wi.hostDir) + successMu.Unlock() + } + } + }() + } + + for i := range levels { + hostDir := path.Join(dir, path.Join(selectors[i]...)) + work <- workItem{ + level: levels[i], + hostDir: hostDir, + selector: selectors[i], + } + } + close(work) + wg.Wait() + + if errs > 0 { + return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n) + } + return int(n), successDirs, nil +} + +// toCheckpointBinary writes a binary snapshot file for a single host-level node. +// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash. +func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error { + cf, err := l.toCheckpointFile(from, to, m) + if err != nil { + return err + } + if cf == nil { + return ErrNoNewArchiveData + } + + if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + + // Write to a temp file first, then rename (atomic on POSIX). + tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from)) + finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from)) + + f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms) + if err != nil { + return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err) + } + + bw := bufio.NewWriter(f) + if err := writeBinarySnapshotFile(bw, cf); err != nil { + f.Close() + os.Remove(tmpPath) + return fmt.Errorf("write binary snapshot: %w", err) + } + if err := bw.Flush(); err != nil { + f.Close() + os.Remove(tmpPath) + return err + } + f.Close() + + return os.Rename(tmpPath, finalPath) +} + +// writeBinarySnapshotFile writes the binary snapshot file header and level tree. +func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error { + if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil { + return err + } + return writeBinaryLevel(w, cf) +} + +// writeBinaryLevel recursively writes a CheckpointFile level in binary format. +func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error { + if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil { + return err + } + + for name, metric := range cf.Metrics { + if err := writeString16(w, name); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil { + return err + } + for _, v := range metric.Data { + if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil { + return err + } + } + } + + if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil { + return err + } + + for name, child := range cf.Children { + if err := writeString16(w, name); err != nil { + return err + } + if err := writeBinaryLevel(w, child); err != nil { + return err + } + } + + return nil +} + +// writeString16 writes a 2-byte length-prefixed string to w. +func writeString16(w io.Writer, s string) error { + if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil { + return err + } + _, err := io.WriteString(w, s) + return err +} + +// loadBinaryFile reads a binary snapshot file and loads data into the Level tree. +// The retention check (from) is applied to the file's 'to' timestamp. +func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error { + br := bufio.NewReader(f) + + var magic uint32 + if err := binary.Read(br, binary.LittleEndian, &magic); err != nil { + return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err) + } + if magic != snapFileMagic { + return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic) + } + + var fileFrom, fileTo int64 + if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil { + return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err) + } + if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil { + return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err) + } + + if fileTo != 0 && fileTo < from { + return nil // File is older than retention window, skip it + } + + cf, err := readBinaryLevel(br) + if err != nil { + return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err) + } + cf.From = fileFrom + cf.To = fileTo + + return l.loadFile(cf, m) +} + +// readBinaryLevel recursively reads a level from the binary snapshot format. +func readBinaryLevel(r io.Reader) (*CheckpointFile, error) { + cf := &CheckpointFile{ + Metrics: make(map[string]*CheckpointMetrics), + Children: make(map[string]*CheckpointFile), + } + + var numMetrics uint32 + if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil { + return nil, fmt.Errorf("read num_metrics: %w", err) + } + + for range numMetrics { + name, err := readString16(r) + if err != nil { + return nil, fmt.Errorf("read metric name: %w", err) + } + + var freq, start int64 + if err := binary.Read(r, binary.LittleEndian, &freq); err != nil { + return nil, fmt.Errorf("read frequency for %s: %w", name, err) + } + if err := binary.Read(r, binary.LittleEndian, &start); err != nil { + return nil, fmt.Errorf("read start for %s: %w", name, err) + } + + var numValues uint32 + if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil { + return nil, fmt.Errorf("read num_values for %s: %w", name, err) + } + + data := make([]schema.Float, numValues) + for i := range numValues { + var bits uint32 + if err := binary.Read(r, binary.LittleEndian, &bits); err != nil { + return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err) + } + data[i] = schema.Float(math.Float32frombits(bits)) + } + + cf.Metrics[name] = &CheckpointMetrics{ + Frequency: freq, + Start: start, + Data: data, + } + } + + var numChildren uint32 + if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil { + return nil, fmt.Errorf("read num_children: %w", err) + } + + for range numChildren { + childName, err := readString16(r) + if err != nil { + return nil, fmt.Errorf("read child name: %w", err) + } + + child, err := readBinaryLevel(r) + if err != nil { + return nil, fmt.Errorf("read child %s: %w", childName, err) + } + cf.Children[childName] = child + } + + return cf, nil +} + +// readString16 reads a 2-byte length-prefixed string from r. +func readString16(r io.Reader) (string, error) { + var sLen uint16 + if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil { + return "", err + } + buf := make([]byte, sLen) + if _, err := io.ReadFull(r, buf); err != nil { + return "", err + } + return string(buf), nil +} From 348b6010e8eeeb4c80d9c1c4b5a1e737374dd229 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 26 Feb 2026 15:09:01 +0100 Subject: [PATCH 299/341] fix typo preventing template condition to work --- internal/auth/auth.go | 2 +- web/templates/login.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 9b1e2121..69f4f078 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -263,7 +263,7 @@ func GetAuthInstance() *Authentication { } // handleUserSync syncs or updates a user in the database based on configuration. -// This is used for both JWT and OIDC authentication when syncUserOnLogin or updateUserOnLogin is enabled. +// This is used for LDAP, JWT and OIDC authentications when syncUserOnLogin or updateUserOnLogin is enabled. func handleUserSync(user *schema.User, syncUserOnLogin, updateUserOnLogin bool) { r := repository.GetUserRepository() dbUser, err := r.GetUser(user.Username) diff --git a/web/templates/login.tmpl b/web/templates/login.tmpl index cd139261..4c4d9be8 100644 --- a/web/templates/login.tmpl +++ b/web/templates/login.tmpl @@ -38,8 +38,8 @@
    - {{- if .Infos.hasOpenIDConnect}} - OpenID Connect Login + {{if .Infos.hasOpenIDConnect}} + OpenID Connect Login {{end}} From 6ecb9349677d5d2a399aa26b86d12c987a7cb3ef Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 27 Feb 2026 08:55:33 +0100 Subject: [PATCH 300/341] Switch to CC line-protocol package. Update cc-lib. --- go.mod | 14 +++++----- go.sum | 48 +++++++++++---------------------- internal/api/metricstore.go | 2 +- internal/api/nats.go | 2 +- pkg/metricstore/lineprotocol.go | 2 +- 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index e244062c..afc21f2a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ tool ( require ( github.com/99designs/gqlgen v0.17.86 - github.com/ClusterCockpit/cc-lib/v2 v2.6.0 + github.com/ClusterCockpit/cc-lib/v2 v2.7.0 + github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.8 @@ -25,7 +26,6 @@ require ( github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/gops v0.3.29 github.com/gorilla/sessions v1.4.0 - github.com/influxdata/line-protocol/v2 v2.2.1 github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/linkedin/goavro/v2 v2.15.0 @@ -92,10 +92,10 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/nats-io/nats.go v1.48.0 // indirect + github.com/nats-io/nats.go v1.49.0 // indirect github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/oapi-codegen/runtime v1.1.2 // indirect + github.com/oapi-codegen/runtime v1.2.0 // indirect github.com/parquet-go/bitpack v1.0.0 // indirect github.com/parquet-go/jsonlite v1.4.0 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect @@ -104,7 +104,7 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sosodev/duration v1.3.1 // indirect - github.com/stmcginnis/gofish v0.21.1 // indirect + github.com/stmcginnis/gofish v0.21.3 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/swaggo/files v1.0.1 // indirect github.com/twpayne/go-geom v1.6.1 // indirect @@ -113,9 +113,9 @@ require ( github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/mod v0.33.0 // indirect - golang.org/x/net v0.50.0 // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect diff --git a/go.sum b/go.sum index f2929454..cedddd62 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,10 @@ github.com/99designs/gqlgen v0.17.86 h1:C8N3UTa5heXX6twl+b0AJyGkTwYL6dNmFrgZNLRc github.com/99designs/gqlgen v0.17.86/go.mod h1:KTrPl+vHA1IUzNlh4EYkl7+tcErL3MgKnhHrBcV74Fw= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= -github.com/ClusterCockpit/cc-lib/v2 v2.5.1 h1:s6M9tyPDty+4zTdQGJYKpGJM9Nz7N6ITMdjPvNSLX5g= -github.com/ClusterCockpit/cc-lib/v2 v2.5.1/go.mod h1:DZ8OIHPUZJpWqErLITt0B8P6/Q7CBW2IQSQ5YiFFaG0= -github.com/ClusterCockpit/cc-lib/v2 v2.6.0 h1:Q7zvRAVhfYA9PDB18pfY9A/6Ws4oWpnv8+P9MBRUDzg= -github.com/ClusterCockpit/cc-lib/v2 v2.6.0/go.mod h1:DZ8OIHPUZJpWqErLITt0B8P6/Q7CBW2IQSQ5YiFFaG0= +github.com/ClusterCockpit/cc-lib/v2 v2.7.0 h1:EMTShk6rMTR1wlfmQ8SVCawH1OdltUbD3kVQmaW+5pE= +github.com/ClusterCockpit/cc-lib/v2 v2.7.0/go.mod h1:0Etx8WMs0lYZ4tiOQizY18CQop+2i3WROvU9rMUxHA4= +github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q= +github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -95,8 +95,6 @@ github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7c github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM= github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= -github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= -github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -154,8 +152,6 @@ github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjY github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA= @@ -184,13 +180,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjw github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98= github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig= github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo= -github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY= -github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY= -github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE= -github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -212,11 +203,8 @@ github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7X github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= @@ -240,15 +228,14 @@ github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g= github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs= github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y= -github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= -github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= +github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw= github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI= -github.com/oapi-codegen/runtime v1.1.2/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4= +github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= @@ -268,8 +255,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= -github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q= +github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0= github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -286,8 +273,8 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= -github.com/stmcginnis/gofish v0.21.1 h1:sutDvBhmLh4RDOZ1DN8GUyYRu7f1ggvKMMnSaiqhwn4= -github.com/stmcginnis/gofish v0.21.1/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= +github.com/stmcginnis/gofish v0.21.3 h1:EBLCHfORnbx7MPw7lplOOVe9QAD1T3XRVz6+a1Z4z5Q= +github.com/stmcginnis/gofish v0.21.3/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -328,8 +315,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= @@ -337,8 +324,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -370,16 +357,13 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= diff --git a/internal/api/metricstore.go b/internal/api/metricstore.go index 5c15bb2c..ff4deb6a 100644 --- a/internal/api/metricstore.go +++ b/internal/api/metricstore.go @@ -18,7 +18,7 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/metricstore" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" - "github.com/influxdata/line-protocol/v2/lineprotocol" + "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol" ) // handleFree godoc diff --git a/internal/api/nats.go b/internal/api/nats.go index 02a03fae..efa4ab6f 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -21,7 +21,7 @@ import ( "github.com/ClusterCockpit/cc-lib/v2/nats" "github.com/ClusterCockpit/cc-lib/v2/receivers" "github.com/ClusterCockpit/cc-lib/v2/schema" - influx "github.com/influxdata/line-protocol/v2/lineprotocol" + influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol" ) // NatsAPI provides NATS subscription-based handlers for Job and Node operations. diff --git a/pkg/metricstore/lineprotocol.go b/pkg/metricstore/lineprotocol.go index bfbbef2d..ed30dec7 100644 --- a/pkg/metricstore/lineprotocol.go +++ b/pkg/metricstore/lineprotocol.go @@ -14,7 +14,7 @@ import ( cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/nats" "github.com/ClusterCockpit/cc-lib/v2/schema" - "github.com/influxdata/line-protocol/v2/lineprotocol" + "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol" ) func ReceiveNats(ms *MemoryStore, From a1db8263d72b9727347ea69e0cc832ec67bd1235 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 27 Feb 2026 12:30:27 +0100 Subject: [PATCH 301/341] Document line protocol. Optimize REST writeMetric path --- internal/api/metricstore.go | 30 ++++-- pkg/metricstore/lineprotocol.go | 163 ++++++++++++++++++++++++++------ 2 files changed, 155 insertions(+), 38 deletions(-) diff --git a/internal/api/metricstore.go b/internal/api/metricstore.go index ff4deb6a..325b26ba 100644 --- a/internal/api/metricstore.go +++ b/internal/api/metricstore.go @@ -10,7 +10,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "strconv" "strings" @@ -90,16 +89,17 @@ func freeMetrics(rw http.ResponseWriter, r *http.Request) { // @security ApiKeyAuth // @router /write/ [post] func writeMetrics(rw http.ResponseWriter, r *http.Request) { - bytes, err := io.ReadAll(r.Body) rw.Header().Add("Content-Type", "application/json") - if err != nil { - handleError(err, http.StatusInternalServerError, rw) - return - } + // Extract the "cluster" query parameter without allocating a url.Values map. + cluster := queryParam(r.URL.RawQuery, "cluster") + + // Stream directly from the request body instead of copying it into a + // temporary buffer via io.ReadAll. The line-protocol decoder supports + // io.Reader natively, so this avoids the largest heap allocation. ms := metricstore.GetMemoryStore() - dec := lineprotocol.NewDecoderWithBytes(bytes) - if err := metricstore.DecodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { + dec := lineprotocol.NewDecoder(r.Body) + if err := metricstore.DecodeLine(dec, ms, cluster); err != nil { cclog.Errorf("/api/write error: %s", err.Error()) handleError(err, http.StatusBadRequest, rw) return @@ -107,6 +107,20 @@ func writeMetrics(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusOK) } +// queryParam extracts a single query-parameter value from a raw query string +// without allocating a url.Values map. Returns "" if the key is not present. +func queryParam(raw, key string) string { + for raw != "" { + var kv string + kv, raw, _ = strings.Cut(raw, "&") + k, v, _ := strings.Cut(kv, "=") + if k == key { + return v + } + } + return "" +} + // handleDebug godoc // @summary Debug endpoint // @tags debug diff --git a/pkg/metricstore/lineprotocol.go b/pkg/metricstore/lineprotocol.go index f8c83e31..ecae3df1 100644 --- a/pkg/metricstore/lineprotocol.go +++ b/pkg/metricstore/lineprotocol.go @@ -3,9 +3,23 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// This file implements ingestion of InfluxDB line-protocol metric data received +// over NATS. Each line encodes one metric sample with the following structure: +// +// [,cluster=][,hostname=][,type=][,type-id=][,subtype=][,stype-id=] value= [] +// +// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide +// routing information (cluster, host) and optional sub-device selectors (type, +// subtype). Only one field is expected per line: "value". +// +// After decoding, each sample is: +// 1. Written to the in-memory store via ms.WriteToLevel. +// 2. If the checkpoint format is "wal", also forwarded to the WAL staging +// goroutine via the WALMessages channel for durable write-ahead logging. package metricstore import ( + "bytes" "context" "fmt" "sync" @@ -17,6 +31,16 @@ import ( "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol" ) +// ReceiveNats subscribes to all configured NATS subjects and feeds incoming +// line-protocol messages into the MemoryStore. +// +// When workers > 1 a pool of goroutines drains a shared channel so that +// multiple messages can be decoded in parallel. With workers == 1 the NATS +// callback decodes inline (no channel overhead, lower latency). +// +// The function blocks until ctx is cancelled and all worker goroutines have +// finished. It returns nil when the NATS client is not configured; callers +// should treat that as a no-op rather than an error. func ReceiveNats(ms *MemoryStore, workers int, ctx context.Context, @@ -75,8 +99,13 @@ func ReceiveNats(ms *MemoryStore, return nil } -// Place `prefix` in front of `buf` but if possible, -// do that inplace in `buf`. +// reorder prepends prefix to buf in-place when buf has enough spare capacity, +// avoiding an allocation. Falls back to a regular append otherwise. +// +// It is used to assemble the "type" and "subtype" selector +// strings when the type tag arrives before the type-id tag in the line, so the +// two byte slices need to be concatenated in tag-declaration order regardless +// of wire order. func reorder(buf, prefix []byte) []byte { n := len(prefix) m := len(buf) @@ -94,17 +123,83 @@ func reorder(buf, prefix []byte) []byte { } } -// Decode lines using dec and make write calls to the MemoryStore. -// If a line is missing its cluster tag, use clusterDefault as default. +// decodeState holds the per-call scratch buffers used by DecodeLine. +// Instances are recycled via decodeStatePool to avoid repeated allocations +// during high-throughput ingestion. +type decodeState struct { + // metricBuf holds a copy of the current measurement name (line-protocol + // measurement field). Copied because dec.Measurement() returns a slice + // that is invalidated by the next decoder call. + metricBuf []byte + + // selector is the sub-device path passed to WriteToLevel and WALMessage + // (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines. + selector []string + + // typeBuf accumulates the concatenated "type"+"type-id" tag value for the + // current line. Reset at the start of each line's tag-decode loop. + typeBuf []byte + + // subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value. + // Reset at the start of each line's tag-decode loop. + subTypeBuf []byte + + // prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its + // string conversion. Because consecutive lines in a batch typically address + // the same sub-device, the cache hit rate is very high and avoids + // repeated []byte→string allocations. + prevTypeBytes []byte + prevTypeStr string + + // prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype. + prevSubTypeBytes []byte + prevSubTypeStr string +} + +// decodeStatePool recycles decodeState values across DecodeLine calls to +// reduce GC pressure during sustained metric ingestion. +var decodeStatePool = sync.Pool{ + New: func() any { + return &decodeState{ + metricBuf: make([]byte, 0, 16), + selector: make([]string, 0, 4), + typeBuf: make([]byte, 0, 16), + subTypeBuf: make([]byte, 0, 16), + } + }, +} + +// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each +// decoded metric sample into ms. +// +// clusterDefault is used as the cluster name for lines that do not carry a +// "cluster" tag. Callers typically supply the ClusterTag value from the NATS +// subscription configuration. +// +// Performance notes: +// - A decodeState is obtained from decodeStatePool to reuse scratch buffers. +// - The Level pointer (host-level node in the metric tree) is cached across +// consecutive lines that share the same cluster+host pair to avoid +// repeated lock acquisitions on the root and cluster levels. +// - []byte→string conversions for type/subtype selectors are cached via +// prevType*/prevSubType* fields because batches typically repeat the same +// sub-device identifiers. +// - Timestamp parsing tries Second precision first; if that fails it retries +// Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp +// falls back to time.Now(). +// +// When the checkpoint format is "wal" each successfully decoded sample is also +// sent to WALMessages so the WAL staging goroutine can persist it durably +// before the next binary snapshot. func DecodeLine(dec *lineprotocol.Decoder, ms *MemoryStore, clusterDefault string, ) error { // Reduce allocations in loop: t := time.Now() - metric, metricBuf := Metric{}, make([]byte, 0, 16) - selector := make([]string, 0, 4) - typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0) + metric := Metric{} + st := decodeStatePool.Get().(*decodeState) + defer decodeStatePool.Put(st) // Optimize for the case where all lines in a "batch" are about the same // cluster and host. By using `WriteToLevel` (level = host), we do not need @@ -121,7 +216,7 @@ func DecodeLine(dec *lineprotocol.Decoder, // Needs to be copied because another call to dec.* would // invalidate the returned slice. - metricBuf = append(metricBuf[:0], rawmeasurement...) + st.metricBuf = append(st.metricBuf[:0], rawmeasurement...) // The go compiler optimizes map[string(byteslice)] lookups: metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)] @@ -129,7 +224,7 @@ func DecodeLine(dec *lineprotocol.Decoder, continue } - typeBuf, subTypeBuf := typeBuf[:0], subTypeBuf[:0] + st.typeBuf, st.subTypeBuf = st.typeBuf[:0], st.subTypeBuf[:0] cluster, host := clusterDefault, "" for { key, val, err := dec.NextTag() @@ -162,41 +257,49 @@ func DecodeLine(dec *lineprotocol.Decoder, } // We cannot be sure that the "type" tag comes before the "type-id" tag: - if len(typeBuf) == 0 { - typeBuf = append(typeBuf, val...) + if len(st.typeBuf) == 0 { + st.typeBuf = append(st.typeBuf, val...) } else { - typeBuf = reorder(typeBuf, val) + st.typeBuf = reorder(st.typeBuf, val) } case "type-id": - typeBuf = append(typeBuf, val...) + st.typeBuf = append(st.typeBuf, val...) case "subtype": // We cannot be sure that the "subtype" tag comes before the "stype-id" tag: - if len(subTypeBuf) == 0 { - subTypeBuf = append(subTypeBuf, val...) + if len(st.subTypeBuf) == 0 { + st.subTypeBuf = append(st.subTypeBuf, val...) } else { - subTypeBuf = reorder(subTypeBuf, val) - // subTypeBuf = reorder(typeBuf, val) + st.subTypeBuf = reorder(st.subTypeBuf, val) } case "stype-id": - subTypeBuf = append(subTypeBuf, val...) + st.subTypeBuf = append(st.subTypeBuf, val...) default: } } // If the cluster or host changed, the lvl was set to nil if lvl == nil { - selector = selector[:2] - selector[0], selector[1] = cluster, host - lvl = ms.GetLevel(selector) + st.selector = st.selector[:2] + st.selector[0], st.selector[1] = cluster, host + lvl = ms.GetLevel(st.selector) prevCluster, prevHost = cluster, host } - // subtypes: - selector = selector[:0] - if len(typeBuf) > 0 { - selector = append(selector, string(typeBuf)) // <- Allocation :( - if len(subTypeBuf) > 0 { - selector = append(selector, string(subTypeBuf)) + // subtypes: cache []byte→string conversions; messages in a batch typically + // share the same type/subtype so the hit rate is very high. + st.selector = st.selector[:0] + if len(st.typeBuf) > 0 { + if !bytes.Equal(st.typeBuf, st.prevTypeBytes) { + st.prevTypeBytes = append(st.prevTypeBytes[:0], st.typeBuf...) + st.prevTypeStr = string(st.typeBuf) + } + st.selector = append(st.selector, st.prevTypeStr) + if len(st.subTypeBuf) > 0 { + if !bytes.Equal(st.subTypeBuf, st.prevSubTypeBytes) { + st.prevSubTypeBytes = append(st.prevSubTypeBytes[:0], st.subTypeBuf...) + st.prevSubTypeStr = string(st.subTypeBuf) + } + st.selector = append(st.selector, st.prevSubTypeStr) } } @@ -246,16 +349,16 @@ func DecodeLine(dec *lineprotocol.Decoder, if Keys.Checkpoints.FileFormat == "wal" { WALMessages <- &WALMessage{ - MetricName: string(metricBuf), + MetricName: string(st.metricBuf), Cluster: cluster, Node: host, - Selector: append([]string{}, selector...), + Selector: append([]string{}, st.selector...), Value: metric.Value, Timestamp: time, } } - if err := ms.WriteToLevel(lvl, selector, time, []Metric{metric}); err != nil { + if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil { return err } } From a418abc7d5ccfc806318caa9effa99d9f955fbcb Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 27 Feb 2026 14:40:26 +0100 Subject: [PATCH 302/341] Run go fix --- internal/api/rest.go | 2 +- pkg/archive/fsBackend.go | 6 ++---- pkg/archive/s3Backend.go | 6 ++---- pkg/archive/sqliteBackend.go | 6 ++---- pkg/metricstore/archive.go | 6 ++---- pkg/metricstore/checkpoint.go | 6 ++---- pkg/metricstore/metricstore.go | 12 ++++-------- pkg/metricstore/walCheckpoint.go | 14 +++++++------- 8 files changed, 22 insertions(+), 36 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 4d2385e3..613867a8 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -302,7 +302,7 @@ func (api *RestAPI) runTagger(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "text/plain") rw.WriteHeader(http.StatusOK) - if _, err := rw.Write([]byte(fmt.Sprintf("Tagger %s started", name))); err != nil { + if _, err := rw.Write(fmt.Appendf(nil, "Tagger %s started", name)); err != nil { cclog.Errorf("Failed to write response: %v", err) } } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 07b86e2b..dfc870b4 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -501,9 +501,7 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { var wg sync.WaitGroup for range numWorkers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for jobPath := range jobPaths { job, err := loadJobMeta(filepath.Join(jobPath, "meta.json")) if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { @@ -529,7 +527,7 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { ch <- JobContainer{Meta: job, Data: nil} } } - }() + }) } clustersDir, err := os.ReadDir(fsa.path) diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index 84abd713..7b82d309 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -821,9 +821,7 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { var wg sync.WaitGroup for range numWorkers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for metaKey := range metaKeys { result, err := s3a.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s3a.bucket), @@ -859,7 +857,7 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { ch <- JobContainer{Meta: job, Data: nil} } } - }() + }) } for _, cluster := range s3a.clusters { diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 50821367..3f214136 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -576,9 +576,7 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { var wg sync.WaitGroup for range numWorkers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for row := range jobRows { job, err := DecodeJobMeta(bytes.NewReader(row.metaBlob)) if err != nil { @@ -617,7 +615,7 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { ch <- JobContainer{Meta: job, Data: nil} } } - }() + }) } for { diff --git a/pkg/metricstore/archive.go b/pkg/metricstore/archive.go index 784348b5..d3617f2c 100644 --- a/pkg/metricstore/archive.go +++ b/pkg/metricstore/archive.go @@ -49,9 +49,7 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) { // runWorker takes simple values to configure what it does func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { d, err := time.ParseDuration(interval) if err != nil { @@ -85,7 +83,7 @@ func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mod } } } - }() + }) } var ErrNoNewArchiveData error = errors.New("all data already archived") diff --git a/pkg/metricstore/checkpoint.go b/pkg/metricstore/checkpoint.go index 590197e3..45b2bc2a 100644 --- a/pkg/metricstore/checkpoint.go +++ b/pkg/metricstore/checkpoint.go @@ -96,9 +96,7 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { ms := GetMemoryStore() - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { d, err := time.ParseDuration(Keys.Checkpoints.Interval) if err != nil { @@ -149,7 +147,7 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { } } } - }() + }) } // MarshalJSON provides optimized JSON encoding for CheckpointMetrics. diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index 3fe64d55..d46c0d15 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -320,9 +320,7 @@ func Shutdown() { func Retention(wg *sync.WaitGroup, ctx context.Context) { ms := GetMemoryStore() - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { d, err := time.ParseDuration(Keys.RetentionInMemory) if err != nil { cclog.Fatal(err) @@ -361,7 +359,7 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { state.mu.Unlock() } } - }() + }) } // MemoryUsageTracker starts a background goroutine that monitors memory usage. @@ -382,9 +380,7 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) { ms := GetMemoryStore() - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { d := DefaultMemoryUsageTrackerInterval if d <= 0 { @@ -470,7 +466,7 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) { } } } - }() + }) } // Free removes metric data older than the given time while preserving data for active nodes. diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go index e8a71ce2..685a8388 100644 --- a/pkg/metricstore/walCheckpoint.go +++ b/pkg/metricstore/walCheckpoint.go @@ -65,6 +65,7 @@ import ( "math" "os" "path" + "strings" "sync" "sync/atomic" @@ -114,9 +115,7 @@ type walFileState struct { // and appends binary WAL records to per-host current.wal files. // Also handles WAL rotation requests from the checkpoint goroutine. func WALStaging(wg *sync.WaitGroup, ctx context.Context) { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { if Keys.Checkpoints.FileFormat == "json" { return @@ -220,7 +219,7 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) { processRotate(req) } } - }() + }) } // RotateWALFiles sends rotation requests for the given host directories @@ -478,11 +477,12 @@ func joinSelector(sel []string) string { if len(sel) == 0 { return "" } - result := sel[0] + var result strings.Builder + result.WriteString(sel[0]) for i := 1; i < len(sel); i++ { - result += "\x00" + sel[i] + result.WriteString("\x00" + sel[i]) } - return result + return result.String() } // ToCheckpointWAL writes binary snapshot files for all hosts in parallel. From 07b989cb81538bdce1e6dce50222c3cc7d76ab58 Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Fri, 27 Feb 2026 14:44:32 +0100 Subject: [PATCH 303/341] Add new bufferPool implementation --- pkg/metricstore/buffer.go | 101 ++++++++++++++++++++++++++-- pkg/metricstore/level.go | 2 + pkg/metricstore/metricstore.go | 6 ++ pkg/metricstore/metricstore_test.go | 50 ++++++++++++++ 4 files changed, 155 insertions(+), 4 deletions(-) diff --git a/pkg/metricstore/buffer.go b/pkg/metricstore/buffer.go index 665d8012..f486e645 100644 --- a/pkg/metricstore/buffer.go +++ b/pkg/metricstore/buffer.go @@ -43,6 +43,7 @@ package metricstore import ( "errors" "sync" + "time" "github.com/ClusterCockpit/cc-lib/v2/schema" ) @@ -53,12 +54,102 @@ import ( // of data or reallocation needs to happen on writes. const BufferCap int = DefaultBufferCapacity -var bufferPool sync.Pool = sync.Pool{ - New: func() any { +// BufferPool is the global instance. +// It is initialized immediately when the package loads. +var bufferPool = NewPersistentBufferPool() + +type PersistentBufferPool struct { + pool []*buffer + mu sync.Mutex +} + +// NewPersistentBufferPool creates a dynamic pool for buffers. +func NewPersistentBufferPool() *PersistentBufferPool { + return &PersistentBufferPool{ + pool: make([]*buffer, 0), + } +} + +func (p *PersistentBufferPool) Get() *buffer { + p.mu.Lock() + defer p.mu.Unlock() + + n := len(p.pool) + if n == 0 { + // Pool is empty, allocate a new one return &buffer{ data: make([]schema.Float, 0, BufferCap), } - }, + } + + // Reuse existing buffer from the pool + b := p.pool[n-1] + p.pool[n-1] = nil // Avoid memory leak + p.pool = p.pool[:n-1] + return b +} + +func (p *PersistentBufferPool) Put(b *buffer) { + // Reset the buffer before putting it back + b.data = b.data[:0] + + p.mu.Lock() + defer p.mu.Unlock() + p.pool = append(p.pool, b) +} + +// GetSize returns the exact number of buffers currently sitting in the pool. +func (p *PersistentBufferPool) GetSize() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.pool) +} + +// Clear drains all buffers currently in the pool, allowing the GC to collect them. +func (p *PersistentBufferPool) Clear() { + p.mu.Lock() + defer p.mu.Unlock() + for i := range p.pool { + p.pool[i] = nil + } + p.pool = p.pool[:0] +} + +// Clean removes buffers from the pool that haven't been used in the given duration. +// It uses a simple LRU approach based on the lastUsed timestamp. +func (p *PersistentBufferPool) Clean(threshold int64) { + p.mu.Lock() + defer p.mu.Unlock() + + // Filter in place + active := p.pool[:0] + for _, b := range p.pool { + if b.lastUsed >= threshold { + active = append(active, b) + } else { + // Buffer is older than the threshold, let it be collected by GC + } + } + + // Nullify the rest to prevent memory leaks + for i := len(active); i < len(p.pool); i++ { + p.pool[i] = nil + } + + p.pool = active +} + +// CleanAll removes all buffers from the pool. +func (p *PersistentBufferPool) CleanAll() { + p.mu.Lock() + defer p.mu.Unlock() + + // Nullify all buffers to prevent memory leaks + for i := range p.pool { + p.pool[i] = nil + } + + p.pool = p.pool[:0] } var ( @@ -94,10 +185,11 @@ type buffer struct { start int64 archived bool closed bool + lastUsed int64 } func newBuffer(ts, freq int64) *buffer { - b := bufferPool.Get().(*buffer) + b := bufferPool.Get() b.frequency = freq b.start = ts - (freq / 2) b.prev = nil @@ -240,6 +332,7 @@ func (b *buffer) free(t int64) (delme bool, n int) { if cap(b.prev.data) != BufferCap { b.prev.data = make([]schema.Float, 0, BufferCap) } + b.prev.lastUsed = time.Now().Unix() bufferPool.Put(b.prev) b.prev = nil } diff --git a/pkg/metricstore/level.go b/pkg/metricstore/level.go index 85c2ba7b..ef082579 100644 --- a/pkg/metricstore/level.go +++ b/pkg/metricstore/level.go @@ -42,6 +42,7 @@ package metricstore import ( "sync" + "time" "unsafe" "github.com/ClusterCockpit/cc-lib/v2/schema" @@ -192,6 +193,7 @@ func (l *Level) free(t int64) (int, error) { if cap(b.data) != BufferCap { b.data = make([]schema.Float, 0, BufferCap) } + b.lastUsed = time.Now().Unix() bufferPool.Put(b) l.metrics[i] = nil } diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index d46c0d15..db3e4357 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -357,6 +357,9 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { } state.mu.Unlock() + + // Clean up the buffer pool + bufferPool.Clean(state.lastRetentionTime) } } }) @@ -425,6 +428,9 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) { runtime.ReadMemStats(&mem) actualMemoryGB = float64(mem.Alloc) / 1e9 + bufferPool.CleanAll() + cclog.Infof("[METRICSTORE]> Cleaned up bufferPool\n") + if actualMemoryGB > float64(Keys.MemoryCap) { cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap) diff --git a/pkg/metricstore/metricstore_test.go b/pkg/metricstore/metricstore_test.go index eb1aff15..55c97e60 100644 --- a/pkg/metricstore/metricstore_test.go +++ b/pkg/metricstore/metricstore_test.go @@ -464,3 +464,53 @@ func TestBufferHealthChecks(t *testing.T) { }) } } + +func TestBufferPoolClean(t *testing.T) { + // Use a fresh pool for testing + pool := NewPersistentBufferPool() + + now := time.Now().Unix() + + // Create some buffers and put them in the pool with different lastUsed times + b1 := &buffer{lastUsed: now - 3600, data: make([]schema.Float, 0)} // 1 hour ago + b2 := &buffer{lastUsed: now - 7200, data: make([]schema.Float, 0)} // 2 hours ago + b3 := &buffer{lastUsed: now - 180000, data: make([]schema.Float, 0)} // 50 hours ago + b4 := &buffer{lastUsed: now - 200000, data: make([]schema.Float, 0)} // 55 hours ago + b5 := &buffer{lastUsed: now, data: make([]schema.Float, 0)} + + pool.Put(b1) + pool.Put(b2) + pool.Put(b3) + pool.Put(b4) + pool.Put(b5) + + if pool.GetSize() != 5 { + t.Fatalf("Expected pool size 5, got %d", pool.GetSize()) + } + + // Clean buffers older than 48 hours + timeUpdate := time.Now().Add(48 * time.Hour).Unix() + pool.Clean(timeUpdate) + + // Expected: b1, b2, b5 should remain. b3, b4 should be cleaned. + if pool.GetSize() != 3 { + t.Fatalf("Expected pool size 3 after clean, got %d", pool.GetSize()) + } + + validBufs := map[int64]bool{ + b1.lastUsed: true, + b2.lastUsed: true, + b5.lastUsed: true, + } + + for i := 0; i < 3; i++ { + b := pool.Get() + if !validBufs[b.lastUsed] { + t.Errorf("Found unexpected buffer with lastUsed %d", b.lastUsed) + } + } + + if pool.GetSize() != 0 { + t.Fatalf("Expected pool to be empty, got %d", pool.GetSize()) + } +} From 2e5d85c2231342dcac0034b93933f0bd969b1b13 Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Fri, 27 Feb 2026 15:09:06 +0100 Subject: [PATCH 304/341] Udpate testcase --- pkg/metricstore/metricstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metricstore/metricstore_test.go b/pkg/metricstore/metricstore_test.go index 55c97e60..772fd7ea 100644 --- a/pkg/metricstore/metricstore_test.go +++ b/pkg/metricstore/metricstore_test.go @@ -489,7 +489,7 @@ func TestBufferPoolClean(t *testing.T) { } // Clean buffers older than 48 hours - timeUpdate := time.Now().Add(48 * time.Hour).Unix() + timeUpdate := time.Now().Add(-48 * time.Hour).Unix() pool.Clean(timeUpdate) // Expected: b1, b2, b5 should remain. b3, b4 should be cleaned. From d00aa2666dd2a5da67291638105a74012cc0062e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 27 Feb 2026 15:20:09 +0100 Subject: [PATCH 305/341] activate update of roles and projects if updateUserOnLogin is set --- internal/repository/user.go | 70 ++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/internal/repository/user.go b/internal/repository/user.go index 966646dd..38a4980b 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" @@ -187,8 +188,8 @@ func (r *UserRepository) AddUser(user *schema.User) error { } func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) error { - // user contains updated info, apply to dbuser - // TODO: Discuss updatable fields + // user contains updated info -> Apply to dbUser + // --- Simple Name Update --- if dbUser.Name != user.Name { if _, err := sq.Update("hpc_user").Set("name", user.Name).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { cclog.Errorf("error while updating name of user '%s'", user.Username) @@ -196,13 +197,64 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro } } - // Toggled until greenlit - // if dbUser.HasRole(schema.RoleManager) && !reflect.DeepEqual(dbUser.Projects, user.Projects) { - // projects, _ := json.Marshal(user.Projects) - // if _, err := sq.Update("hpc_user").Set("projects", projects).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { - // return err - // } - // } + // --- Def Helpers --- + // Helper to update roles + updateRoles := func(roles []string) error { + rolesJSON, _ := json.Marshal(roles) + _, err := sq.Update("hpc_user").Set("roles", rolesJSON).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec() + return err + } + + // Helper to update projects + updateProjects := func(projects []string) error { + projectsJSON, _ := json.Marshal(projects) + _, err := sq.Update("hpc_user").Set("projects", projectsJSON).Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec() + return err + } + + // Helper to clear projects + clearProjects := func() error { + _, err := sq.Update("hpc_user").Set("projects", "[]").Where("hpc_user.username = ?", dbUser.Username).RunWith(r.DB).Exec() + return err + } + + // --- Manager Role Handling --- + if dbUser.HasRole(schema.RoleManager) && user.HasRole(schema.RoleManager) && !reflect.DeepEqual(dbUser.Projects, user.Projects) { + // Existing Manager: update projects + if err := updateProjects(user.Projects); err != nil { + return err + } + } else if dbUser.HasRole(schema.RoleUser) && user.HasRole(schema.RoleManager) && user.HasNotRoles([]schema.Role{schema.RoleAdmin}) { + // New Manager: update roles and projects + if err := updateRoles(user.Roles); err != nil { + return err + } + if err := updateProjects(user.Projects); err != nil { + return err + } + } else if dbUser.HasRole(schema.RoleManager) && user.HasNotRoles([]schema.Role{schema.RoleAdmin, schema.RoleManager}) { + // Remove Manager: update roles and clear projects + if err := updateRoles(user.Roles); err != nil { + return err + } + if err := clearProjects(); err != nil { + return err + } + } + + // --- Support Role Handling --- + if dbUser.HasRole(schema.RoleUser) && dbUser.HasNotRoles([]schema.Role{schema.RoleSupport}) && + user.HasRole(schema.RoleSupport) && user.HasNotRoles([]schema.Role{schema.RoleAdmin}) { + // New Support: update roles + if err := updateRoles(user.Roles); err != nil { + return err + } + } else if dbUser.HasRole(schema.RoleSupport) && user.HasNotRoles([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { + // Remove Support: update roles + if err := updateRoles(user.Roles); err != nil { + return err + } + } return nil } From adebffd2515541da99098dea0bf03fd5ad789935 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 27 Feb 2026 17:40:32 +0100 Subject: [PATCH 306/341] Replace the old zip archive options for the metricstore node data by parquet files --- pkg/metricstore/archive.go | 221 +++++++++++++-------- pkg/metricstore/parquetArchive.go | 213 +++++++++++++++++++++ pkg/metricstore/parquetArchive_test.go | 255 +++++++++++++++++++++++++ 3 files changed, 606 insertions(+), 83 deletions(-) create mode 100644 pkg/metricstore/parquetArchive.go create mode 100644 pkg/metricstore/parquetArchive_test.go diff --git a/pkg/metricstore/archive.go b/pkg/metricstore/archive.go index d3617f2c..77f4264a 100644 --- a/pkg/metricstore/archive.go +++ b/pkg/metricstore/archive.go @@ -6,12 +6,9 @@ package metricstore import ( - "archive/zip" - "bufio" "context" "errors" "fmt" - "io" "os" "path/filepath" "sync" @@ -47,7 +44,7 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) { } } -// runWorker takes simple values to configure what it does +// cleanUpWorker takes simple values to configure what it does func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) { wg.Go(func() { @@ -75,10 +72,10 @@ func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mod if err != nil { cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error()) } else { - if delete && cleanupDir == "" { + if delete { cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n) } else { - cclog.Infof("[METRICSTORE]> done: %d files zipped and moved to archive", n) + cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n) } } } @@ -88,17 +85,26 @@ func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mod var ErrNoNewArchiveData error = errors.New("all data already archived") -// Delete or ZIP all checkpoint files older than `from` together and write them to the `cleanupDir`, -// deleting/moving them from the `checkpointsDir`. +// CleanupCheckpoints deletes or archives all checkpoint files older than `from`. +// When archiving, consolidates all hosts per cluster into a single Parquet file. func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) { + if deleteInstead { + return deleteCheckpoints(checkpointsDir, from) + } + + return archiveCheckpoints(checkpointsDir, cleanupDir, from) +} + +// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts. +func deleteCheckpoints(checkpointsDir string, from int64) (int, error) { entries1, err := os.ReadDir(checkpointsDir) if err != nil { return 0, err } type workItem struct { - cdir, adir string - cluster, host string + dir string + cluster, host string } var wg sync.WaitGroup @@ -109,13 +115,29 @@ func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteIns for worker := 0; worker < Keys.NumWorkers; worker++ { go func() { defer wg.Done() - for workItem := range work { - m, err := cleanupCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead) + for item := range work { + entries, err := os.ReadDir(item.dir) if err != nil { - cclog.Errorf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error()) + cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error()) atomic.AddInt32(&errs, 1) + continue + } + + files, err := findFiles(entries, from, false) + if err != nil { + cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error()) + atomic.AddInt32(&errs, 1) + continue + } + + for _, checkpoint := range files { + if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil { + cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error()) + atomic.AddInt32(&errs, 1) + } else { + atomic.AddInt32(&n, 1) + } } - atomic.AddInt32(&n, int32(m)) } }() } @@ -124,14 +146,14 @@ func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteIns entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name())) if e != nil { err = e + continue } for _, de2 := range entries2 { - cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name()) - adir := filepath.Join(cleanupDir, de1.Name(), de2.Name()) work <- workItem{ - adir: adir, cdir: cdir, - cluster: de1.Name(), host: de2.Name(), + dir: filepath.Join(checkpointsDir, de1.Name(), de2.Name()), + cluster: de1.Name(), + host: de2.Name(), } } } @@ -142,85 +164,118 @@ func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteIns if err != nil { return int(n), err } - if errs > 0 { - return int(n), fmt.Errorf("%d errors happened while archiving (%d successes)", errs, n) + return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n) } return int(n), nil } -// Helper function for `CleanupCheckpoints`. -func cleanupCheckpoints(dir string, cleanupDir string, from int64, deleteInstead bool) (int, error) { - entries, err := os.ReadDir(dir) +// archiveCheckpoints archives checkpoint files to Parquet format. +// Produces one Parquet file per cluster: //.parquet +func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) { + clusterEntries, err := os.ReadDir(checkpointsDir) if err != nil { return 0, err } - files, err := findFiles(entries, from, false) - if err != nil { - return 0, err - } + totalFiles := 0 - if deleteInstead { - n := 0 - for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - if err = os.Remove(filename); err != nil { - return n, err - } - n += 1 + for _, clusterEntry := range clusterEntries { + if !clusterEntry.IsDir() { + continue } - return n, nil - } - filename := filepath.Join(cleanupDir, fmt.Sprintf("%d.zip", from)) - f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, CheckpointFilePerms) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(cleanupDir, CheckpointDirPerms) - if err == nil { - f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, CheckpointFilePerms) - } - } - if err != nil { - return 0, err - } - defer f.Close() - bw := bufio.NewWriter(f) - defer bw.Flush() - zw := zip.NewWriter(bw) - defer zw.Close() - - n := 0 - for _, checkpoint := range files { - // Use closure to ensure file is closed immediately after use, - // avoiding file descriptor leak from defer in loop - err := func() error { - filename := filepath.Join(dir, checkpoint) - r, err := os.Open(filename) - if err != nil { - return err - } - defer r.Close() - - w, err := zw.Create(checkpoint) - if err != nil { - return err - } - - if _, err = io.Copy(w, r); err != nil { - return err - } - - if err = os.Remove(filename); err != nil { - return err - } - return nil - }() + cluster := clusterEntry.Name() + hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster)) if err != nil { - return n, err + return totalFiles, err } - n += 1 + + // Collect rows from all hosts in this cluster using worker pool + type hostResult struct { + rows []ParquetMetricRow + files []string // checkpoint filenames to delete after successful write + dir string // checkpoint directory for this host + } + + results := make(chan hostResult, len(hostEntries)) + work := make(chan struct { + dir, host string + }, Keys.NumWorkers) + + var wg sync.WaitGroup + errs := int32(0) + + wg.Add(Keys.NumWorkers) + for w := 0; w < Keys.NumWorkers; w++ { + go func() { + defer wg.Done() + for item := range work { + rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from) + if err != nil { + cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error()) + atomic.AddInt32(&errs, 1) + continue + } + if len(rows) > 0 { + results <- hostResult{rows: rows, files: files, dir: item.dir} + } + } + }() + } + + go func() { + for _, hostEntry := range hostEntries { + if !hostEntry.IsDir() { + continue + } + dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name()) + work <- struct { + dir, host string + }{dir: dir, host: hostEntry.Name()} + } + close(work) + wg.Wait() + close(results) + }() + + // Collect all rows and file info + var allRows []ParquetMetricRow + var allResults []hostResult + for r := range results { + allRows = append(allRows, r.rows...) + allResults = append(allResults, r) + } + + if errs > 0 { + return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster) + } + + if len(allRows) == 0 { + continue + } + + // Write one Parquet file per cluster + parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from)) + if err := writeParquetArchive(parquetFile, allRows); err != nil { + return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, err) + } + + // Delete archived checkpoint files + for _, result := range allResults { + for _, file := range result.files { + filename := filepath.Join(result.dir, file) + if err := os.Remove(filename); err != nil { + cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err) + } else { + totalFiles++ + } + } + } + + cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s", + len(allRows), totalFiles, cluster, parquetFile) } - return n, nil + return totalFiles, nil } diff --git a/pkg/metricstore/parquetArchive.go b/pkg/metricstore/parquetArchive.go new file mode 100644 index 00000000..420ee4e5 --- /dev/null +++ b/pkg/metricstore/parquetArchive.go @@ -0,0 +1,213 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package metricstore + +import ( + "bufio" + "encoding/binary" + "encoding/json" + "fmt" + "os" + "path/filepath" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + pq "github.com/parquet-go/parquet-go" +) + +// ParquetMetricRow is the long-format schema for archived metric data. +// One row per (host, metric, scope, scope_id, timestamp) data point. +// Sorted by (cluster, hostname, metric, timestamp) for optimal compression. +type ParquetMetricRow struct { + Cluster string `parquet:"cluster"` + Hostname string `parquet:"hostname"` + Metric string `parquet:"metric"` + Scope string `parquet:"scope"` + ScopeID string `parquet:"scope_id"` + Timestamp int64 `parquet:"timestamp"` + Frequency int64 `parquet:"frequency"` + Value float32 `parquet:"value"` +} + +// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows. +// The scope path is built from the hierarchy: host level is "node", then child names +// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0"). +func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow { + for metricName, cm := range cf.Metrics { + ts := cm.Start + for _, v := range cm.Data { + if !v.IsNaN() { + rows = append(rows, ParquetMetricRow{ + Cluster: cluster, + Hostname: hostname, + Metric: metricName, + Scope: scope, + ScopeID: scopeID, + Timestamp: ts, + Frequency: cm.Frequency, + Value: float32(v), + }) + } + ts += cm.Frequency + } + } + + for childName, childCf := range cf.Children { + childScope, childScopeID := parseScopeFromName(childName) + rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows) + } + + return rows +} + +// parseScopeFromName infers scope and scope_id from a child level name. +// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"), +// "a0" (accelerator) → ("accelerator", "0"). +// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id. +func parseScopeFromName(name string) (string, string) { + prefixes := []struct { + prefix string + scope string + }{ + {"socket", "socket"}, + {"memoryDomain", "memoryDomain"}, + {"core", "core"}, + {"hwthread", "hwthread"}, + {"cpu", "hwthread"}, + {"accelerator", "accelerator"}, + } + + for _, p := range prefixes { + if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix { + id := name[len(p.prefix):] + if len(id) > 0 && id[0] >= '0' && id[0] <= '9' { + return p.scope, id + } + } + } + + return name, "" +} + +// writeParquetArchive writes rows to a Parquet file with Zstd compression. +func writeParquetArchive(filename string, rows []ParquetMetricRow) error { + if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil { + return fmt.Errorf("creating archive directory: %w", err) + } + + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms) + if err != nil { + return fmt.Errorf("creating parquet file: %w", err) + } + defer f.Close() + + bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer + + writer := pq.NewGenericWriter[ParquetMetricRow](bw, + pq.Compression(&pq.Zstd), + pq.SortingWriterConfig(pq.SortingColumns( + pq.Ascending("cluster"), + pq.Ascending("hostname"), + pq.Ascending("metric"), + pq.Ascending("timestamp"), + )), + ) + + if _, err := writer.Write(rows); err != nil { + return fmt.Errorf("writing parquet rows: %w", err) + } + + if err := writer.Close(); err != nil { + return fmt.Errorf("closing parquet writer: %w", err) + } + + if err := bw.Flush(); err != nil { + return fmt.Errorf("flushing parquet file: %w", err) + } + + return nil +} + +// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns +// a CheckpointFile. Used by the Parquet archiver to read checkpoint data +// before converting it to Parquet format. +func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + ext := filepath.Ext(filename) + switch ext { + case ".json": + cf := &CheckpointFile{} + br := bufio.NewReader(f) + if err := json.NewDecoder(br).Decode(cf); err != nil { + return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err) + } + return cf, nil + + case ".bin": + br := bufio.NewReader(f) + var magic uint32 + if err := binary.Read(br, binary.LittleEndian, &magic); err != nil { + return nil, fmt.Errorf("reading magic from %s: %w", filename, err) + } + if magic != snapFileMagic { + return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic) + } + var fileFrom, fileTo int64 + if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil { + return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err) + } + if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil { + return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err) + } + cf, err := readBinaryLevel(br) + if err != nil { + return nil, fmt.Errorf("reading binary level from %s: %w", filename, err) + } + cf.From = fileFrom + cf.To = fileTo + return cf, nil + + default: + return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext) + } +} + +// archiveCheckpointsToParquet reads checkpoint files for a host directory, +// converts them to Parquet rows. Returns the rows and filenames that were processed. +func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, nil, err + } + + files, err := findFiles(entries, from, false) + if err != nil { + return nil, nil, err + } + + if len(files) == 0 { + return nil, nil, nil + } + + var rows []ParquetMetricRow + + for _, checkpoint := range files { + filename := filepath.Join(dir, checkpoint) + cf, err := loadCheckpointFileFromDisk(filename) + if err != nil { + cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err) + continue + } + + rows = flattenCheckpointFile(cf, cluster, host, "node", "", rows) + } + + return rows, files, nil +} diff --git a/pkg/metricstore/parquetArchive_test.go b/pkg/metricstore/parquetArchive_test.go new file mode 100644 index 00000000..d3d70c02 --- /dev/null +++ b/pkg/metricstore/parquetArchive_test.go @@ -0,0 +1,255 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package metricstore + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/ClusterCockpit/cc-lib/v2/schema" + pq "github.com/parquet-go/parquet-go" +) + +func TestParseScopeFromName(t *testing.T) { + tests := []struct { + name string + wantScope string + wantID string + }{ + {"socket0", "socket", "0"}, + {"socket12", "socket", "12"}, + {"core0", "core", "0"}, + {"core127", "core", "127"}, + {"cpu0", "hwthread", "0"}, + {"hwthread5", "hwthread", "5"}, + {"memoryDomain0", "memoryDomain", "0"}, + {"accelerator0", "accelerator", "0"}, + {"unknown", "unknown", ""}, + {"socketX", "socketX", ""}, // not numeric suffix + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scope, id := parseScopeFromName(tt.name) + if scope != tt.wantScope || id != tt.wantID { + t.Errorf("parseScopeFromName(%q) = (%q, %q), want (%q, %q)", + tt.name, scope, id, tt.wantScope, tt.wantID) + } + }) + } +} + +func TestFlattenCheckpointFile(t *testing.T) { + cf := &CheckpointFile{ + From: 1000, + To: 1060, + Metrics: map[string]*CheckpointMetrics{ + "cpu_load": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{0.5, 0.7, schema.NaN}, + }, + }, + Children: map[string]*CheckpointFile{ + "socket0": { + Metrics: map[string]*CheckpointMetrics{ + "mem_bw": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{100.0, schema.NaN, 200.0}, + }, + }, + Children: make(map[string]*CheckpointFile), + }, + }, + } + + rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil) + + // cpu_load: 2 non-NaN values at node scope + // mem_bw: 2 non-NaN values at socket0 scope + if len(rows) != 4 { + t.Fatalf("expected 4 rows, got %d", len(rows)) + } + + // Verify a node-scope row + found := false + for _, r := range rows { + if r.Metric == "cpu_load" && r.Timestamp == 1000 { + found = true + if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 { + t.Errorf("unexpected row: %+v", r) + } + } + } + if !found { + t.Error("expected cpu_load row at timestamp 1000") + } + + // Verify a socket-scope row + found = false + for _, r := range rows { + if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" { + found = true + } + } + if !found { + t.Error("expected mem_bw row with scope=socket, scope_id=0") + } +} + +func TestParquetArchiveRoundtrip(t *testing.T) { + tmpDir := t.TempDir() + + // Create checkpoint files on disk (JSON format) + cpDir := filepath.Join(tmpDir, "checkpoints", "testcluster", "node001") + if err := os.MkdirAll(cpDir, 0o755); err != nil { + t.Fatal(err) + } + + cf := &CheckpointFile{ + From: 1000, + To: 1180, + Metrics: map[string]*CheckpointMetrics{ + "cpu_load": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{0.5, 0.7, 0.9}, + }, + "mem_used": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{45.0, 46.0, 47.0}, + }, + }, + Children: map[string]*CheckpointFile{ + "socket0": { + Metrics: map[string]*CheckpointMetrics{ + "mem_bw": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{100.0, 110.0, 120.0}, + }, + }, + Children: make(map[string]*CheckpointFile), + }, + }, + } + + // Write JSON checkpoint + cpFile := filepath.Join(cpDir, "1000.json") + data, err := json.Marshal(cf) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cpFile, data, 0o644); err != nil { + t.Fatal(err) + } + + // Archive to Parquet + archiveDir := filepath.Join(tmpDir, "archive") + rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || files[0] != "1000.json" { + t.Fatalf("expected 1 file, got %v", files) + } + + parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet") + if err := writeParquetArchive(parquetFile, rows); err != nil { + t.Fatal(err) + } + + // Read back and verify + f, err := os.Open(parquetFile) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + stat, _ := f.Stat() + pf, err := pq.OpenFile(f, stat.Size()) + if err != nil { + t.Fatal(err) + } + + reader := pq.NewGenericReader[ParquetMetricRow](pf) + readRows := make([]ParquetMetricRow, 100) + n, err := reader.Read(readRows) + if err != nil && n == 0 { + t.Fatal(err) + } + readRows = readRows[:n] + reader.Close() + + // We expect: cpu_load(3) + mem_used(3) + mem_bw(3) = 9 rows + if n != 9 { + t.Fatalf("expected 9 rows in parquet file, got %d", n) + } + + // Verify cluster and hostname are set correctly + for _, r := range readRows { + if r.Cluster != "testcluster" { + t.Errorf("expected cluster=testcluster, got %s", r.Cluster) + } + if r.Hostname != "node001" { + t.Errorf("expected hostname=node001, got %s", r.Hostname) + } + } + + // Verify parquet file is smaller than JSON (compression working) + if stat.Size() == 0 { + t.Error("parquet file is empty") + } + + t.Logf("Parquet file size: %d bytes for %d rows", stat.Size(), n) +} + +func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) { + tmpDir := t.TempDir() + + cf := &CheckpointFile{ + From: 1000, + To: 1060, + Metrics: map[string]*CheckpointMetrics{ + "test_metric": { + Frequency: 60, + Start: 1000, + Data: []schema.Float{1.0, 2.0, 3.0}, + }, + }, + Children: make(map[string]*CheckpointFile), + } + + filename := filepath.Join(tmpDir, "1000.json") + data, err := json.Marshal(cf) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, data, 0o644); err != nil { + t.Fatal(err) + } + + loaded, err := loadCheckpointFileFromDisk(filename) + if err != nil { + t.Fatal(err) + } + + if loaded.From != 1000 || loaded.To != 1060 { + t.Errorf("expected From=1000, To=1060, got From=%d, To=%d", loaded.From, loaded.To) + } + + m, ok := loaded.Metrics["test_metric"] + if !ok { + t.Fatal("expected test_metric in loaded checkpoint") + } + if m.Frequency != 60 || m.Start != 1000 || len(m.Data) != 3 { + t.Errorf("unexpected metric data: %+v", m) + } +} From 1ec41d8389e81e7c517960dda251ec6a8a53ad39 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 28 Feb 2026 19:34:33 +0100 Subject: [PATCH 307/341] Review and improve buffer pool implmentation. Add unit tests. --- pkg/metricstore/buffer.go | 75 ++-- pkg/metricstore/level.go | 9 +- pkg/metricstore/metricstore.go | 2 +- pkg/metricstore/metricstore_test.go | 520 ++++++++++++++++++++++++++++ 4 files changed, 566 insertions(+), 40 deletions(-) diff --git a/pkg/metricstore/buffer.go b/pkg/metricstore/buffer.go index f486e645..557a941c 100644 --- a/pkg/metricstore/buffer.go +++ b/pkg/metricstore/buffer.go @@ -54,6 +54,10 @@ import ( // of data or reallocation needs to happen on writes. const BufferCap int = DefaultBufferCapacity +// maxPoolSize caps the number of buffers held in the pool at any time. +// Prevents unbounded memory growth after large retention-cleanup bursts. +const maxPoolSize = 4096 + // BufferPool is the global instance. // It is initialized immediately when the package loads. var bufferPool = NewPersistentBufferPool() @@ -89,12 +93,18 @@ func (p *PersistentBufferPool) Get() *buffer { return b } +// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix() +// before calling Put so that Clean() can evict idle entries correctly. func (p *PersistentBufferPool) Put(b *buffer) { // Reset the buffer before putting it back b.data = b.data[:0] p.mu.Lock() defer p.mu.Unlock() + if len(p.pool) >= maxPoolSize { + // Pool is full; drop the buffer and let GC collect it. + return + } p.pool = append(p.pool, b) } @@ -121,13 +131,11 @@ func (p *PersistentBufferPool) Clean(threshold int64) { p.mu.Lock() defer p.mu.Unlock() - // Filter in place + // Filter in place, retaining only buffers returned to the pool recently enough. active := p.pool[:0] for _, b := range p.pool { if b.lastUsed >= threshold { active = append(active, b) - } else { - // Buffer is older than the threshold, let it be collected by GC } } @@ -139,19 +147,6 @@ func (p *PersistentBufferPool) Clean(threshold int64) { p.pool = active } -// CleanAll removes all buffers from the pool. -func (p *PersistentBufferPool) CleanAll() { - p.mu.Lock() - defer p.mu.Unlock() - - // Nullify all buffers to prevent memory leaks - for i := range p.pool { - p.pool[i] = nil - } - - p.pool = p.pool[:0] -} - var ( // ErrNoData indicates no time-series data exists for the requested metric/level. ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level") @@ -276,11 +271,13 @@ func (b *buffer) firstWrite() int64 { // // Panics if 'data' slice is too small to hold all values in [from, to). func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) { - if from < b.firstWrite() { - if b.prev != nil { - return b.prev.read(from, to, data) + // Walk back to the buffer that covers 'from', adjusting if we hit the oldest. + for from < b.firstWrite() { + if b.prev == nil { + from = b.firstWrite() + break } - from = b.firstWrite() + b = b.prev } i := 0 @@ -292,16 +289,17 @@ func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int6 break } b = b.next - idx = 0 + // Recalculate idx in the new buffer; a gap between buffers may exist. + idx = int((t - b.start) / b.frequency) } if idx >= len(b.data) { if b.next == nil || to <= b.next.start { break } - data[i] += schema.NaN + data[i] += schema.NaN // NaN + anything = NaN; propagates missing data } else if t < b.start { - data[i] += schema.NaN + data[i] += schema.NaN // gap before this buffer's first write } else { data[i] += b.data[idx] } @@ -359,11 +357,12 @@ func (b *buffer) forceFreeOldest() (delme bool, n int) { // If the previous buffer signals it should be deleted: if delPrev { - // Clear links on the dying buffer to prevent leaks b.prev.next = nil - b.prev.data = nil // Release the underlying float slice immediately - - // Remove the link from the current buffer + if cap(b.prev.data) != BufferCap { + b.prev.data = make([]schema.Float, 0, BufferCap) + } + b.prev.lastUsed = time.Now().Unix() + bufferPool.Put(b.prev) b.prev = nil } return false, freed @@ -392,21 +391,27 @@ func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) erro return nil } - if err := b.prev.iterFromTo(from, to, callback); err != nil { - return err + // Collect overlapping buffers walking backwards (newest → oldest). + var matching []*buffer + for cur := b; cur != nil; cur = cur.prev { + if from <= cur.end() && cur.start <= to { + matching = append(matching, cur) + } } - if from <= b.end() && b.start <= to { - return callback(b) + // Invoke callback in chronological order (oldest → newest). + for i := len(matching) - 1; i >= 0; i-- { + if err := callback(matching[i]); err != nil { + return err + } } - return nil } func (b *buffer) count() int64 { - res := int64(len(b.data)) - if b.prev != nil { - res += b.prev.count() + var res int64 + for ; b != nil; b = b.prev { + res += int64(len(b.data)) } return res } diff --git a/pkg/metricstore/level.go b/pkg/metricstore/level.go index ef082579..2b24a2ea 100644 --- a/pkg/metricstore/level.go +++ b/pkg/metricstore/level.go @@ -238,12 +238,13 @@ func (l *Level) forceFree() (int, error) { // If delme is true, it means 'b' itself (the head) was the oldest // and needs to be removed from the slice. if delme { - // Nil out fields to ensure no hanging references - b.next = nil b.prev = nil - b.data = nil - + if cap(b.data) != BufferCap { + b.data = make([]schema.Float, 0, BufferCap) + } + b.lastUsed = time.Now().Unix() + bufferPool.Put(b) l.metrics[i] = nil } } diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index db3e4357..b5b1a528 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -428,7 +428,7 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) { runtime.ReadMemStats(&mem) actualMemoryGB = float64(mem.Alloc) / 1e9 - bufferPool.CleanAll() + bufferPool.Clear() cclog.Infof("[METRICSTORE]> Cleaned up bufferPool\n") if actualMemoryGB > float64(Keys.MemoryCap) { diff --git a/pkg/metricstore/metricstore_test.go b/pkg/metricstore/metricstore_test.go index 772fd7ea..9087df2a 100644 --- a/pkg/metricstore/metricstore_test.go +++ b/pkg/metricstore/metricstore_test.go @@ -12,6 +12,526 @@ import ( "github.com/ClusterCockpit/cc-lib/v2/schema" ) +// ─── Buffer pool ───────────────────────────────────────────────────────────── + +// TestBufferPoolGetReuse verifies that Get() returns pooled buffers before +// allocating new ones, and that an empty pool allocates a fresh BufferCap buffer. +func TestBufferPoolGetReuse(t *testing.T) { + pool := NewPersistentBufferPool() + + original := &buffer{data: make([]schema.Float, 0, BufferCap), lastUsed: time.Now().Unix()} + pool.Put(original) + + reused := pool.Get() + if reused != original { + t.Error("Get() should return the previously pooled buffer") + } + if pool.GetSize() != 0 { + t.Errorf("pool size after Get() = %d, want 0", pool.GetSize()) + } + + // Empty pool must allocate a fresh buffer with the standard capacity. + fresh := pool.Get() + if fresh == nil { + t.Fatal("Get() from empty pool returned nil") + } + if cap(fresh.data) != BufferCap { + t.Errorf("fresh buffer cap = %d, want %d", cap(fresh.data), BufferCap) + } +} + +// TestBufferPoolClear verifies that Clear() drains all entries. +func TestBufferPoolClear(t *testing.T) { + pool := NewPersistentBufferPool() + for i := 0; i < 10; i++ { + pool.Put(&buffer{data: make([]schema.Float, 0), lastUsed: time.Now().Unix()}) + } + pool.Clear() + if pool.GetSize() != 0 { + t.Errorf("pool size after Clear() = %d, want 0", pool.GetSize()) + } +} + +// TestBufferPoolMaxSize verifies that Put() silently drops buffers once the +// pool reaches maxPoolSize, preventing unbounded memory growth. +func TestBufferPoolMaxSize(t *testing.T) { + pool := NewPersistentBufferPool() + for i := 0; i < maxPoolSize; i++ { + pool.Put(&buffer{data: make([]schema.Float, 0, BufferCap), lastUsed: time.Now().Unix()}) + } + if pool.GetSize() != maxPoolSize { + t.Fatalf("pool size = %d, want %d", pool.GetSize(), maxPoolSize) + } + + pool.Put(&buffer{data: make([]schema.Float, 0, BufferCap), lastUsed: time.Now().Unix()}) + if pool.GetSize() != maxPoolSize { + t.Errorf("pool size after overflow Put = %d, want %d (should not grow)", pool.GetSize(), maxPoolSize) + } +} + +// ─── Buffer helpers ─────────────────────────────────────────────────────────── + +// TestBufferEndFirstWrite verifies the end() and firstWrite() calculations. +func TestBufferEndFirstWrite(t *testing.T) { + // start=90, freq=10 → firstWrite = 90+5 = 95 + b := &buffer{data: make([]schema.Float, 4, BufferCap), frequency: 10, start: 90} + if fw := b.firstWrite(); fw != 95 { + t.Errorf("firstWrite() = %d, want 95", fw) + } + // end = firstWrite + len(data)*freq = 95 + 4*10 = 135 + if e := b.end(); e != 135 { + t.Errorf("end() = %d, want 135", e) + } +} + +// ─── Buffer write ───────────────────────────────────────────────────────────── + +// TestBufferWriteNaNFill verifies that skipped timestamps are filled with NaN. +func TestBufferWriteNaNFill(t *testing.T) { + b := newBuffer(100, 10) + b.write(100, schema.Float(1.0)) + // skip 110 and 120 + b.write(130, schema.Float(4.0)) + + if len(b.data) != 4 { + t.Fatalf("len(data) = %d, want 4 (1 value + 2 NaN + 1 value)", len(b.data)) + } + if b.data[0] != schema.Float(1.0) { + t.Errorf("data[0] = %v, want 1.0", b.data[0]) + } + if !b.data[1].IsNaN() { + t.Errorf("data[1] should be NaN (gap), got %v", b.data[1]) + } + if !b.data[2].IsNaN() { + t.Errorf("data[2] should be NaN (gap), got %v", b.data[2]) + } + if b.data[3] != schema.Float(4.0) { + t.Errorf("data[3] = %v, want 4.0", b.data[3]) + } +} + +// TestBufferWriteCapacityOverflow verifies that exceeding capacity creates and +// links a new buffer rather than panicking or silently dropping data. +func TestBufferWriteCapacityOverflow(t *testing.T) { + // Cap=2 so the third write must overflow into a new buffer. + b := &buffer{data: make([]schema.Float, 0, 2), frequency: 10, start: 95} + + nb, _ := b.write(100, schema.Float(1.0)) + nb, _ = nb.write(110, schema.Float(2.0)) + nb, err := nb.write(120, schema.Float(3.0)) + if err != nil { + t.Fatalf("write() error = %v", err) + } + if nb == b { + t.Fatal("write() should have returned a new buffer after overflow") + } + if nb.prev != b { + t.Error("new buffer should link back to old via prev") + } + if b.next != nb { + t.Error("old buffer should link forward to new via next") + } + if len(b.data) != 2 { + t.Errorf("old buffer len = %d, want 2 (full)", len(b.data)) + } + if nb.data[0] != schema.Float(3.0) { + t.Errorf("new buffer data[0] = %v, want 3.0", nb.data[0]) + } +} + +// TestBufferWriteOverwrite verifies that writing to an already-occupied index +// replaces the value rather than appending. +func TestBufferWriteOverwrite(t *testing.T) { + b := newBuffer(100, 10) + b.write(100, schema.Float(1.0)) + b.write(110, schema.Float(2.0)) + + // Overwrite the first slot. + b.write(100, schema.Float(99.0)) + if len(b.data) != 2 { + t.Errorf("len(data) after overwrite = %d, want 2 (no append)", len(b.data)) + } + if b.data[0] != schema.Float(99.0) { + t.Errorf("data[0] after overwrite = %v, want 99.0", b.data[0]) + } +} + +// ─── Buffer read ────────────────────────────────────────────────────────────── + +// TestBufferReadBeforeFirstWrite verifies that 'from' is clamped to firstWrite +// when the requested range starts before any data in the chain. +func TestBufferReadBeforeFirstWrite(t *testing.T) { + b := newBuffer(100, 10) // firstWrite = 100 + b.write(100, schema.Float(1.0)) + b.write(110, schema.Float(2.0)) + + data := make([]schema.Float, 10) + result, adjustedFrom, _, err := b.read(50, 120, data) + if err != nil { + t.Fatalf("read() error = %v", err) + } + if adjustedFrom != 100 { + t.Errorf("adjustedFrom = %d, want 100 (clamped to firstWrite)", adjustedFrom) + } + if len(result) != 2 { + t.Errorf("len(result) = %d, want 2", len(result)) + } +} + +// TestBufferReadChain verifies that read() traverses a multi-buffer chain and +// returns contiguous values from both buffers. +// +// The switch to b.next in read() triggers on idx >= cap(b.data), so b1 must +// be full (len == cap) for the loop to advance to b2 without producing NaN. +func TestBufferReadChain(t *testing.T) { + // b1: cap=3, covers t=100..120. b2: covers t=130..150. b2 is head. + b1 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 95} + b1.data = append(b1.data, 1.0, 2.0, 3.0) // fills b1: len=cap=3 + + b2 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 125} + b2.data = append(b2.data, 4.0, 5.0, 6.0) // t=130,140,150 + b2.prev = b1 + b1.next = b2 + + data := make([]schema.Float, 6) + result, from, to, err := b2.read(100, 160, data) + if err != nil { + t.Fatalf("read() error = %v", err) + } + if from != 100 || to != 160 { + t.Errorf("read() from/to = %d/%d, want 100/160", from, to) + } + if len(result) != 6 { + t.Fatalf("len(result) = %d, want 6", len(result)) + } + for i, want := range []schema.Float{1, 2, 3, 4, 5, 6} { + if result[i] != want { + t.Errorf("result[%d] = %v, want %v", i, result[i], want) + } + } +} + +// TestBufferReadIdxAfterSwitch is a regression test for the index recalculation +// bug after switching to b.next during a read. +// +// When both buffers share the same start time (can happen with checkpoint-loaded +// chains), the old code hardcoded idx=0 after the switch, causing reads at time t +// to return the wrong element from the next buffer. +func TestBufferReadIdxAfterSwitch(t *testing.T) { + // b1: cap=2, both buffers start at 0 (firstWrite=5). + // b1 carries t=5 and t=15; b2 carries t=5,15,25,35 with the same start. + // When reading reaches t=25 the loop overflows b1 (idx=2 >= cap=2) and + // switches to b2. The correct index in b2 is (25-0)/10=2 → b2.data[2]=30.0. + // The old code set idx=0 → b2.data[0]=10.0 (wrong). + b1 := &buffer{data: make([]schema.Float, 0, 2), frequency: 10, start: 0} + b1.data = append(b1.data, schema.Float(1.0), schema.Float(2.0)) // t=5, t=15 + + b2 := &buffer{data: make([]schema.Float, 0, 10), frequency: 10, start: 0} + b2.data = append(b2.data, + schema.Float(10.0), schema.Float(20.0), + schema.Float(30.0), schema.Float(40.0)) // t=5,15,25,35 + b2.prev = b1 + b1.next = b2 + + // from=0 triggers the walkback to b1 (from < b2.firstWrite=5). + // After clamping, the loop runs t=5,15,25,35. + data := make([]schema.Float, 4) + result, _, _, err := b2.read(0, 36, data) + if err != nil { + t.Fatalf("read() error = %v", err) + } + if len(result) < 3 { + t.Fatalf("len(result) = %d, want >= 3", len(result)) + } + if result[0] != schema.Float(1.0) { + t.Errorf("result[0] (t=5) = %v, want 1.0 (from b1)", result[0]) + } + if result[1] != schema.Float(2.0) { + t.Errorf("result[1] (t=15) = %v, want 2.0 (from b1)", result[1]) + } + // This is the critical assertion: old code returned 10.0 (b2.data[0]). + if result[2] != schema.Float(30.0) { + t.Errorf("result[2] (t=25) = %v, want 30.0 (idx recalculation fix)", result[2]) + } +} + +// TestBufferReadNaNValues verifies that NaN slots written to the buffer are +// returned as NaN during read. +func TestBufferReadNaNValues(t *testing.T) { + b := newBuffer(100, 10) + b.write(100, schema.Float(1.0)) + b.write(110, schema.NaN) + b.write(120, schema.Float(3.0)) + + data := make([]schema.Float, 3) + result, _, _, err := b.read(100, 130, data) + if err != nil { + t.Fatalf("read() error = %v", err) + } + if len(result) != 3 { + t.Fatalf("len(result) = %d, want 3", len(result)) + } + if result[0] != schema.Float(1.0) { + t.Errorf("result[0] = %v, want 1.0", result[0]) + } + if !result[1].IsNaN() { + t.Errorf("result[1] should be NaN, got %v", result[1]) + } + if result[2] != schema.Float(3.0) { + t.Errorf("result[2] = %v, want 3.0", result[2]) + } +} + +// TestBufferReadAccumulation verifies the += accumulation pattern used for +// aggregation: values are added to whatever was already in the data slice. +func TestBufferReadAccumulation(t *testing.T) { + b := newBuffer(100, 10) + b.write(100, schema.Float(3.0)) + b.write(110, schema.Float(5.0)) + + // Pre-populate data slice (simulates a second metric being summed in). + data := []schema.Float{2.0, 1.0, 0.0} + result, _, _, err := b.read(100, 120, data) + if err != nil { + t.Fatalf("read() error = %v", err) + } + // 2.0+3.0=5.0, 1.0+5.0=6.0 + if result[0] != schema.Float(5.0) { + t.Errorf("result[0] = %v, want 5.0 (2+3)", result[0]) + } + if result[1] != schema.Float(6.0) { + t.Errorf("result[1] = %v, want 6.0 (1+5)", result[1]) + } +} + +// ─── Buffer free ───────────────────────────────────────────────────────────── + +// newTestPool swaps out the package-level bufferPool for a fresh isolated one +// and returns a cleanup function that restores the original. +func newTestPool(t *testing.T) *PersistentBufferPool { + t.Helper() + pool := NewPersistentBufferPool() + saved := bufferPool + bufferPool = pool + t.Cleanup(func() { bufferPool = saved }) + return pool +} + +// TestBufferFreeRetention verifies that free() removes buffers whose entire +// time range falls before the retention threshold and returns them to the pool. +func TestBufferFreeRetention(t *testing.T) { + pool := newTestPool(t) + + // b1: firstWrite=5, end=25 b2: firstWrite=25, end=45 b3: firstWrite=45, end=65 + b1 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 0} + b1.data = append(b1.data, 1.0, 2.0) + + b2 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 20} + b2.data = append(b2.data, 3.0, 4.0) + b2.prev = b1 + b1.next = b2 + + b3 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 40} + b3.data = append(b3.data, 5.0, 6.0) + b3.prev = b2 + b2.next = b3 + + // Threshold=30: b1.end()=25 < 30 → freed; b2.end()=45 >= 30 → kept. + delme, n := b3.free(30) + if delme { + t.Error("head buffer b3 should not be marked for deletion") + } + if n != 1 { + t.Errorf("freed count = %d, want 1", n) + } + if b2.prev != nil { + t.Error("b1 should have been unlinked from b2.prev") + } + if b3.prev != b2 { + t.Error("b3 should still reference b2") + } + if pool.GetSize() != 1 { + t.Errorf("pool size = %d, want 1 (b1 returned)", pool.GetSize()) + } +} + +// TestBufferFreeAll verifies that free() removes all buffers and signals the +// caller to delete the head when the entire chain is older than the threshold. +func TestBufferFreeAll(t *testing.T) { + pool := newTestPool(t) + + b1 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 0} + b1.data = append(b1.data, 1.0, 2.0) // end=25 + + b2 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 20} + b2.data = append(b2.data, 3.0, 4.0) // end=45 + b2.prev = b1 + b1.next = b2 + + // Threshold=100 > both ends → both should be freed. + delme, n := b2.free(100) + if !delme { + t.Error("head buffer b2 should be marked for deletion when all data is stale") + } + if n != 2 { + t.Errorf("freed count = %d, want 2", n) + } + // b1 was freed inside free(); b2 is returned with delme=true for the caller. + if pool.GetSize() != 1 { + t.Errorf("pool size = %d, want 1 (b1 returned; b2 returned by caller)", pool.GetSize()) + } +} + +// ─── forceFreeOldest ───────────────────────────────────────────────────────── + +// TestForceFreeOldestPoolReturn verifies that forceFreeOldest() returns the +// freed buffer to the pool (regression: previously it was just dropped). +func TestForceFreeOldestPoolReturn(t *testing.T) { + pool := newTestPool(t) + + b1 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 0} + b2 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 20} + b3 := &buffer{data: make([]schema.Float, 0, BufferCap), frequency: 10, start: 40} + b1.data = append(b1.data, 1.0) + b2.data = append(b2.data, 2.0) + b3.data = append(b3.data, 3.0) + b2.prev = b1 + b1.next = b2 + b3.prev = b2 + b2.next = b3 + + delme, n := b3.forceFreeOldest() + if delme { + t.Error("head b3 should not be marked for deletion (chain has 3 buffers)") + } + if n != 1 { + t.Errorf("freed count = %d, want 1", n) + } + if b2.prev != nil { + t.Error("b1 should have been unlinked from b2.prev after forceFreeOldest") + } + if b3.prev != b2 { + t.Error("b3 should still link to b2") + } + if pool.GetSize() != 1 { + t.Errorf("pool size = %d, want 1 (b1 returned to pool)", pool.GetSize()) + } +} + +// TestForceFreeOldestSingleBuffer verifies that forceFreeOldest() returns +// delme=true when the buffer is the only one in the chain. +func TestForceFreeOldestSingleBuffer(t *testing.T) { + b := newBuffer(100, 10) + b.write(100, schema.Float(1.0)) + + delme, n := b.forceFreeOldest() + if !delme { + t.Error("single-buffer chain: expected delme=true (the buffer IS the oldest)") + } + if n != 1 { + t.Errorf("freed count = %d, want 1", n) + } +} + +// ─── iterFromTo ─────────────────────────────────────────────────────────────── + +// TestBufferIterFromToOrder verifies that iterFromTo invokes the callback in +// chronological order (oldest → newest). +func TestBufferIterFromToOrder(t *testing.T) { + // Each buffer has 2 data points so end() = firstWrite + 2*freq. + b1 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 0} // end=25 + b2 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 20} // end=45 + b3 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 40} // end=65 + b2.prev = b1 + b1.next = b2 + b3.prev = b2 + b2.next = b3 + + var order []*buffer + err := b3.iterFromTo(0, 100, func(b *buffer) error { + order = append(order, b) + return nil + }) + if err != nil { + t.Fatalf("iterFromTo() error = %v", err) + } + if len(order) != 3 { + t.Fatalf("callback count = %d, want 3", len(order)) + } + if order[0] != b1 || order[1] != b2 || order[2] != b3 { + t.Error("iterFromTo() did not call callbacks in chronological (oldest→newest) order") + } +} + +// TestBufferIterFromToFiltered verifies that iterFromTo only calls the callback +// for buffers whose time range overlaps [from, to]. +func TestBufferIterFromToFiltered(t *testing.T) { + // b1: end=25 b2: start=20, end=45 b3: start=40, end=65 + b1 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 0} + b2 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 20} + b3 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 40} + b2.prev = b1 + b1.next = b2 + b3.prev = b2 + b2.next = b3 + + // [30,50]: b1.end=25 < 30 → excluded; b2 and b3 overlap → included. + var visited []*buffer + b3.iterFromTo(30, 50, func(b *buffer) error { + visited = append(visited, b) + return nil + }) + if len(visited) != 2 { + t.Fatalf("visited count = %d, want 2 (b2 and b3)", len(visited)) + } + if visited[0] != b2 || visited[1] != b3 { + t.Errorf("visited = %v, want [b2, b3]", visited) + } +} + +// TestBufferIterFromToNilBuffer verifies that iterFromTo on a nil buffer is a +// safe no-op. +func TestBufferIterFromToNilBuffer(t *testing.T) { + var b *buffer + called := false + err := b.iterFromTo(0, 100, func(_ *buffer) error { + called = true + return nil + }) + if err != nil { + t.Errorf("iterFromTo(nil) error = %v, want nil", err) + } + if called { + t.Error("callback should not be called for a nil buffer") + } +} + +// ─── count ──────────────────────────────────────────────────────────────────── + +// TestBufferCount verifies that count() sums data-point lengths across the +// entire chain, including all prev links. +func TestBufferCount(t *testing.T) { + b1 := &buffer{data: make([]schema.Float, 3, BufferCap), frequency: 10, start: 0} + b2 := &buffer{data: make([]schema.Float, 2, BufferCap), frequency: 10, start: 35} + b3 := &buffer{data: make([]schema.Float, 5, BufferCap), frequency: 10, start: 60} + b2.prev = b1 + b1.next = b2 + b3.prev = b2 + b2.next = b3 + + if got := b3.count(); got != 10 { + t.Errorf("count() = %d, want 10 (3+2+5)", got) + } + + // Single buffer. + lone := &buffer{data: make([]schema.Float, 7, BufferCap)} + if got := lone.count(); got != 7 { + t.Errorf("count() single buffer = %d, want 7", got) + } +} + +// ─── Existing tests below ──────────────────────────────────────────────────── + func TestAssignAggregationStrategy(t *testing.T) { tests := []struct { name string From 3d5a124321763c3b89323f3e02fcc3584245392a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 2 Mar 2026 15:01:44 +0100 Subject: [PATCH 308/341] Refine patterns. Do not match commented lines. --- configs/tagger/apps/caracal.txt | 1 - configs/tagger/apps/lammps.txt | 2 +- internal/tagger/detectApp.go | 6 ++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configs/tagger/apps/caracal.txt b/configs/tagger/apps/caracal.txt index ed615121..5c5311f7 100644 --- a/configs/tagger/apps/caracal.txt +++ b/configs/tagger/apps/caracal.txt @@ -2,6 +2,5 @@ calc_rate qmdffgen dynamic evbopt -explore black_box poly_qmdff diff --git a/configs/tagger/apps/lammps.txt b/configs/tagger/apps/lammps.txt index d254f82f..38d3aa5d 100644 --- a/configs/tagger/apps/lammps.txt +++ b/configs/tagger/apps/lammps.txt @@ -1 +1 @@ -lmp +\blmp\s+ diff --git a/internal/tagger/detectApp.go b/internal/tagger/detectApp.go index f86dcb6c..54626eff 100644 --- a/internal/tagger/detectApp.go +++ b/internal/tagger/detectApp.go @@ -64,9 +64,11 @@ func (t *AppTagger) scanApp(f *os.File, fns string) { if line == "" { continue } - re, err := regexp.Compile(line) + // Wrap pattern to skip comment lines: match only if not preceded by # on the same line + wrapped := `(?m)^[^#]*` + line + re, err := regexp.Compile(wrapped) if err != nil { - cclog.Errorf("invalid regex pattern '%s' in %s: %v", line, fns, err) + cclog.Errorf("invalid regex pattern '%s' (wrapped: '%s') in %s: %v", line, wrapped, fns, err) continue } ai.patterns = append(ai.patterns, re) From a243e1749921abe59480c7b171a6c43cbcbbf09a Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Mon, 2 Mar 2026 15:27:06 +0100 Subject: [PATCH 309/341] Update to shutdown worker for WAL checkpointing mode --- configs/config-demo.json | 1 + pkg/metricstore/metricstore.go | 2 +- pkg/metricstore/walCheckpoint.go | 12 +++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/configs/config-demo.json b/configs/config-demo.json index 509c8f18..50dff298 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -21,6 +21,7 @@ ], "metric-store": { "checkpoints": { + "file-format": "wal", "interval": "12h" }, "retention-in-memory": "48h", diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index b5b1a528..6d49624a 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -294,7 +294,7 @@ func Shutdown() { var hostDirs []string files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix()) if err == nil { - RotateWALFiles(hostDirs) + RotateWALFilesAfterShutdown(hostDirs) } } else { files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix()) diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go index 685a8388..07414d98 100644 --- a/pkg/metricstore/walCheckpoint.go +++ b/pkg/metricstore/walCheckpoint.go @@ -116,7 +116,6 @@ type walFileState struct { // Also handles WAL rotation requests from the checkpoint goroutine. func WALStaging(wg *sync.WaitGroup, ctx context.Context) { wg.Go(func() { - if Keys.Checkpoints.FileFormat == "json" { return } @@ -235,6 +234,17 @@ func RotateWALFiles(hostDirs []string) { } } +// RotateWALFiles sends rotation requests for the given host directories +// and blocks until all rotations complete. +func RotateWALFilesAfterShutdown(hostDirs []string) { + for _, dir := range hostDirs { + walPath := path.Join(dir, "current.wal") + if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) { + cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err) + } + } +} + // buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC). func buildWALPayload(msg *WALMessage) []byte { size := 8 + 2 + len(msg.MetricName) + 1 + 4 From 718ff60221028881d01d3b7dd05f21991cc84018 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 2 Mar 2026 14:10:28 +0100 Subject: [PATCH 310/341] clarify ccms logs --- cmd/cc-backend/main.go | 2 +- internal/metricdispatch/metricdata.go | 4 ++-- .../metricstoreclient/cc-metric-store-queries.go | 4 ++-- internal/metricstoreclient/cc-metric-store.go | 10 +++++----- pkg/metricstore/query.go | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 81d397d2..5b51b963 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -339,7 +339,7 @@ func runServer(ctx context.Context) error { err := metricdispatch.Init(mscfg) if err != nil { - cclog.Debugf("initializing metricdispatch: %v", err) + cclog.Debugf("error while initializing external metricdispatch: %v", err) } else { haveMetricstore = true } diff --git a/internal/metricdispatch/metricdata.go b/internal/metricdispatch/metricdata.go index 36a10004..3f03234e 100755 --- a/internal/metricdispatch/metricdata.go +++ b/internal/metricdispatch/metricdata.go @@ -74,11 +74,11 @@ func Init(rawConfig json.RawMessage) error { dec := json.NewDecoder(bytes.NewReader(rawConfig)) dec.DisallowUnknownFields() if err := dec.Decode(&configs); err != nil { - return fmt.Errorf("[METRICDISPATCH]> Metric Store Config Init: Could not decode config file '%s' Error: %s", rawConfig, err.Error()) + return fmt.Errorf("[METRICDISPATCH]> External Metric Store Config Init: Could not decode config file '%s' Error: %s", rawConfig, err.Error()) } if len(configs) == 0 { - return fmt.Errorf("[METRICDISPATCH]> No metric store configurations found in config file") + return fmt.Errorf("[METRICDISPATCH]> No external metric store configurations found in config file") } for _, config := range configs { diff --git a/internal/metricstoreclient/cc-metric-store-queries.go b/internal/metricstoreclient/cc-metric-store-queries.go index d42c9355..949efa10 100644 --- a/internal/metricstoreclient/cc-metric-store-queries.go +++ b/internal/metricstoreclient/cc-metric-store-queries.go @@ -134,7 +134,7 @@ func (ccms *CCMetricStore) buildQueries( ) if len(hostQueries) == 0 && len(hostScopes) == 0 { - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } queries = append(queries, hostQueries...) @@ -237,7 +237,7 @@ func (ccms *CCMetricStore) buildNodeQueries( ) if len(nodeQueries) == 0 && len(nodeScopes) == 0 { - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } queries = append(queries, nodeQueries...) diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go index 7bf7d146..39c028d5 100644 --- a/internal/metricstoreclient/cc-metric-store.go +++ b/internal/metricstoreclient/cc-metric-store.go @@ -123,7 +123,7 @@ type APIMetricData struct { Max schema.Float `json:"max"` // Maximum value in time range } -// NewCCMetricStore creates and initializes a new CCMetricStore client. +// NewCCMetricStore creates and initializes a new (external) CCMetricStore client. // The url parameter should include the protocol and port (e.g., "http://localhost:8080"). // The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled. func NewCCMetricStore(url string, token string) *CCMetricStore { @@ -356,7 +356,7 @@ func (ccms *CCMetricStore) LoadData( if len(errors) != 0 { /* Returns list for "partial errors" */ - return jobData, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return jobData, nil } @@ -514,7 +514,7 @@ func (ccms *CCMetricStore) LoadScopedStats( if len(errors) != 0 { /* Returns list for "partial errors" */ - return scopedJobStats, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return scopedJobStats, nil } @@ -604,7 +604,7 @@ func (ccms *CCMetricStore) LoadNodeData( if len(errors) != 0 { /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return data, nil @@ -765,7 +765,7 @@ func (ccms *CCMetricStore) LoadNodeListData( if len(errors) != 0 { /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return data, nil diff --git a/pkg/metricstore/query.go b/pkg/metricstore/query.go index 0a61efaa..735c45d6 100644 --- a/pkg/metricstore/query.go +++ b/pkg/metricstore/query.go @@ -211,7 +211,7 @@ func (ccms *InternalMetricStore) LoadData( if len(errors) != 0 { /* Returns list for "partial errors" */ - return jobData, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return jobData, nil } @@ -260,7 +260,7 @@ func buildQueries( resolution int64, ) ([]APIQuery, []schema.MetricScope, error) { if len(job.Resources) == 0 { - return nil, nil, fmt.Errorf("METRICDATA/CCMS > no resources allocated for job %d", job.JobID) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID) } queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) @@ -531,7 +531,7 @@ func buildQueries( continue } - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } } } @@ -719,7 +719,7 @@ func (ccms *InternalMetricStore) LoadScopedStats( if len(errors) != 0 { /* Returns list for "partial errors" */ - return scopedJobStats, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return scopedJobStats, nil } @@ -824,7 +824,7 @@ func (ccms *InternalMetricStore) LoadNodeData( if len(errors) != 0 { /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return data, nil @@ -994,7 +994,7 @@ func (ccms *InternalMetricStore) LoadNodeListData( if len(errors) != 0 { /* Returns list of "partial errors" */ - return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) + return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", ")) } return data, nil @@ -1313,7 +1313,7 @@ func buildNodeQueries( continue } - return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } } } From 32fd18543a150fb8ca9436091dfa37d97d0ffd10 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 2 Mar 2026 15:35:07 +0100 Subject: [PATCH 311/341] differentiate between expected and unexpected cases in external ccms queryBuilder --- .../metricstoreclient/cc-metric-store-queries.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/metricstoreclient/cc-metric-store-queries.go b/internal/metricstoreclient/cc-metric-store-queries.go index 949efa10..7a04efc4 100644 --- a/internal/metricstoreclient/cc-metric-store-queries.go +++ b/internal/metricstoreclient/cc-metric-store-queries.go @@ -126,6 +126,7 @@ func (ccms *CCMetricStore) buildQueries( hwthreads = topology.Node } + // Note: Expected exceptions will return as empty slices -> Continue hostQueries, hostScopes := buildScopeQueries( nativeScope, requestedScope, remoteName, host.Hostname, @@ -133,8 +134,9 @@ func (ccms *CCMetricStore) buildQueries( resolution, ) - if len(hostQueries) == 0 && len(hostScopes) == 0 { - return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + // Note: Unexpected errors, such as unhandled cases, will return as nils -> Error + if hostQueries == nil && hostScopes == nil { + return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } queries = append(queries, hostQueries...) @@ -269,6 +271,7 @@ func buildScopeQueries( // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { if scope != schema.MetricScopeAccelerator { + // Expected Exception -> Continue -> Return Empty Slices return queries, scopes } @@ -287,6 +290,7 @@ func buildScopeQueries( // Accelerator -> Node if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { if len(accelerators) == 0 { + // Expected Exception -> Continue -> Return Empty Slices return queries, scopes } @@ -447,6 +451,7 @@ func buildScopeQueries( socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains) if err != nil { cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err) + // Rare Error Case -> Still Continue -> Return Empty Slices return queries, scopes } @@ -507,8 +512,8 @@ func buildScopeQueries( return queries, scopes } - // Unhandled case - return empty slices - return queries, scopes + // Unhandled Case -> Error -> Return nils + return nil, nil } // intToStringSlice converts a slice of integers to a slice of strings. From 38bb2dd4ec9953dadc271bcb30abd87374c1d601 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 2 Mar 2026 16:24:27 +0100 Subject: [PATCH 312/341] add outofindex checks to external ccms codebase --- internal/metricstoreclient/cc-metric-store.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go index 39c028d5..e2a84466 100644 --- a/internal/metricstoreclient/cc-metric-store.go +++ b/internal/metricstoreclient/cc-metric-store.go @@ -393,6 +393,10 @@ func (ccms *CCMetricStore) LoadStats( stats := make(map[string]map[string]schema.MetricStatistics, len(metrics)) for i, res := range resBody.Results { + if len(res) == 0 { + // No Data Found For Metric, Logged in FetchData to Warn + continue + } query := req.Queries[i] metric := query.Metric data := res[0] @@ -562,6 +566,11 @@ func (ccms *CCMetricStore) LoadNodeData( var errors []string data := make(map[string]map[string][]*schema.JobMetric) for i, res := range resBody.Results { + if len(res) == 0 { + // No Data Found For Metric, Logged in FetchData to Warn + continue + } + var query APIQuery if resBody.Queries != nil { query = resBody.Queries[i] @@ -572,7 +581,6 @@ func (ccms *CCMetricStore) LoadNodeData( metric := query.Metric qdata := res[0] if qdata.Error != nil { - /* Build list for "partial errors", if any */ errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) } From 22c442db5bfb6cb6d074797e105adb6813cef30d Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 2 Mar 2026 18:47:45 +0100 Subject: [PATCH 313/341] Enable entire integration --- .claude/settings.json | 84 +++++++++++++++++++++++++++++++++++++++++++ .entire/.gitignore | 4 +++ .entire/settings.json | 4 +++ 3 files changed, 92 insertions(+) create mode 100644 .claude/settings.json create mode 100644 .entire/.gitignore create mode 100644 .entire/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..5cfa5854 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,84 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Task", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code post-task" + } + ] + }, + { + "matcher": "TodoWrite", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code post-todo" + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "Task", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code pre-task" + } + ] + } + ], + "SessionEnd": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code session-end" + } + ] + } + ], + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code session-start" + } + ] + } + ], + "Stop": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code stop" + } + ] + } + ], + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "entire hooks claude-code user-prompt-submit" + } + ] + } + ] + }, + "permissions": { + "deny": [ + "Read(./.entire/metadata/**)" + ] + } +} diff --git a/.entire/.gitignore b/.entire/.gitignore new file mode 100644 index 00000000..2cffdefa --- /dev/null +++ b/.entire/.gitignore @@ -0,0 +1,4 @@ +tmp/ +settings.local.json +metadata/ +logs/ diff --git a/.entire/settings.json b/.entire/settings.json new file mode 100644 index 00000000..7cce5590 --- /dev/null +++ b/.entire/settings.json @@ -0,0 +1,4 @@ +{ + "enabled": true, + "telemetry": true +} From 15be664ad806a470d57e6675454338077bf937f0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 3 Mar 2026 06:58:03 +0100 Subject: [PATCH 314/341] Add entire gitignore --- .entire/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .entire/.gitignore diff --git a/.entire/.gitignore b/.entire/.gitignore new file mode 100644 index 00000000..2cffdefa --- /dev/null +++ b/.entire/.gitignore @@ -0,0 +1,4 @@ +tmp/ +settings.local.json +metadata/ +logs/ From 74ab51f40945cd340e77ff3f4f632a33d40f7ed3 Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Tue, 3 Mar 2026 09:51:04 +0100 Subject: [PATCH 315/341] Patch bufferPool with no limits to pool size --- pkg/metricstore/buffer.go | 9 +-------- pkg/metricstore/metricstore_test.go | 17 ----------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/pkg/metricstore/buffer.go b/pkg/metricstore/buffer.go index 557a941c..2d752006 100644 --- a/pkg/metricstore/buffer.go +++ b/pkg/metricstore/buffer.go @@ -54,10 +54,6 @@ import ( // of data or reallocation needs to happen on writes. const BufferCap int = DefaultBufferCapacity -// maxPoolSize caps the number of buffers held in the pool at any time. -// Prevents unbounded memory growth after large retention-cleanup bursts. -const maxPoolSize = 4096 - // BufferPool is the global instance. // It is initialized immediately when the package loads. var bufferPool = NewPersistentBufferPool() @@ -101,10 +97,7 @@ func (p *PersistentBufferPool) Put(b *buffer) { p.mu.Lock() defer p.mu.Unlock() - if len(p.pool) >= maxPoolSize { - // Pool is full; drop the buffer and let GC collect it. - return - } + p.pool = append(p.pool, b) } diff --git a/pkg/metricstore/metricstore_test.go b/pkg/metricstore/metricstore_test.go index 9087df2a..35f97278 100644 --- a/pkg/metricstore/metricstore_test.go +++ b/pkg/metricstore/metricstore_test.go @@ -52,23 +52,6 @@ func TestBufferPoolClear(t *testing.T) { } } -// TestBufferPoolMaxSize verifies that Put() silently drops buffers once the -// pool reaches maxPoolSize, preventing unbounded memory growth. -func TestBufferPoolMaxSize(t *testing.T) { - pool := NewPersistentBufferPool() - for i := 0; i < maxPoolSize; i++ { - pool.Put(&buffer{data: make([]schema.Float, 0, BufferCap), lastUsed: time.Now().Unix()}) - } - if pool.GetSize() != maxPoolSize { - t.Fatalf("pool size = %d, want %d", pool.GetSize(), maxPoolSize) - } - - pool.Put(&buffer{data: make([]schema.Float, 0, BufferCap), lastUsed: time.Now().Unix()}) - if pool.GetSize() != maxPoolSize { - t.Errorf("pool size after overflow Put = %d, want %d (should not grow)", pool.GetSize(), maxPoolSize) - } -} - // ─── Buffer helpers ─────────────────────────────────────────────────────────── // TestBufferEndFirstWrite verifies the end() and firstWrite() calculations. From 5669eb5818b0a1c8cbeba6ae74da4e7ce7e40bbd Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 3 Mar 2026 15:41:44 +0100 Subject: [PATCH 316/341] Optimize queries for existing indices --- internal/repository/jobQuery.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 81779583..c3fbe042 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -282,7 +282,7 @@ func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuild if cond.From != 0 && cond.To != 0 { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) } else if cond.From != 0 { - return query.Where("? <= "+field, cond.From) + return query.Where(field+" >= ?", cond.From) } else if cond.To != 0 { return query.Where(field+" <= ?", cond.To) } else { @@ -296,7 +296,7 @@ func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBu if cond.From != 0.0 && cond.To != 0.0 { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) } else if cond.From != 0.0 { - return query.Where("? <= "+field, cond.From) + return query.Where(field+" >= ?", cond.From) } else if cond.To != 0.0 { return query.Where(field+" <= ?", cond.To) } else { @@ -311,7 +311,7 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui if cond.From != nil && cond.To != nil { return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix()) } else if cond.From != nil { - return query.Where("? <= "+field, cond.From.Unix()) + return query.Where(field+" >= ?", cond.From.Unix()) } else if cond.To != nil { return query.Where(field+" <= ?", cond.To.Unix()) } else if cond.Range != "" { @@ -330,7 +330,7 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range) return query } - return query.Where("? <= "+field, then) + return query.Where(field+" >= ?", then) } else { return query } @@ -343,7 +343,7 @@ func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query if condRange.From != 0.0 && condRange.To != 0.0 { return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To) } else if condRange.From != 0.0 { - return query.Where("? <= JSON_EXTRACT(footprint, \"$."+condName+"\")", condRange.From) + return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") >= ?", condRange.From) } else if condRange.To != 0.0 { return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To) } else { From 763e0c8d7c20de834f2825535e6f7967ec2a8e2b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 3 Mar 2026 15:43:38 +0100 Subject: [PATCH 317/341] Upgrade dependencies --- go.mod | 68 +++++++++++++------------- go.sum | 151 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 108 insertions(+), 111 deletions(-) diff --git a/go.mod b/go.mod index a902c09a..2e72e342 100644 --- a/go.mod +++ b/go.mod @@ -8,14 +8,14 @@ tool ( ) require ( - github.com/99designs/gqlgen v0.17.86 + github.com/99designs/gqlgen v0.17.87 github.com/ClusterCockpit/cc-lib/v2 v2.7.0 github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 github.com/Masterminds/squirrel v1.5.4 - github.com/aws/aws-sdk-go-v2 v1.41.1 - github.com/aws/aws-sdk-go-v2/config v1.32.8 - github.com/aws/aws-sdk-go-v2/credentials v1.19.8 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 + github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2/config v1.32.10 + github.com/aws/aws-sdk-go-v2/credentials v1.19.10 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 github.com/coreos/go-oidc/v3 v3.17.0 github.com/expr-lang/expr v1.17.8 github.com/go-chi/chi/v5 v5.2.5 @@ -29,13 +29,13 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/mattn/go-sqlite3 v1.14.34 - github.com/parquet-go/parquet-go v0.27.0 + github.com/parquet-go/parquet-go v0.28.0 github.com/qustavo/sqlhooks/v2 v2.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/stretchr/testify v1.11.1 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 - github.com/vektah/gqlparser/v2 v2.5.31 + github.com/vektah/gqlparser/v2 v2.5.32 golang.org/x/crypto v0.48.0 golang.org/x/oauth2 v0.35.0 golang.org/x/time v0.14.0 @@ -47,36 +47,36 @@ require ( github.com/agnivade/levenshtein v1.2.1 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect - github.com/aws/smithy-go v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect - github.com/go-openapi/jsonpointer v0.22.4 // indirect - github.com/go-openapi/jsonreference v0.21.4 // indirect - github.com/go-openapi/spec v0.22.3 // indirect - github.com/go-openapi/swag/conv v0.25.4 // indirect - github.com/go-openapi/swag/jsonname v0.25.4 // indirect - github.com/go-openapi/swag/jsonutils v0.25.4 // indirect - github.com/go-openapi/swag/loading v0.25.4 // indirect - github.com/go-openapi/swag/stringutils v0.25.4 // indirect - github.com/go-openapi/swag/typeutils v0.25.4 // indirect - github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/jsonpointer v0.22.5 // indirect + github.com/go-openapi/jsonreference v0.21.5 // indirect + github.com/go-openapi/spec v0.22.4 // indirect + github.com/go-openapi/swag/conv v0.25.5 // indirect + github.com/go-openapi/swag/jsonname v0.25.5 // indirect + github.com/go-openapi/swag/jsonutils v0.25.5 // indirect + github.com/go-openapi/swag/loading v0.25.5 // indirect + github.com/go-openapi/swag/stringutils v0.25.5 // indirect + github.com/go-openapi/swag/typeutils v0.25.5 // indirect + github.com/go-openapi/swag/yamlutils v0.25.5 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/uuid v1.6.0 // indirect @@ -101,13 +101,13 @@ require ( github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sosodev/duration v1.3.1 // indirect - github.com/stmcginnis/gofish v0.21.3 // indirect + github.com/sosodev/duration v1.4.0 // indirect + github.com/stmcginnis/gofish v0.21.4 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/swaggo/files v1.0.1 // indirect github.com/twpayne/go-geom v1.6.1 // indirect github.com/urfave/cli/v2 v2.27.7 // indirect - github.com/urfave/cli/v3 v3.6.1 // indirect + github.com/urfave/cli/v3 v3.6.2 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/go.sum b/go.sum index 04cc861a..70f98fc8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/99designs/gqlgen v0.17.86 h1:C8N3UTa5heXX6twl+b0AJyGkTwYL6dNmFrgZNLRcU6w= -github.com/99designs/gqlgen v0.17.86/go.mod h1:KTrPl+vHA1IUzNlh4EYkl7+tcErL3MgKnhHrBcV74Fw= +github.com/99designs/gqlgen v0.17.87 h1:pSnCIMhBQezAE8bc1GNmfdLXFmnWtWl1GRDFEE/nHP8= +github.com/99designs/gqlgen v0.17.87/go.mod h1:fK05f1RqSNfQpd4CfW5qk/810Tqi4/56Wf6Nem0khAg= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/ClusterCockpit/cc-lib/v2 v2.7.0 h1:EMTShk6rMTR1wlfmQ8SVCawH1OdltUbD3kVQmaW+5pE= @@ -39,44 +39,44 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= -github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= -github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs= -github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0= -github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= +github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= +github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI= +github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= -github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= -github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= @@ -111,33 +111,33 @@ github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZR github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= -github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= -github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= -github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= -github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/spec v0.22.3 h1:qRSmj6Smz2rEBxMnLRBMeBWxbbOvuOoElvSvObIgwQc= -github.com/go-openapi/spec v0.22.3/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs= +github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= +github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= +github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= +github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= +github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ= +github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= -github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= -github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= -github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= -github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= -github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= -github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= -github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= -github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= -github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= -github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= -github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g= +github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k= +github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= +github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo= +github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo= +github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU= +github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g= +github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M= +github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII= +github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E= +github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc= +github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ= +github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= +github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= +github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= @@ -149,8 +149,6 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA= @@ -238,8 +236,8 @@ github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxP github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= github.com/parquet-go/jsonlite v1.4.0 h1:RTG7prqfO0HD5egejU8MUDBN8oToMj55cgSV1I0zNW4= github.com/parquet-go/jsonlite v1.4.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= -github.com/parquet-go/parquet-go v0.27.0 h1:vHWK2xaHbj+v1DYps03yDRpEsdtOeKbhiXUaixoPb3g= -github.com/parquet-go/parquet-go v0.27.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/parquet-go/parquet-go v0.28.0 h1:ECyksyv8T2pOrlLsN7aWJIoQakyk/HtxQ2lchgS4els= +github.com/parquet-go/parquet-go v0.28.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -267,11 +265,11 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= -github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE= +github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= -github.com/stmcginnis/gofish v0.21.3 h1:EBLCHfORnbx7MPw7lplOOVe9QAD1T3XRVz6+a1Z4z5Q= -github.com/stmcginnis/gofish v0.21.3/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= +github.com/stmcginnis/gofish v0.21.4 h1:daexK8sh31CgeSMkPUNs21HWHHA9ecCPJPyLCTxukCg= +github.com/stmcginnis/gofish v0.21.4/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -290,10 +288,10 @@ github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4 github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= -github.com/urfave/cli/v3 v3.6.1 h1:j8Qq8NyUawj/7rTYdBGrxcH7A/j7/G8Q5LhWEW4G3Mo= -github.com/urfave/cli/v3 v3.6.1/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= -github.com/vektah/gqlparser/v2 v2.5.31 h1:YhWGA1mfTjID7qJhd1+Vxhpk5HTgydrGU9IgkWBTJ7k= -github.com/vektah/gqlparser/v2 v2.5.31/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= +github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8= +github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJrawpFSc= +github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -357,7 +355,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= From 6da41982cebebcc615cd0821fb1ef929d66490ba Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 3 Mar 2026 15:54:51 +0100 Subject: [PATCH 318/341] Further optimize queries for better index usage --- internal/repository/job.go | 2 +- internal/repository/jobFind.go | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 8055ca37..566a13b1 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -718,7 +718,7 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { Set("job_state", schema.JobStateFailed). Where("job.job_state = 'running'"). Where("job.walltime > 0"). - Where("(? - job.start_time) > (job.walltime + ?)", currentTime, seconds). + Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)). RunWith(r.DB).Exec() if err != nil { cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index d79847a0..13dd4418 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -270,24 +270,21 @@ func (r *JobRepository) FindConcurrentJobs( stopTime = startTime + int64(job.Duration) } - // Time buffer constants for finding overlapping jobs - // overlapBufferStart: 10s grace period at job start to catch jobs starting just after + // Time buffer constant for finding overlapping jobs // overlapBufferEnd: 200s buffer at job end to account for scheduling/cleanup overlap - const overlapBufferStart = 10 const overlapBufferEnd = 200 - startTimeTail := startTime + overlapBufferStart stopTimeTail := stopTime - overlapBufferEnd startTimeFront := startTime + overlapBufferEnd - // Reminder: BETWEEN Queries are slower and dont use indices as frequently: Can this be optimized? - queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)", - "running", startTimeTail, stopTimeTail, startTime) + queryRunning := query.Where("job.job_state = ?", "running"). + Where("job.start_time <= ?", stopTimeTail) // Get At Least One Exact Hostname Match from JSON Resources Array in Database queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname) - query = query.Where("job.job_state != ?").Where("((job.start_time BETWEEN ? AND ?) OR (job.start_time + job.duration) BETWEEN ? AND ? OR (job.start_time < ?) AND (job.start_time + job.duration) > ?)", - "running", startTimeTail, stopTimeTail, startTimeFront, stopTimeTail, startTime, stopTime) + query = query.Where("job.job_state != ?", "running"). + Where("job.start_time < ?", stopTimeTail). + Where("(job.start_time + job.duration) > ?", startTimeFront) // Get At Least One Exact Hostname Match from JSON Resources Array in Database query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname) From 4e1b00a03203df9e421bf8a5eff528097bdbe60e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 3 Mar 2026 16:33:19 +0100 Subject: [PATCH 319/341] frontend dependency bump, includes @urql/svelte 5.0.0 --- web/frontend/package-lock.json | 320 +++++++++++++++++++-------------- web/frontend/package.json | 11 +- 2 files changed, 186 insertions(+), 145 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index e293d650..ea4bdcc0 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -11,11 +11,11 @@ "dependencies": { "@rollup/plugin-replace": "^6.0.3", "@sveltestrap/sveltestrap": "^7.1.0", - "@urql/svelte": "^4.2.3", + "@urql/svelte": "^5.0.0", "chart.js": "^4.5.1", "date-fns": "^4.1.0", - "graphql": "^16.12.0", - "mathjs": "^15.1.0", + "graphql": "^16.13.0", + "mathjs": "^15.1.1", "uplot": "^1.6.32", "wonka": "^6.3.5" }, @@ -24,10 +24,10 @@ "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.55.1", + "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.46.4" + "svelte": "^5.53.6" } }, "node_modules/@0no-co/graphql.web": { @@ -244,9 +244,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.55.1.tgz", - "integrity": "sha512-9R0DM/ykwfGIlNu6+2U09ga0WXeZ9MRC2Ter8jnz8415VbuIykVuc6bhdrbORFZANDmTDvq26mJrEVTl8TdnDg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", + "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", "cpu": [ "arm" ], @@ -257,9 +257,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.55.1.tgz", - "integrity": "sha512-eFZCb1YUqhTysgW3sj/55du5cG57S7UTNtdMjCW7LwVcj3dTTcowCsC8p7uBdzKsZYa8J7IDE8lhMI+HX1vQvg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", + "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", "cpu": [ "arm64" ], @@ -270,9 +270,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.55.1.tgz", - "integrity": "sha512-p3grE2PHcQm2e8PSGZdzIhCKbMCw/xi9XvMPErPhwO17vxtvCN5FEA2mSLgmKlCjHGMQTP6phuQTYWUnKewwGg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", + "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", "cpu": [ "arm64" ], @@ -283,9 +283,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.55.1.tgz", - "integrity": "sha512-rDUjG25C9qoTm+e02Esi+aqTKSBYwVTaoS1wxcN47/Luqef57Vgp96xNANwt5npq9GDxsH7kXxNkJVEsWEOEaQ==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", + "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", "cpu": [ "x64" ], @@ -296,9 +296,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.55.1.tgz", - "integrity": "sha512-+JiU7Jbp5cdxekIgdte0jfcu5oqw4GCKr6i3PJTlXTCU5H5Fvtkpbs4XJHRmWNXF+hKmn4v7ogI5OQPaupJgOg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", + "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", "cpu": [ "arm64" ], @@ -309,9 +309,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.55.1.tgz", - "integrity": "sha512-V5xC1tOVWtLLmr3YUk2f6EJK4qksksOYiz/TCsFHu/R+woubcLWdC9nZQmwjOAbmExBIVKsm1/wKmEy4z4u4Bw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", + "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", "cpu": [ "x64" ], @@ -322,12 +322,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.55.1.tgz", - "integrity": "sha512-Rn3n+FUk2J5VWx+ywrG/HGPTD9jXNbicRtTM11e/uorplArnXZYsVifnPPqNNP5BsO3roI4n8332ukpY/zN7rQ==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", + "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -335,12 +338,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.55.1.tgz", - "integrity": "sha512-grPNWydeKtc1aEdrJDWk4opD7nFtQbMmV7769hiAaYyUKCT1faPRm2av8CX1YJsZ4TLAZcg9gTR1KvEzoLjXkg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", + "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", "cpu": [ "arm" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -348,12 +354,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.55.1.tgz", - "integrity": "sha512-a59mwd1k6x8tXKcUxSyISiquLwB5pX+fJW9TkWU46lCqD/GRDe9uDN31jrMmVP3feI3mhAdvcCClhV8V5MhJFQ==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", + "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -361,12 +370,15 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.55.1.tgz", - "integrity": "sha512-puS1MEgWX5GsHSoiAsF0TYrpomdvkaXm0CofIMG5uVkP6IBV+ZO9xhC5YEN49nsgYo1DuuMquF9+7EDBVYu4uA==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", + "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -374,12 +386,15 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.55.1.tgz", - "integrity": "sha512-r3Wv40in+lTsULSb6nnoudVbARdOwb2u5fpeoOAZjFLznp6tDU8kd+GTHmJoqZ9lt6/Sys33KdIHUaQihFcu7g==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", + "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", "cpu": [ "loong64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -387,12 +402,15 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.55.1.tgz", - "integrity": "sha512-MR8c0+UxAlB22Fq4R+aQSPBayvYa3+9DrwG/i1TKQXFYEaoW3B5b/rkSRIypcZDdWjWnpcvxbNaAJDcSbJU3Lw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", + "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", "cpu": [ "loong64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -400,12 +418,15 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.55.1.tgz", - "integrity": "sha512-3KhoECe1BRlSYpMTeVrD4sh2Pw2xgt4jzNSZIIPLFEsnQn9gAnZagW9+VqDqAHgm1Xc77LzJOo2LdigS5qZ+gw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", + "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -413,12 +434,15 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.55.1.tgz", - "integrity": "sha512-ziR1OuZx0vdYZZ30vueNZTg73alF59DicYrPViG0NEgDVN8/Jl87zkAPu4u6VjZST2llgEUjaiNl9JM6HH1Vdw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", + "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", "cpu": [ "ppc64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -426,12 +450,15 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.55.1.tgz", - "integrity": "sha512-uW0Y12ih2XJRERZ4jAfKamTyIHVMPQnTZcQjme2HMVDAHY4amf5u414OqNYC+x+LzRdRcnIG1YodLrrtA8xsxw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", + "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -439,12 +466,15 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.55.1.tgz", - "integrity": "sha512-u9yZ0jUkOED1BFrqu3BwMQoixvGHGZ+JhJNkNKY/hyoEgOwlqKb62qu+7UjbPSHYjiVy8kKJHvXKv5coH4wDeg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", + "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", "cpu": [ "riscv64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -452,12 +482,15 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.55.1.tgz", - "integrity": "sha512-/0PenBCmqM4ZUd0190j7J0UsQ/1nsi735iPRakO8iPciE7BQ495Y6msPzaOmvx0/pn+eJVVlZrNrSh4WSYLxNg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", + "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -465,12 +498,15 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.55.1.tgz", - "integrity": "sha512-a8G4wiQxQG2BAvo+gU6XrReRRqj+pLS2NGXKm8io19goR+K8lw269eTrPkSdDTALwMmJp4th2Uh0D8J9bEV1vg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", + "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -478,12 +514,15 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.55.1.tgz", - "integrity": "sha512-bD+zjpFrMpP/hqkfEcnjXWHMw5BIghGisOKPj+2NaNDuVT+8Ds4mPf3XcPHuat1tz89WRL+1wbcxKY3WSbiT7w==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", + "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -491,9 +530,9 @@ ] }, "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.55.1.tgz", - "integrity": "sha512-eLXw0dOiqE4QmvikfQ6yjgkg/xDM+MdU9YJuP4ySTibXU0oAvnEWXt7UDJmD4UkYialMfOGFPJnIHSe/kdzPxg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", + "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", "cpu": [ "x64" ], @@ -504,9 +543,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.55.1.tgz", - "integrity": "sha512-xzm44KgEP11te3S2HCSyYf5zIzWmx3n8HDCc7EE59+lTcswEWNpvMLfd9uJvVX8LCg9QWG67Xt75AuHn4vgsXw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", + "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", "cpu": [ "arm64" ], @@ -517,9 +556,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.55.1.tgz", - "integrity": "sha512-yR6Bl3tMC/gBok5cz/Qi0xYnVbIxGx5Fcf/ca0eB6/6JwOY+SRUcJfI0OpeTpPls7f194as62thCt/2BjxYN8g==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", + "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", "cpu": [ "arm64" ], @@ -530,9 +569,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.55.1.tgz", - "integrity": "sha512-3fZBidchE0eY0oFZBnekYCfg+5wAB0mbpCBuofh5mZuzIU/4jIVkbESmd2dOsFNS78b53CYv3OAtwqkZZmU5nA==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", + "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", "cpu": [ "ia32" ], @@ -543,9 +582,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.55.1.tgz", - "integrity": "sha512-xGGY5pXj69IxKb4yv/POoocPy/qmEGhimy/FoTpTSVju3FYXUQQMFCaZZXJVidsmGxRioZAwpThl/4zX41gRKg==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", + "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", "cpu": [ "x64" ], @@ -556,9 +595,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.55.1.tgz", - "integrity": "sha512-SPEpaL6DX4rmcXtnhdrQYgzQ5W2uW3SCJch88lB2zImhJRhIIK44fkUrgIV/Q8yUNfw5oyZ5vkeQsZLhCb06lw==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", + "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", "cpu": [ "x64" ], @@ -569,9 +608,9 @@ ] }, "node_modules/@sveltejs/acorn-typescript": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.8.tgz", - "integrity": "sha512-esgN+54+q0NjB0Y/4BomT9samII7jGwNy/2a3wNZbT2A2RpmXsXwUt24LvLhx6jUq2gVk4cWEvcRO6MFQbOfNA==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.9.tgz", + "integrity": "sha512-lVJX6qEgs/4DOcRTpo56tmKzVPtoWAaVbL4hfO7t7NVwl9AAXzQR6cihesW1BmNMPl+bK6dreu2sOKBP2Q9CIA==", "license": "MIT", "peerDependencies": { "acorn": "^8.9.0" @@ -616,9 +655,9 @@ "license": "MIT" }, "node_modules/@urql/core": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.2.0.tgz", - "integrity": "sha512-/n0ieD0mvvDnVAXEQgX/7qJiVcvYvNkOHeBvkwtylfjydar123caCXcl58PXFY11oU1oquJocVXHxLAbtv4x1A==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@urql/core/-/core-6.0.1.tgz", + "integrity": "sha512-FZDiQk6jxbj5hixf2rEPv0jI+IZz0EqqGW8mJBEug68/zHTtT+f34guZDmyjJZyiWbj0vL165LoMr/TkeDHaug==", "license": "MIT", "dependencies": { "@0no-co/graphql.web": "^1.0.13", @@ -626,23 +665,23 @@ } }, "node_modules/@urql/svelte": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-4.2.3.tgz", - "integrity": "sha512-v3eArfymhdjaM5VQFp3QZxq9veYPadmDfX7ueid/kD4DlRplIycPakJ2FrKigh46SXa5mWqJ3QWuWyRKVu61sw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-5.0.0.tgz", + "integrity": "sha512-tHYEyFZwWsBW9GfpXbK+GImWhyZO1TJkhHsquosza0D0qOZyL+wGp/qT74WPUBJaF4gkUSXOQtUidDI7uvnuoQ==", "license": "MIT", "dependencies": { - "@urql/core": "^5.1.1", + "@urql/core": "^6.0.0", "wonka": "^6.3.2" }, "peerDependencies": { - "@urql/core": "^5.0.0", + "@urql/core": "^6.0.0", "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" } }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -652,9 +691,9 @@ } }, "node_modules/aria-query": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", - "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", "license": "Apache-2.0", "engines": { "node": ">= 0.4" @@ -839,9 +878,9 @@ } }, "node_modules/graphql": { - "version": "16.12.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.12.0.tgz", - "integrity": "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ==", + "version": "16.13.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.13.0.tgz", + "integrity": "sha512-uSisMYERbaB9bkA9M4/4dnqyktaEkf1kMHNKq/7DHyxVeWqHQ2mBmVqm5u6/FVHwF3iCNalKcg82Zfl+tffWoA==", "license": "MIT", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" @@ -915,9 +954,9 @@ } }, "node_modules/mathjs": { - "version": "15.1.0", - "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.0.tgz", - "integrity": "sha512-HfnAcScQm9drGryodlDqeS3WAl4gUTYGDcOtcqL/8s23MZ28Ib1i8XnYK3ZdjNuaW/L4BAp9lIp8vxAMrcuu1w==", + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.1.tgz", + "integrity": "sha512-rM668DTtpSzMVoh/cKAllyQVEbBApM5g//IMGD8vD7YlrIz9ITRr3SrdhjaDxcBNTdyETWwPebj2unZyHD7ZdA==", "license": "Apache-2.0", "dependencies": { "@babel/runtime": "^7.26.10", @@ -998,9 +1037,9 @@ } }, "node_modules/rollup": { - "version": "4.55.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.55.1.tgz", - "integrity": "sha512-wDv/Ht1BNHB4upNbK74s9usvl7hObDnvVzknxqY/E/O3X6rW1U1rV1aENEfJ54eFZDTNo7zv1f5N4edCluH7+A==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", + "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "devOptional": true, "license": "MIT", "dependencies": { @@ -1014,31 +1053,31 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.55.1", - "@rollup/rollup-android-arm64": "4.55.1", - "@rollup/rollup-darwin-arm64": "4.55.1", - "@rollup/rollup-darwin-x64": "4.55.1", - "@rollup/rollup-freebsd-arm64": "4.55.1", - "@rollup/rollup-freebsd-x64": "4.55.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.55.1", - "@rollup/rollup-linux-arm-musleabihf": "4.55.1", - "@rollup/rollup-linux-arm64-gnu": "4.55.1", - "@rollup/rollup-linux-arm64-musl": "4.55.1", - "@rollup/rollup-linux-loong64-gnu": "4.55.1", - "@rollup/rollup-linux-loong64-musl": "4.55.1", - "@rollup/rollup-linux-ppc64-gnu": "4.55.1", - "@rollup/rollup-linux-ppc64-musl": "4.55.1", - "@rollup/rollup-linux-riscv64-gnu": "4.55.1", - "@rollup/rollup-linux-riscv64-musl": "4.55.1", - "@rollup/rollup-linux-s390x-gnu": "4.55.1", - "@rollup/rollup-linux-x64-gnu": "4.55.1", - "@rollup/rollup-linux-x64-musl": "4.55.1", - "@rollup/rollup-openbsd-x64": "4.55.1", - "@rollup/rollup-openharmony-arm64": "4.55.1", - "@rollup/rollup-win32-arm64-msvc": "4.55.1", - "@rollup/rollup-win32-ia32-msvc": "4.55.1", - "@rollup/rollup-win32-x64-gnu": "4.55.1", - "@rollup/rollup-win32-x64-msvc": "4.55.1", + "@rollup/rollup-android-arm-eabi": "4.59.0", + "@rollup/rollup-android-arm64": "4.59.0", + "@rollup/rollup-darwin-arm64": "4.59.0", + "@rollup/rollup-darwin-x64": "4.59.0", + "@rollup/rollup-freebsd-arm64": "4.59.0", + "@rollup/rollup-freebsd-x64": "4.59.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", + "@rollup/rollup-linux-arm-musleabihf": "4.59.0", + "@rollup/rollup-linux-arm64-gnu": "4.59.0", + "@rollup/rollup-linux-arm64-musl": "4.59.0", + "@rollup/rollup-linux-loong64-gnu": "4.59.0", + "@rollup/rollup-linux-loong64-musl": "4.59.0", + "@rollup/rollup-linux-ppc64-gnu": "4.59.0", + "@rollup/rollup-linux-ppc64-musl": "4.59.0", + "@rollup/rollup-linux-riscv64-gnu": "4.59.0", + "@rollup/rollup-linux-riscv64-musl": "4.59.0", + "@rollup/rollup-linux-s390x-gnu": "4.59.0", + "@rollup/rollup-linux-x64-gnu": "4.59.0", + "@rollup/rollup-linux-x64-musl": "4.59.0", + "@rollup/rollup-openbsd-x64": "4.59.0", + "@rollup/rollup-openharmony-arm64": "4.59.0", + "@rollup/rollup-win32-arm64-msvc": "4.59.0", + "@rollup/rollup-win32-ia32-msvc": "4.59.0", + "@rollup/rollup-win32-x64-gnu": "4.59.0", + "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" } }, @@ -1141,11 +1180,14 @@ } }, "node_modules/smob": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/smob/-/smob-1.5.0.tgz", - "integrity": "sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smob/-/smob-1.6.1.tgz", + "integrity": "sha512-KAkBqZl3c2GvNgNhcoyJae1aKldDW0LO279wF9bk1PnluRTETKBq0WyzRXxEhoQLk56yHaOY4JCBEKDuJIET5g==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } }, "node_modules/source-map": { "version": "0.6.1", @@ -1182,9 +1224,9 @@ } }, "node_modules/svelte": { - "version": "5.53.2", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.2.tgz", - "integrity": "sha512-yGONuIrcl/BMmqbm6/52Q/NYzfkta7uVlos5NSzGTfNJTTFtPPzra6rAQoQIwAqupeM3s9uuTf5PvioeiCdg9g==", + "version": "5.53.6", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.6.tgz", + "integrity": "sha512-lP5DGF3oDDI9fhHcSpaBiJEkFLuS16h92DhM1L5K1lFm0WjOmUh1i2sNkBBk8rkxJRpob0dBE75jRfUzGZUOGA==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", @@ -1193,7 +1235,7 @@ "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", - "aria-query": "^5.3.1", + "aria-query": "5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.3", diff --git a/web/frontend/package.json b/web/frontend/package.json index e9bcf8a2..077126ba 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -8,23 +8,22 @@ }, "devDependencies": { "@rollup/plugin-commonjs": "^29.0.0", - "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.55.1", + "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.46.4" + "svelte": "^5.53.6" }, "dependencies": { "@rollup/plugin-replace": "^6.0.3", "@sveltestrap/sveltestrap": "^7.1.0", - "@urql/svelte": "^4.2.3", + "@urql/svelte": "^5.0.0", "chart.js": "^4.5.1", "date-fns": "^4.1.0", - "graphql": "^16.12.0", - "mathjs": "^15.1.0", + "graphql": "^16.13.0", + "mathjs": "^15.1.1", "uplot": "^1.6.32", "wonka": "^6.3.5" } From f11ff3302d22671d5e5698ef10cce941ff96a352 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 3 Mar 2026 17:47:17 +0100 Subject: [PATCH 320/341] revert urql/svelte bump, bump svelte patch version, regenerate backend --- internal/graph/generated/generated.go | 2840 ++++++++----------------- internal/graph/schema.resolvers.go | 2 +- web/frontend/package-lock.json | 26 +- web/frontend/package.json | 4 +- 4 files changed, 881 insertions(+), 1991 deletions(-) diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 136a123b..8d773222 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "strconv" - "sync" "sync/atomic" "time" @@ -25,20 +24,10 @@ import ( // NewExecutableSchema creates an ExecutableSchema from the ResolverRoot interface. func NewExecutableSchema(cfg Config) graphql.ExecutableSchema { - return &executableSchema{ - schema: cfg.Schema, - resolvers: cfg.Resolvers, - directives: cfg.Directives, - complexity: cfg.Complexity, - } + return &executableSchema{SchemaData: cfg.Schema, Resolvers: cfg.Resolvers, Directives: cfg.Directives, ComplexityRoot: cfg.Complexity} } -type Config struct { - Schema *ast.Schema - Resolvers ResolverRoot - Directives DirectiveRoot - Complexity ComplexityRoot -} +type Config = graphql.Config[ResolverRoot, DirectiveRoot, ComplexityRoot] type ResolverRoot interface { Cluster() ClusterResolver @@ -508,839 +497,834 @@ type SubClusterResolver interface { NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) } -type executableSchema struct { - schema *ast.Schema - resolvers ResolverRoot - directives DirectiveRoot - complexity ComplexityRoot -} +type executableSchema graphql.ExecutableSchemaState[ResolverRoot, DirectiveRoot, ComplexityRoot] func (e *executableSchema) Schema() *ast.Schema { - if e.schema != nil { - return e.schema + if e.SchemaData != nil { + return e.SchemaData } return parsedSchema } func (e *executableSchema) Complexity(ctx context.Context, typeName, field string, childComplexity int, rawArgs map[string]any) (int, bool) { - ec := executionContext{nil, e, 0, 0, nil} + ec := newExecutionContext(nil, e, nil) _ = ec switch typeName + "." + field { case "Accelerator.id": - if e.complexity.Accelerator.ID == nil { + if e.ComplexityRoot.Accelerator.ID == nil { break } - return e.complexity.Accelerator.ID(childComplexity), true + return e.ComplexityRoot.Accelerator.ID(childComplexity), true case "Accelerator.model": - if e.complexity.Accelerator.Model == nil { + if e.ComplexityRoot.Accelerator.Model == nil { break } - return e.complexity.Accelerator.Model(childComplexity), true + return e.ComplexityRoot.Accelerator.Model(childComplexity), true case "Accelerator.type": - if e.complexity.Accelerator.Type == nil { + if e.ComplexityRoot.Accelerator.Type == nil { break } - return e.complexity.Accelerator.Type(childComplexity), true + return e.ComplexityRoot.Accelerator.Type(childComplexity), true case "Cluster.name": - if e.complexity.Cluster.Name == nil { + if e.ComplexityRoot.Cluster.Name == nil { break } - return e.complexity.Cluster.Name(childComplexity), true + return e.ComplexityRoot.Cluster.Name(childComplexity), true case "Cluster.partitions": - if e.complexity.Cluster.Partitions == nil { + if e.ComplexityRoot.Cluster.Partitions == nil { break } - return e.complexity.Cluster.Partitions(childComplexity), true + return e.ComplexityRoot.Cluster.Partitions(childComplexity), true case "Cluster.subClusters": - if e.complexity.Cluster.SubClusters == nil { + if e.ComplexityRoot.Cluster.SubClusters == nil { break } - return e.complexity.Cluster.SubClusters(childComplexity), true + return e.ComplexityRoot.Cluster.SubClusters(childComplexity), true case "ClusterMetricWithName.data": - if e.complexity.ClusterMetricWithName.Data == nil { + if e.ComplexityRoot.ClusterMetricWithName.Data == nil { break } - return e.complexity.ClusterMetricWithName.Data(childComplexity), true + return e.ComplexityRoot.ClusterMetricWithName.Data(childComplexity), true case "ClusterMetricWithName.name": - if e.complexity.ClusterMetricWithName.Name == nil { + if e.ComplexityRoot.ClusterMetricWithName.Name == nil { break } - return e.complexity.ClusterMetricWithName.Name(childComplexity), true + return e.ComplexityRoot.ClusterMetricWithName.Name(childComplexity), true case "ClusterMetricWithName.timestep": - if e.complexity.ClusterMetricWithName.Timestep == nil { + if e.ComplexityRoot.ClusterMetricWithName.Timestep == nil { break } - return e.complexity.ClusterMetricWithName.Timestep(childComplexity), true + return e.ComplexityRoot.ClusterMetricWithName.Timestep(childComplexity), true case "ClusterMetricWithName.unit": - if e.complexity.ClusterMetricWithName.Unit == nil { + if e.ComplexityRoot.ClusterMetricWithName.Unit == nil { break } - return e.complexity.ClusterMetricWithName.Unit(childComplexity), true + return e.ComplexityRoot.ClusterMetricWithName.Unit(childComplexity), true case "ClusterMetrics.metrics": - if e.complexity.ClusterMetrics.Metrics == nil { + if e.ComplexityRoot.ClusterMetrics.Metrics == nil { break } - return e.complexity.ClusterMetrics.Metrics(childComplexity), true + return e.ComplexityRoot.ClusterMetrics.Metrics(childComplexity), true case "ClusterMetrics.nodeCount": - if e.complexity.ClusterMetrics.NodeCount == nil { + if e.ComplexityRoot.ClusterMetrics.NodeCount == nil { break } - return e.complexity.ClusterMetrics.NodeCount(childComplexity), true + return e.ComplexityRoot.ClusterMetrics.NodeCount(childComplexity), true case "ClusterSupport.cluster": - if e.complexity.ClusterSupport.Cluster == nil { + if e.ComplexityRoot.ClusterSupport.Cluster == nil { break } - return e.complexity.ClusterSupport.Cluster(childComplexity), true + return e.ComplexityRoot.ClusterSupport.Cluster(childComplexity), true case "ClusterSupport.subClusters": - if e.complexity.ClusterSupport.SubClusters == nil { + if e.ComplexityRoot.ClusterSupport.SubClusters == nil { break } - return e.complexity.ClusterSupport.SubClusters(childComplexity), true + return e.ComplexityRoot.ClusterSupport.SubClusters(childComplexity), true case "Count.count": - if e.complexity.Count.Count == nil { + if e.ComplexityRoot.Count.Count == nil { break } - return e.complexity.Count.Count(childComplexity), true + return e.ComplexityRoot.Count.Count(childComplexity), true case "Count.name": - if e.complexity.Count.Name == nil { + if e.ComplexityRoot.Count.Name == nil { break } - return e.complexity.Count.Name(childComplexity), true + return e.ComplexityRoot.Count.Name(childComplexity), true case "EnergyFootprintValue.hardware": - if e.complexity.EnergyFootprintValue.Hardware == nil { + if e.ComplexityRoot.EnergyFootprintValue.Hardware == nil { break } - return e.complexity.EnergyFootprintValue.Hardware(childComplexity), true + return e.ComplexityRoot.EnergyFootprintValue.Hardware(childComplexity), true case "EnergyFootprintValue.metric": - if e.complexity.EnergyFootprintValue.Metric == nil { + if e.ComplexityRoot.EnergyFootprintValue.Metric == nil { break } - return e.complexity.EnergyFootprintValue.Metric(childComplexity), true + return e.ComplexityRoot.EnergyFootprintValue.Metric(childComplexity), true case "EnergyFootprintValue.value": - if e.complexity.EnergyFootprintValue.Value == nil { + if e.ComplexityRoot.EnergyFootprintValue.Value == nil { break } - return e.complexity.EnergyFootprintValue.Value(childComplexity), true + return e.ComplexityRoot.EnergyFootprintValue.Value(childComplexity), true case "FootprintValue.name": - if e.complexity.FootprintValue.Name == nil { + if e.ComplexityRoot.FootprintValue.Name == nil { break } - return e.complexity.FootprintValue.Name(childComplexity), true + return e.ComplexityRoot.FootprintValue.Name(childComplexity), true case "FootprintValue.stat": - if e.complexity.FootprintValue.Stat == nil { + if e.ComplexityRoot.FootprintValue.Stat == nil { break } - return e.complexity.FootprintValue.Stat(childComplexity), true + return e.ComplexityRoot.FootprintValue.Stat(childComplexity), true case "FootprintValue.value": - if e.complexity.FootprintValue.Value == nil { + if e.ComplexityRoot.FootprintValue.Value == nil { break } - return e.complexity.FootprintValue.Value(childComplexity), true + return e.ComplexityRoot.FootprintValue.Value(childComplexity), true case "Footprints.metrics": - if e.complexity.Footprints.Metrics == nil { + if e.ComplexityRoot.Footprints.Metrics == nil { break } - return e.complexity.Footprints.Metrics(childComplexity), true + return e.ComplexityRoot.Footprints.Metrics(childComplexity), true case "Footprints.timeWeights": - if e.complexity.Footprints.TimeWeights == nil { + if e.ComplexityRoot.Footprints.TimeWeights == nil { break } - return e.complexity.Footprints.TimeWeights(childComplexity), true + return e.ComplexityRoot.Footprints.TimeWeights(childComplexity), true case "GlobalMetricListItem.availability": - if e.complexity.GlobalMetricListItem.Availability == nil { + if e.ComplexityRoot.GlobalMetricListItem.Availability == nil { break } - return e.complexity.GlobalMetricListItem.Availability(childComplexity), true + return e.ComplexityRoot.GlobalMetricListItem.Availability(childComplexity), true case "GlobalMetricListItem.footprint": - if e.complexity.GlobalMetricListItem.Footprint == nil { + if e.ComplexityRoot.GlobalMetricListItem.Footprint == nil { break } - return e.complexity.GlobalMetricListItem.Footprint(childComplexity), true + return e.ComplexityRoot.GlobalMetricListItem.Footprint(childComplexity), true case "GlobalMetricListItem.name": - if e.complexity.GlobalMetricListItem.Name == nil { + if e.ComplexityRoot.GlobalMetricListItem.Name == nil { break } - return e.complexity.GlobalMetricListItem.Name(childComplexity), true + return e.ComplexityRoot.GlobalMetricListItem.Name(childComplexity), true case "GlobalMetricListItem.scope": - if e.complexity.GlobalMetricListItem.Scope == nil { + if e.ComplexityRoot.GlobalMetricListItem.Scope == nil { break } - return e.complexity.GlobalMetricListItem.Scope(childComplexity), true + return e.ComplexityRoot.GlobalMetricListItem.Scope(childComplexity), true case "GlobalMetricListItem.unit": - if e.complexity.GlobalMetricListItem.Unit == nil { + if e.ComplexityRoot.GlobalMetricListItem.Unit == nil { break } - return e.complexity.GlobalMetricListItem.Unit(childComplexity), true + return e.ComplexityRoot.GlobalMetricListItem.Unit(childComplexity), true case "HistoPoint.count": - if e.complexity.HistoPoint.Count == nil { + if e.ComplexityRoot.HistoPoint.Count == nil { break } - return e.complexity.HistoPoint.Count(childComplexity), true + return e.ComplexityRoot.HistoPoint.Count(childComplexity), true case "HistoPoint.value": - if e.complexity.HistoPoint.Value == nil { + if e.ComplexityRoot.HistoPoint.Value == nil { break } - return e.complexity.HistoPoint.Value(childComplexity), true + return e.ComplexityRoot.HistoPoint.Value(childComplexity), true case "IntRangeOutput.from": - if e.complexity.IntRangeOutput.From == nil { + if e.ComplexityRoot.IntRangeOutput.From == nil { break } - return e.complexity.IntRangeOutput.From(childComplexity), true + return e.ComplexityRoot.IntRangeOutput.From(childComplexity), true case "IntRangeOutput.to": - if e.complexity.IntRangeOutput.To == nil { + if e.ComplexityRoot.IntRangeOutput.To == nil { break } - return e.complexity.IntRangeOutput.To(childComplexity), true + return e.ComplexityRoot.IntRangeOutput.To(childComplexity), true case "Job.arrayJobId": - if e.complexity.Job.ArrayJobID == nil { + if e.ComplexityRoot.Job.ArrayJobID == nil { break } - return e.complexity.Job.ArrayJobID(childComplexity), true + return e.ComplexityRoot.Job.ArrayJobID(childComplexity), true case "Job.cluster": - if e.complexity.Job.Cluster == nil { + if e.ComplexityRoot.Job.Cluster == nil { break } - return e.complexity.Job.Cluster(childComplexity), true + return e.ComplexityRoot.Job.Cluster(childComplexity), true case "Job.concurrentJobs": - if e.complexity.Job.ConcurrentJobs == nil { + if e.ComplexityRoot.Job.ConcurrentJobs == nil { break } - return e.complexity.Job.ConcurrentJobs(childComplexity), true + return e.ComplexityRoot.Job.ConcurrentJobs(childComplexity), true case "Job.duration": - if e.complexity.Job.Duration == nil { + if e.ComplexityRoot.Job.Duration == nil { break } - return e.complexity.Job.Duration(childComplexity), true + return e.ComplexityRoot.Job.Duration(childComplexity), true case "Job.energy": - if e.complexity.Job.Energy == nil { + if e.ComplexityRoot.Job.Energy == nil { break } - return e.complexity.Job.Energy(childComplexity), true + return e.ComplexityRoot.Job.Energy(childComplexity), true case "Job.energyFootprint": - if e.complexity.Job.EnergyFootprint == nil { + if e.ComplexityRoot.Job.EnergyFootprint == nil { break } - return e.complexity.Job.EnergyFootprint(childComplexity), true + return e.ComplexityRoot.Job.EnergyFootprint(childComplexity), true case "Job.footprint": - if e.complexity.Job.Footprint == nil { + if e.ComplexityRoot.Job.Footprint == nil { break } - return e.complexity.Job.Footprint(childComplexity), true + return e.ComplexityRoot.Job.Footprint(childComplexity), true case "Job.id": - if e.complexity.Job.ID == nil { + if e.ComplexityRoot.Job.ID == nil { break } - return e.complexity.Job.ID(childComplexity), true + return e.ComplexityRoot.Job.ID(childComplexity), true case "Job.jobId": - if e.complexity.Job.JobID == nil { + if e.ComplexityRoot.Job.JobID == nil { break } - return e.complexity.Job.JobID(childComplexity), true + return e.ComplexityRoot.Job.JobID(childComplexity), true case "Job.metaData": - if e.complexity.Job.MetaData == nil { + if e.ComplexityRoot.Job.MetaData == nil { break } - return e.complexity.Job.MetaData(childComplexity), true + return e.ComplexityRoot.Job.MetaData(childComplexity), true case "Job.monitoringStatus": - if e.complexity.Job.MonitoringStatus == nil { + if e.ComplexityRoot.Job.MonitoringStatus == nil { break } - return e.complexity.Job.MonitoringStatus(childComplexity), true + return e.ComplexityRoot.Job.MonitoringStatus(childComplexity), true case "Job.numAcc": - if e.complexity.Job.NumAcc == nil { + if e.ComplexityRoot.Job.NumAcc == nil { break } - return e.complexity.Job.NumAcc(childComplexity), true + return e.ComplexityRoot.Job.NumAcc(childComplexity), true case "Job.numHWThreads": - if e.complexity.Job.NumHWThreads == nil { + if e.ComplexityRoot.Job.NumHWThreads == nil { break } - return e.complexity.Job.NumHWThreads(childComplexity), true + return e.ComplexityRoot.Job.NumHWThreads(childComplexity), true case "Job.numNodes": - if e.complexity.Job.NumNodes == nil { + if e.ComplexityRoot.Job.NumNodes == nil { break } - return e.complexity.Job.NumNodes(childComplexity), true + return e.ComplexityRoot.Job.NumNodes(childComplexity), true case "Job.partition": - if e.complexity.Job.Partition == nil { + if e.ComplexityRoot.Job.Partition == nil { break } - return e.complexity.Job.Partition(childComplexity), true + return e.ComplexityRoot.Job.Partition(childComplexity), true case "Job.project": - if e.complexity.Job.Project == nil { + if e.ComplexityRoot.Job.Project == nil { break } - return e.complexity.Job.Project(childComplexity), true + return e.ComplexityRoot.Job.Project(childComplexity), true case "Job.resources": - if e.complexity.Job.Resources == nil { + if e.ComplexityRoot.Job.Resources == nil { break } - return e.complexity.Job.Resources(childComplexity), true + return e.ComplexityRoot.Job.Resources(childComplexity), true case "Job.SMT": - if e.complexity.Job.SMT == nil { + if e.ComplexityRoot.Job.SMT == nil { break } - return e.complexity.Job.SMT(childComplexity), true + return e.ComplexityRoot.Job.SMT(childComplexity), true case "Job.shared": - if e.complexity.Job.Shared == nil { + if e.ComplexityRoot.Job.Shared == nil { break } - return e.complexity.Job.Shared(childComplexity), true + return e.ComplexityRoot.Job.Shared(childComplexity), true case "Job.startTime": - if e.complexity.Job.StartTime == nil { + if e.ComplexityRoot.Job.StartTime == nil { break } - return e.complexity.Job.StartTime(childComplexity), true + return e.ComplexityRoot.Job.StartTime(childComplexity), true case "Job.state": - if e.complexity.Job.State == nil { + if e.ComplexityRoot.Job.State == nil { break } - return e.complexity.Job.State(childComplexity), true + return e.ComplexityRoot.Job.State(childComplexity), true case "Job.subCluster": - if e.complexity.Job.SubCluster == nil { + if e.ComplexityRoot.Job.SubCluster == nil { break } - return e.complexity.Job.SubCluster(childComplexity), true + return e.ComplexityRoot.Job.SubCluster(childComplexity), true case "Job.tags": - if e.complexity.Job.Tags == nil { + if e.ComplexityRoot.Job.Tags == nil { break } - return e.complexity.Job.Tags(childComplexity), true + return e.ComplexityRoot.Job.Tags(childComplexity), true case "Job.user": - if e.complexity.Job.User == nil { + if e.ComplexityRoot.Job.User == nil { break } - return e.complexity.Job.User(childComplexity), true + return e.ComplexityRoot.Job.User(childComplexity), true case "Job.userData": - if e.complexity.Job.UserData == nil { + if e.ComplexityRoot.Job.UserData == nil { break } - return e.complexity.Job.UserData(childComplexity), true + return e.ComplexityRoot.Job.UserData(childComplexity), true case "Job.walltime": - if e.complexity.Job.Walltime == nil { + if e.ComplexityRoot.Job.Walltime == nil { break } - return e.complexity.Job.Walltime(childComplexity), true + return e.ComplexityRoot.Job.Walltime(childComplexity), true case "JobLink.id": - if e.complexity.JobLink.ID == nil { + if e.ComplexityRoot.JobLink.ID == nil { break } - return e.complexity.JobLink.ID(childComplexity), true + return e.ComplexityRoot.JobLink.ID(childComplexity), true case "JobLink.jobId": - if e.complexity.JobLink.JobID == nil { + if e.ComplexityRoot.JobLink.JobID == nil { break } - return e.complexity.JobLink.JobID(childComplexity), true + return e.ComplexityRoot.JobLink.JobID(childComplexity), true case "JobLinkResultList.count": - if e.complexity.JobLinkResultList.Count == nil { + if e.ComplexityRoot.JobLinkResultList.Count == nil { break } - return e.complexity.JobLinkResultList.Count(childComplexity), true + return e.ComplexityRoot.JobLinkResultList.Count(childComplexity), true case "JobLinkResultList.items": - if e.complexity.JobLinkResultList.Items == nil { + if e.ComplexityRoot.JobLinkResultList.Items == nil { break } - return e.complexity.JobLinkResultList.Items(childComplexity), true + return e.ComplexityRoot.JobLinkResultList.Items(childComplexity), true case "JobLinkResultList.listQuery": - if e.complexity.JobLinkResultList.ListQuery == nil { + if e.ComplexityRoot.JobLinkResultList.ListQuery == nil { break } - return e.complexity.JobLinkResultList.ListQuery(childComplexity), true + return e.ComplexityRoot.JobLinkResultList.ListQuery(childComplexity), true case "JobMetric.series": - if e.complexity.JobMetric.Series == nil { + if e.ComplexityRoot.JobMetric.Series == nil { break } - return e.complexity.JobMetric.Series(childComplexity), true + return e.ComplexityRoot.JobMetric.Series(childComplexity), true case "JobMetric.statisticsSeries": - if e.complexity.JobMetric.StatisticsSeries == nil { + if e.ComplexityRoot.JobMetric.StatisticsSeries == nil { break } - return e.complexity.JobMetric.StatisticsSeries(childComplexity), true + return e.ComplexityRoot.JobMetric.StatisticsSeries(childComplexity), true case "JobMetric.timestep": - if e.complexity.JobMetric.Timestep == nil { + if e.ComplexityRoot.JobMetric.Timestep == nil { break } - return e.complexity.JobMetric.Timestep(childComplexity), true + return e.ComplexityRoot.JobMetric.Timestep(childComplexity), true case "JobMetric.unit": - if e.complexity.JobMetric.Unit == nil { + if e.ComplexityRoot.JobMetric.Unit == nil { break } - return e.complexity.JobMetric.Unit(childComplexity), true + return e.ComplexityRoot.JobMetric.Unit(childComplexity), true case "JobMetricWithName.metric": - if e.complexity.JobMetricWithName.Metric == nil { + if e.ComplexityRoot.JobMetricWithName.Metric == nil { break } - return e.complexity.JobMetricWithName.Metric(childComplexity), true + return e.ComplexityRoot.JobMetricWithName.Metric(childComplexity), true case "JobMetricWithName.name": - if e.complexity.JobMetricWithName.Name == nil { + if e.ComplexityRoot.JobMetricWithName.Name == nil { break } - return e.complexity.JobMetricWithName.Name(childComplexity), true + return e.ComplexityRoot.JobMetricWithName.Name(childComplexity), true case "JobMetricWithName.scope": - if e.complexity.JobMetricWithName.Scope == nil { + if e.ComplexityRoot.JobMetricWithName.Scope == nil { break } - return e.complexity.JobMetricWithName.Scope(childComplexity), true + return e.ComplexityRoot.JobMetricWithName.Scope(childComplexity), true case "JobResultList.count": - if e.complexity.JobResultList.Count == nil { + if e.ComplexityRoot.JobResultList.Count == nil { break } - return e.complexity.JobResultList.Count(childComplexity), true + return e.ComplexityRoot.JobResultList.Count(childComplexity), true case "JobResultList.hasNextPage": - if e.complexity.JobResultList.HasNextPage == nil { + if e.ComplexityRoot.JobResultList.HasNextPage == nil { break } - return e.complexity.JobResultList.HasNextPage(childComplexity), true + return e.ComplexityRoot.JobResultList.HasNextPage(childComplexity), true case "JobResultList.items": - if e.complexity.JobResultList.Items == nil { + if e.ComplexityRoot.JobResultList.Items == nil { break } - return e.complexity.JobResultList.Items(childComplexity), true + return e.ComplexityRoot.JobResultList.Items(childComplexity), true case "JobResultList.limit": - if e.complexity.JobResultList.Limit == nil { + if e.ComplexityRoot.JobResultList.Limit == nil { break } - return e.complexity.JobResultList.Limit(childComplexity), true + return e.ComplexityRoot.JobResultList.Limit(childComplexity), true case "JobResultList.offset": - if e.complexity.JobResultList.Offset == nil { + if e.ComplexityRoot.JobResultList.Offset == nil { break } - return e.complexity.JobResultList.Offset(childComplexity), true + return e.ComplexityRoot.JobResultList.Offset(childComplexity), true case "JobStats.cluster": - if e.complexity.JobStats.Cluster == nil { + if e.ComplexityRoot.JobStats.Cluster == nil { break } - return e.complexity.JobStats.Cluster(childComplexity), true + return e.ComplexityRoot.JobStats.Cluster(childComplexity), true case "JobStats.duration": - if e.complexity.JobStats.Duration == nil { + if e.ComplexityRoot.JobStats.Duration == nil { break } - return e.complexity.JobStats.Duration(childComplexity), true + return e.ComplexityRoot.JobStats.Duration(childComplexity), true case "JobStats.id": - if e.complexity.JobStats.ID == nil { + if e.ComplexityRoot.JobStats.ID == nil { break } - return e.complexity.JobStats.ID(childComplexity), true + return e.ComplexityRoot.JobStats.ID(childComplexity), true case "JobStats.jobId": - if e.complexity.JobStats.JobID == nil { + if e.ComplexityRoot.JobStats.JobID == nil { break } - return e.complexity.JobStats.JobID(childComplexity), true + return e.ComplexityRoot.JobStats.JobID(childComplexity), true case "JobStats.numAccelerators": - if e.complexity.JobStats.NumAccelerators == nil { + if e.ComplexityRoot.JobStats.NumAccelerators == nil { break } - return e.complexity.JobStats.NumAccelerators(childComplexity), true + return e.ComplexityRoot.JobStats.NumAccelerators(childComplexity), true case "JobStats.numHWThreads": - if e.complexity.JobStats.NumHWThreads == nil { + if e.ComplexityRoot.JobStats.NumHWThreads == nil { break } - return e.complexity.JobStats.NumHWThreads(childComplexity), true + return e.ComplexityRoot.JobStats.NumHWThreads(childComplexity), true case "JobStats.numNodes": - if e.complexity.JobStats.NumNodes == nil { + if e.ComplexityRoot.JobStats.NumNodes == nil { break } - return e.complexity.JobStats.NumNodes(childComplexity), true + return e.ComplexityRoot.JobStats.NumNodes(childComplexity), true case "JobStats.startTime": - if e.complexity.JobStats.StartTime == nil { + if e.ComplexityRoot.JobStats.StartTime == nil { break } - return e.complexity.JobStats.StartTime(childComplexity), true + return e.ComplexityRoot.JobStats.StartTime(childComplexity), true case "JobStats.stats": - if e.complexity.JobStats.Stats == nil { + if e.ComplexityRoot.JobStats.Stats == nil { break } - return e.complexity.JobStats.Stats(childComplexity), true + return e.ComplexityRoot.JobStats.Stats(childComplexity), true case "JobStats.subCluster": - if e.complexity.JobStats.SubCluster == nil { + if e.ComplexityRoot.JobStats.SubCluster == nil { break } - return e.complexity.JobStats.SubCluster(childComplexity), true + return e.ComplexityRoot.JobStats.SubCluster(childComplexity), true case "JobsStatistics.histDuration": - if e.complexity.JobsStatistics.HistDuration == nil { + if e.ComplexityRoot.JobsStatistics.HistDuration == nil { break } - return e.complexity.JobsStatistics.HistDuration(childComplexity), true + return e.ComplexityRoot.JobsStatistics.HistDuration(childComplexity), true case "JobsStatistics.histMetrics": - if e.complexity.JobsStatistics.HistMetrics == nil { + if e.ComplexityRoot.JobsStatistics.HistMetrics == nil { break } - return e.complexity.JobsStatistics.HistMetrics(childComplexity), true + return e.ComplexityRoot.JobsStatistics.HistMetrics(childComplexity), true case "JobsStatistics.histNumAccs": - if e.complexity.JobsStatistics.HistNumAccs == nil { + if e.ComplexityRoot.JobsStatistics.HistNumAccs == nil { break } - return e.complexity.JobsStatistics.HistNumAccs(childComplexity), true + return e.ComplexityRoot.JobsStatistics.HistNumAccs(childComplexity), true case "JobsStatistics.histNumCores": - if e.complexity.JobsStatistics.HistNumCores == nil { + if e.ComplexityRoot.JobsStatistics.HistNumCores == nil { break } - return e.complexity.JobsStatistics.HistNumCores(childComplexity), true + return e.ComplexityRoot.JobsStatistics.HistNumCores(childComplexity), true case "JobsStatistics.histNumNodes": - if e.complexity.JobsStatistics.HistNumNodes == nil { + if e.ComplexityRoot.JobsStatistics.HistNumNodes == nil { break } - return e.complexity.JobsStatistics.HistNumNodes(childComplexity), true + return e.ComplexityRoot.JobsStatistics.HistNumNodes(childComplexity), true case "JobsStatistics.id": - if e.complexity.JobsStatistics.ID == nil { + if e.ComplexityRoot.JobsStatistics.ID == nil { break } - return e.complexity.JobsStatistics.ID(childComplexity), true + return e.ComplexityRoot.JobsStatistics.ID(childComplexity), true case "JobsStatistics.name": - if e.complexity.JobsStatistics.Name == nil { + if e.ComplexityRoot.JobsStatistics.Name == nil { break } - return e.complexity.JobsStatistics.Name(childComplexity), true + return e.ComplexityRoot.JobsStatistics.Name(childComplexity), true case "JobsStatistics.runningJobs": - if e.complexity.JobsStatistics.RunningJobs == nil { + if e.ComplexityRoot.JobsStatistics.RunningJobs == nil { break } - return e.complexity.JobsStatistics.RunningJobs(childComplexity), true + return e.ComplexityRoot.JobsStatistics.RunningJobs(childComplexity), true case "JobsStatistics.shortJobs": - if e.complexity.JobsStatistics.ShortJobs == nil { + if e.ComplexityRoot.JobsStatistics.ShortJobs == nil { break } - return e.complexity.JobsStatistics.ShortJobs(childComplexity), true + return e.ComplexityRoot.JobsStatistics.ShortJobs(childComplexity), true case "JobsStatistics.totalAccHours": - if e.complexity.JobsStatistics.TotalAccHours == nil { + if e.ComplexityRoot.JobsStatistics.TotalAccHours == nil { break } - return e.complexity.JobsStatistics.TotalAccHours(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalAccHours(childComplexity), true case "JobsStatistics.totalAccs": - if e.complexity.JobsStatistics.TotalAccs == nil { + if e.ComplexityRoot.JobsStatistics.TotalAccs == nil { break } - return e.complexity.JobsStatistics.TotalAccs(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalAccs(childComplexity), true case "JobsStatistics.totalCoreHours": - if e.complexity.JobsStatistics.TotalCoreHours == nil { + if e.ComplexityRoot.JobsStatistics.TotalCoreHours == nil { break } - return e.complexity.JobsStatistics.TotalCoreHours(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalCoreHours(childComplexity), true case "JobsStatistics.totalCores": - if e.complexity.JobsStatistics.TotalCores == nil { + if e.ComplexityRoot.JobsStatistics.TotalCores == nil { break } - return e.complexity.JobsStatistics.TotalCores(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalCores(childComplexity), true case "JobsStatistics.totalJobs": - if e.complexity.JobsStatistics.TotalJobs == nil { + if e.ComplexityRoot.JobsStatistics.TotalJobs == nil { break } - return e.complexity.JobsStatistics.TotalJobs(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalJobs(childComplexity), true case "JobsStatistics.totalNodeHours": - if e.complexity.JobsStatistics.TotalNodeHours == nil { + if e.ComplexityRoot.JobsStatistics.TotalNodeHours == nil { break } - return e.complexity.JobsStatistics.TotalNodeHours(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalNodeHours(childComplexity), true case "JobsStatistics.totalNodes": - if e.complexity.JobsStatistics.TotalNodes == nil { + if e.ComplexityRoot.JobsStatistics.TotalNodes == nil { break } - return e.complexity.JobsStatistics.TotalNodes(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalNodes(childComplexity), true case "JobsStatistics.totalUsers": - if e.complexity.JobsStatistics.TotalUsers == nil { + if e.ComplexityRoot.JobsStatistics.TotalUsers == nil { break } - return e.complexity.JobsStatistics.TotalUsers(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalUsers(childComplexity), true case "JobsStatistics.totalWalltime": - if e.complexity.JobsStatistics.TotalWalltime == nil { + if e.ComplexityRoot.JobsStatistics.TotalWalltime == nil { break } - return e.complexity.JobsStatistics.TotalWalltime(childComplexity), true + return e.ComplexityRoot.JobsStatistics.TotalWalltime(childComplexity), true case "MetricConfig.aggregation": - if e.complexity.MetricConfig.Aggregation == nil { + if e.ComplexityRoot.MetricConfig.Aggregation == nil { break } - return e.complexity.MetricConfig.Aggregation(childComplexity), true + return e.ComplexityRoot.MetricConfig.Aggregation(childComplexity), true case "MetricConfig.alert": - if e.complexity.MetricConfig.Alert == nil { + if e.ComplexityRoot.MetricConfig.Alert == nil { break } - return e.complexity.MetricConfig.Alert(childComplexity), true + return e.ComplexityRoot.MetricConfig.Alert(childComplexity), true case "MetricConfig.caution": - if e.complexity.MetricConfig.Caution == nil { + if e.ComplexityRoot.MetricConfig.Caution == nil { break } - return e.complexity.MetricConfig.Caution(childComplexity), true + return e.ComplexityRoot.MetricConfig.Caution(childComplexity), true case "MetricConfig.lowerIsBetter": - if e.complexity.MetricConfig.LowerIsBetter == nil { + if e.ComplexityRoot.MetricConfig.LowerIsBetter == nil { break } - return e.complexity.MetricConfig.LowerIsBetter(childComplexity), true + return e.ComplexityRoot.MetricConfig.LowerIsBetter(childComplexity), true case "MetricConfig.name": - if e.complexity.MetricConfig.Name == nil { + if e.ComplexityRoot.MetricConfig.Name == nil { break } - return e.complexity.MetricConfig.Name(childComplexity), true + return e.ComplexityRoot.MetricConfig.Name(childComplexity), true case "MetricConfig.normal": - if e.complexity.MetricConfig.Normal == nil { + if e.ComplexityRoot.MetricConfig.Normal == nil { break } - return e.complexity.MetricConfig.Normal(childComplexity), true + return e.ComplexityRoot.MetricConfig.Normal(childComplexity), true case "MetricConfig.peak": - if e.complexity.MetricConfig.Peak == nil { + if e.ComplexityRoot.MetricConfig.Peak == nil { break } - return e.complexity.MetricConfig.Peak(childComplexity), true + return e.ComplexityRoot.MetricConfig.Peak(childComplexity), true case "MetricConfig.scope": - if e.complexity.MetricConfig.Scope == nil { + if e.ComplexityRoot.MetricConfig.Scope == nil { break } - return e.complexity.MetricConfig.Scope(childComplexity), true + return e.ComplexityRoot.MetricConfig.Scope(childComplexity), true case "MetricConfig.subClusters": - if e.complexity.MetricConfig.SubClusters == nil { + if e.ComplexityRoot.MetricConfig.SubClusters == nil { break } - return e.complexity.MetricConfig.SubClusters(childComplexity), true + return e.ComplexityRoot.MetricConfig.SubClusters(childComplexity), true case "MetricConfig.timestep": - if e.complexity.MetricConfig.Timestep == nil { + if e.ComplexityRoot.MetricConfig.Timestep == nil { break } - return e.complexity.MetricConfig.Timestep(childComplexity), true + return e.ComplexityRoot.MetricConfig.Timestep(childComplexity), true case "MetricConfig.unit": - if e.complexity.MetricConfig.Unit == nil { + if e.ComplexityRoot.MetricConfig.Unit == nil { break } - return e.complexity.MetricConfig.Unit(childComplexity), true + return e.ComplexityRoot.MetricConfig.Unit(childComplexity), true case "MetricFootprints.data": - if e.complexity.MetricFootprints.Data == nil { + if e.ComplexityRoot.MetricFootprints.Data == nil { break } - return e.complexity.MetricFootprints.Data(childComplexity), true + return e.ComplexityRoot.MetricFootprints.Data(childComplexity), true case "MetricFootprints.metric": - if e.complexity.MetricFootprints.Metric == nil { + if e.ComplexityRoot.MetricFootprints.Metric == nil { break } - return e.complexity.MetricFootprints.Metric(childComplexity), true + return e.ComplexityRoot.MetricFootprints.Metric(childComplexity), true case "MetricHistoPoint.bin": - if e.complexity.MetricHistoPoint.Bin == nil { + if e.ComplexityRoot.MetricHistoPoint.Bin == nil { break } - return e.complexity.MetricHistoPoint.Bin(childComplexity), true + return e.ComplexityRoot.MetricHistoPoint.Bin(childComplexity), true case "MetricHistoPoint.count": - if e.complexity.MetricHistoPoint.Count == nil { + if e.ComplexityRoot.MetricHistoPoint.Count == nil { break } - return e.complexity.MetricHistoPoint.Count(childComplexity), true + return e.ComplexityRoot.MetricHistoPoint.Count(childComplexity), true case "MetricHistoPoint.max": - if e.complexity.MetricHistoPoint.Max == nil { + if e.ComplexityRoot.MetricHistoPoint.Max == nil { break } - return e.complexity.MetricHistoPoint.Max(childComplexity), true + return e.ComplexityRoot.MetricHistoPoint.Max(childComplexity), true case "MetricHistoPoint.min": - if e.complexity.MetricHistoPoint.Min == nil { + if e.ComplexityRoot.MetricHistoPoint.Min == nil { break } - return e.complexity.MetricHistoPoint.Min(childComplexity), true + return e.ComplexityRoot.MetricHistoPoint.Min(childComplexity), true case "MetricHistoPoints.data": - if e.complexity.MetricHistoPoints.Data == nil { + if e.ComplexityRoot.MetricHistoPoints.Data == nil { break } - return e.complexity.MetricHistoPoints.Data(childComplexity), true + return e.ComplexityRoot.MetricHistoPoints.Data(childComplexity), true case "MetricHistoPoints.metric": - if e.complexity.MetricHistoPoints.Metric == nil { + if e.ComplexityRoot.MetricHistoPoints.Metric == nil { break } - return e.complexity.MetricHistoPoints.Metric(childComplexity), true + return e.ComplexityRoot.MetricHistoPoints.Metric(childComplexity), true case "MetricHistoPoints.stat": - if e.complexity.MetricHistoPoints.Stat == nil { + if e.ComplexityRoot.MetricHistoPoints.Stat == nil { break } - return e.complexity.MetricHistoPoints.Stat(childComplexity), true + return e.ComplexityRoot.MetricHistoPoints.Stat(childComplexity), true case "MetricHistoPoints.unit": - if e.complexity.MetricHistoPoints.Unit == nil { + if e.ComplexityRoot.MetricHistoPoints.Unit == nil { break } - return e.complexity.MetricHistoPoints.Unit(childComplexity), true + return e.ComplexityRoot.MetricHistoPoints.Unit(childComplexity), true case "MetricStatistics.avg": - if e.complexity.MetricStatistics.Avg == nil { + if e.ComplexityRoot.MetricStatistics.Avg == nil { break } - return e.complexity.MetricStatistics.Avg(childComplexity), true + return e.ComplexityRoot.MetricStatistics.Avg(childComplexity), true case "MetricStatistics.max": - if e.complexity.MetricStatistics.Max == nil { + if e.ComplexityRoot.MetricStatistics.Max == nil { break } - return e.complexity.MetricStatistics.Max(childComplexity), true + return e.ComplexityRoot.MetricStatistics.Max(childComplexity), true case "MetricStatistics.min": - if e.complexity.MetricStatistics.Min == nil { + if e.ComplexityRoot.MetricStatistics.Min == nil { break } - return e.complexity.MetricStatistics.Min(childComplexity), true + return e.ComplexityRoot.MetricStatistics.Min(childComplexity), true case "MetricValue.name": - if e.complexity.MetricValue.Name == nil { + if e.ComplexityRoot.MetricValue.Name == nil { break } - return e.complexity.MetricValue.Name(childComplexity), true + return e.ComplexityRoot.MetricValue.Name(childComplexity), true case "MetricValue.unit": - if e.complexity.MetricValue.Unit == nil { + if e.ComplexityRoot.MetricValue.Unit == nil { break } - return e.complexity.MetricValue.Unit(childComplexity), true + return e.ComplexityRoot.MetricValue.Unit(childComplexity), true case "MetricValue.value": - if e.complexity.MetricValue.Value == nil { + if e.ComplexityRoot.MetricValue.Value == nil { break } - return e.complexity.MetricValue.Value(childComplexity), true + return e.ComplexityRoot.MetricValue.Value(childComplexity), true case "Mutation.addTagsToJob": - if e.complexity.Mutation.AddTagsToJob == nil { + if e.ComplexityRoot.Mutation.AddTagsToJob == nil { break } @@ -1349,9 +1333,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.AddTagsToJob(childComplexity, args["job"].(string), args["tagIds"].([]string)), true + return e.ComplexityRoot.Mutation.AddTagsToJob(childComplexity, args["job"].(string), args["tagIds"].([]string)), true case "Mutation.createTag": - if e.complexity.Mutation.CreateTag == nil { + if e.ComplexityRoot.Mutation.CreateTag == nil { break } @@ -1360,9 +1344,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string), args["scope"].(string)), true + return e.ComplexityRoot.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string), args["scope"].(string)), true case "Mutation.deleteTag": - if e.complexity.Mutation.DeleteTag == nil { + if e.ComplexityRoot.Mutation.DeleteTag == nil { break } @@ -1371,9 +1355,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.DeleteTag(childComplexity, args["id"].(string)), true + return e.ComplexityRoot.Mutation.DeleteTag(childComplexity, args["id"].(string)), true case "Mutation.removeTagFromList": - if e.complexity.Mutation.RemoveTagFromList == nil { + if e.ComplexityRoot.Mutation.RemoveTagFromList == nil { break } @@ -1382,9 +1366,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.RemoveTagFromList(childComplexity, args["tagIds"].([]string)), true + return e.ComplexityRoot.Mutation.RemoveTagFromList(childComplexity, args["tagIds"].([]string)), true case "Mutation.removeTagsFromJob": - if e.complexity.Mutation.RemoveTagsFromJob == nil { + if e.ComplexityRoot.Mutation.RemoveTagsFromJob == nil { break } @@ -1393,9 +1377,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.RemoveTagsFromJob(childComplexity, args["job"].(string), args["tagIds"].([]string)), true + return e.ComplexityRoot.Mutation.RemoveTagsFromJob(childComplexity, args["job"].(string), args["tagIds"].([]string)), true case "Mutation.updateConfiguration": - if e.complexity.Mutation.UpdateConfiguration == nil { + if e.ComplexityRoot.Mutation.UpdateConfiguration == nil { break } @@ -1404,222 +1388,222 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Mutation.UpdateConfiguration(childComplexity, args["name"].(string), args["value"].(string)), true + return e.ComplexityRoot.Mutation.UpdateConfiguration(childComplexity, args["name"].(string), args["value"].(string)), true case "NamedStats.data": - if e.complexity.NamedStats.Data == nil { + if e.ComplexityRoot.NamedStats.Data == nil { break } - return e.complexity.NamedStats.Data(childComplexity), true + return e.ComplexityRoot.NamedStats.Data(childComplexity), true case "NamedStats.name": - if e.complexity.NamedStats.Name == nil { + if e.ComplexityRoot.NamedStats.Name == nil { break } - return e.complexity.NamedStats.Name(childComplexity), true + return e.ComplexityRoot.NamedStats.Name(childComplexity), true case "NamedStatsWithScope.name": - if e.complexity.NamedStatsWithScope.Name == nil { + if e.ComplexityRoot.NamedStatsWithScope.Name == nil { break } - return e.complexity.NamedStatsWithScope.Name(childComplexity), true + return e.ComplexityRoot.NamedStatsWithScope.Name(childComplexity), true case "NamedStatsWithScope.scope": - if e.complexity.NamedStatsWithScope.Scope == nil { + if e.ComplexityRoot.NamedStatsWithScope.Scope == nil { break } - return e.complexity.NamedStatsWithScope.Scope(childComplexity), true + return e.ComplexityRoot.NamedStatsWithScope.Scope(childComplexity), true case "NamedStatsWithScope.stats": - if e.complexity.NamedStatsWithScope.Stats == nil { + if e.ComplexityRoot.NamedStatsWithScope.Stats == nil { break } - return e.complexity.NamedStatsWithScope.Stats(childComplexity), true + return e.ComplexityRoot.NamedStatsWithScope.Stats(childComplexity), true case "Node.cluster": - if e.complexity.Node.Cluster == nil { + if e.ComplexityRoot.Node.Cluster == nil { break } - return e.complexity.Node.Cluster(childComplexity), true + return e.ComplexityRoot.Node.Cluster(childComplexity), true case "Node.cpusAllocated": - if e.complexity.Node.CpusAllocated == nil { + if e.ComplexityRoot.Node.CpusAllocated == nil { break } - return e.complexity.Node.CpusAllocated(childComplexity), true + return e.ComplexityRoot.Node.CpusAllocated(childComplexity), true case "Node.gpusAllocated": - if e.complexity.Node.GpusAllocated == nil { + if e.ComplexityRoot.Node.GpusAllocated == nil { break } - return e.complexity.Node.GpusAllocated(childComplexity), true + return e.ComplexityRoot.Node.GpusAllocated(childComplexity), true case "Node.healthData": - if e.complexity.Node.HealthData == nil { + if e.ComplexityRoot.Node.HealthData == nil { break } - return e.complexity.Node.HealthData(childComplexity), true + return e.ComplexityRoot.Node.HealthData(childComplexity), true case "Node.healthState": - if e.complexity.Node.HealthState == nil { + if e.ComplexityRoot.Node.HealthState == nil { break } - return e.complexity.Node.HealthState(childComplexity), true + return e.ComplexityRoot.Node.HealthState(childComplexity), true case "Node.hostname": - if e.complexity.Node.Hostname == nil { + if e.ComplexityRoot.Node.Hostname == nil { break } - return e.complexity.Node.Hostname(childComplexity), true + return e.ComplexityRoot.Node.Hostname(childComplexity), true case "Node.id": - if e.complexity.Node.ID == nil { + if e.ComplexityRoot.Node.ID == nil { break } - return e.complexity.Node.ID(childComplexity), true + return e.ComplexityRoot.Node.ID(childComplexity), true case "Node.jobsRunning": - if e.complexity.Node.JobsRunning == nil { + if e.ComplexityRoot.Node.JobsRunning == nil { break } - return e.complexity.Node.JobsRunning(childComplexity), true + return e.ComplexityRoot.Node.JobsRunning(childComplexity), true case "Node.memoryAllocated": - if e.complexity.Node.MemoryAllocated == nil { + if e.ComplexityRoot.Node.MemoryAllocated == nil { break } - return e.complexity.Node.MemoryAllocated(childComplexity), true + return e.ComplexityRoot.Node.MemoryAllocated(childComplexity), true case "Node.metaData": - if e.complexity.Node.MetaData == nil { + if e.ComplexityRoot.Node.MetaData == nil { break } - return e.complexity.Node.MetaData(childComplexity), true + return e.ComplexityRoot.Node.MetaData(childComplexity), true case "Node.schedulerState": - if e.complexity.Node.SchedulerState == nil { + if e.ComplexityRoot.Node.SchedulerState == nil { break } - return e.complexity.Node.SchedulerState(childComplexity), true + return e.ComplexityRoot.Node.SchedulerState(childComplexity), true case "Node.subCluster": - if e.complexity.Node.SubCluster == nil { + if e.ComplexityRoot.Node.SubCluster == nil { break } - return e.complexity.Node.SubCluster(childComplexity), true + return e.ComplexityRoot.Node.SubCluster(childComplexity), true case "NodeMetrics.host": - if e.complexity.NodeMetrics.Host == nil { + if e.ComplexityRoot.NodeMetrics.Host == nil { break } - return e.complexity.NodeMetrics.Host(childComplexity), true + return e.ComplexityRoot.NodeMetrics.Host(childComplexity), true case "NodeMetrics.metrics": - if e.complexity.NodeMetrics.Metrics == nil { + if e.ComplexityRoot.NodeMetrics.Metrics == nil { break } - return e.complexity.NodeMetrics.Metrics(childComplexity), true + return e.ComplexityRoot.NodeMetrics.Metrics(childComplexity), true case "NodeMetrics.state": - if e.complexity.NodeMetrics.State == nil { + if e.ComplexityRoot.NodeMetrics.State == nil { break } - return e.complexity.NodeMetrics.State(childComplexity), true + return e.ComplexityRoot.NodeMetrics.State(childComplexity), true case "NodeMetrics.subCluster": - if e.complexity.NodeMetrics.SubCluster == nil { + if e.ComplexityRoot.NodeMetrics.SubCluster == nil { break } - return e.complexity.NodeMetrics.SubCluster(childComplexity), true + return e.ComplexityRoot.NodeMetrics.SubCluster(childComplexity), true case "NodeStateResultList.count": - if e.complexity.NodeStateResultList.Count == nil { + if e.ComplexityRoot.NodeStateResultList.Count == nil { break } - return e.complexity.NodeStateResultList.Count(childComplexity), true + return e.ComplexityRoot.NodeStateResultList.Count(childComplexity), true case "NodeStateResultList.items": - if e.complexity.NodeStateResultList.Items == nil { + if e.ComplexityRoot.NodeStateResultList.Items == nil { break } - return e.complexity.NodeStateResultList.Items(childComplexity), true + return e.ComplexityRoot.NodeStateResultList.Items(childComplexity), true case "NodeStates.count": - if e.complexity.NodeStates.Count == nil { + if e.ComplexityRoot.NodeStates.Count == nil { break } - return e.complexity.NodeStates.Count(childComplexity), true + return e.ComplexityRoot.NodeStates.Count(childComplexity), true case "NodeStates.state": - if e.complexity.NodeStates.State == nil { + if e.ComplexityRoot.NodeStates.State == nil { break } - return e.complexity.NodeStates.State(childComplexity), true + return e.ComplexityRoot.NodeStates.State(childComplexity), true case "NodeStatesTimed.counts": - if e.complexity.NodeStatesTimed.Counts == nil { + if e.ComplexityRoot.NodeStatesTimed.Counts == nil { break } - return e.complexity.NodeStatesTimed.Counts(childComplexity), true + return e.ComplexityRoot.NodeStatesTimed.Counts(childComplexity), true case "NodeStatesTimed.state": - if e.complexity.NodeStatesTimed.State == nil { + if e.ComplexityRoot.NodeStatesTimed.State == nil { break } - return e.complexity.NodeStatesTimed.State(childComplexity), true + return e.ComplexityRoot.NodeStatesTimed.State(childComplexity), true case "NodeStatesTimed.times": - if e.complexity.NodeStatesTimed.Times == nil { + if e.ComplexityRoot.NodeStatesTimed.Times == nil { break } - return e.complexity.NodeStatesTimed.Times(childComplexity), true + return e.ComplexityRoot.NodeStatesTimed.Times(childComplexity), true case "NodesResultList.count": - if e.complexity.NodesResultList.Count == nil { + if e.ComplexityRoot.NodesResultList.Count == nil { break } - return e.complexity.NodesResultList.Count(childComplexity), true + return e.ComplexityRoot.NodesResultList.Count(childComplexity), true case "NodesResultList.hasNextPage": - if e.complexity.NodesResultList.HasNextPage == nil { + if e.ComplexityRoot.NodesResultList.HasNextPage == nil { break } - return e.complexity.NodesResultList.HasNextPage(childComplexity), true + return e.ComplexityRoot.NodesResultList.HasNextPage(childComplexity), true case "NodesResultList.items": - if e.complexity.NodesResultList.Items == nil { + if e.ComplexityRoot.NodesResultList.Items == nil { break } - return e.complexity.NodesResultList.Items(childComplexity), true + return e.ComplexityRoot.NodesResultList.Items(childComplexity), true case "NodesResultList.limit": - if e.complexity.NodesResultList.Limit == nil { + if e.ComplexityRoot.NodesResultList.Limit == nil { break } - return e.complexity.NodesResultList.Limit(childComplexity), true + return e.ComplexityRoot.NodesResultList.Limit(childComplexity), true case "NodesResultList.offset": - if e.complexity.NodesResultList.Offset == nil { + if e.ComplexityRoot.NodesResultList.Offset == nil { break } - return e.complexity.NodesResultList.Offset(childComplexity), true + return e.ComplexityRoot.NodesResultList.Offset(childComplexity), true case "NodesResultList.totalNodes": - if e.complexity.NodesResultList.TotalNodes == nil { + if e.ComplexityRoot.NodesResultList.TotalNodes == nil { break } - return e.complexity.NodesResultList.TotalNodes(childComplexity), true + return e.ComplexityRoot.NodesResultList.TotalNodes(childComplexity), true case "Query.allocatedNodes": - if e.complexity.Query.AllocatedNodes == nil { + if e.ComplexityRoot.Query.AllocatedNodes == nil { break } @@ -1628,9 +1612,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.AllocatedNodes(childComplexity, args["cluster"].(string)), true + return e.ComplexityRoot.Query.AllocatedNodes(childComplexity, args["cluster"].(string)), true case "Query.clusterMetrics": - if e.complexity.Query.ClusterMetrics == nil { + if e.ComplexityRoot.Query.ClusterMetrics == nil { break } @@ -1639,21 +1623,22 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.ClusterMetrics(childComplexity, args["cluster"].(string), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true + return e.ComplexityRoot.Query.ClusterMetrics(childComplexity, args["cluster"].(string), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true case "Query.clusters": - if e.complexity.Query.Clusters == nil { + if e.ComplexityRoot.Query.Clusters == nil { break } - return e.complexity.Query.Clusters(childComplexity), true + return e.ComplexityRoot.Query.Clusters(childComplexity), true case "Query.globalMetrics": - if e.complexity.Query.GlobalMetrics == nil { + if e.ComplexityRoot.Query.GlobalMetrics == nil { break } - return e.complexity.Query.GlobalMetrics(childComplexity), true + return e.ComplexityRoot.Query.GlobalMetrics(childComplexity), true + case "Query.job": - if e.complexity.Query.Job == nil { + if e.ComplexityRoot.Query.Job == nil { break } @@ -1662,9 +1647,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.Job(childComplexity, args["id"].(string)), true + return e.ComplexityRoot.Query.Job(childComplexity, args["id"].(string)), true case "Query.jobMetrics": - if e.complexity.Query.JobMetrics == nil { + if e.ComplexityRoot.Query.JobMetrics == nil { break } @@ -1673,9 +1658,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.JobMetrics(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope), args["resolution"].(*int)), true + return e.ComplexityRoot.Query.JobMetrics(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope), args["resolution"].(*int)), true case "Query.jobStats": - if e.complexity.Query.JobStats == nil { + if e.ComplexityRoot.Query.JobStats == nil { break } @@ -1684,9 +1669,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.JobStats(childComplexity, args["id"].(string), args["metrics"].([]string)), true + return e.ComplexityRoot.Query.JobStats(childComplexity, args["id"].(string), args["metrics"].([]string)), true case "Query.jobs": - if e.complexity.Query.Jobs == nil { + if e.ComplexityRoot.Query.Jobs == nil { break } @@ -1695,9 +1680,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.Jobs(childComplexity, args["filter"].([]*model.JobFilter), args["page"].(*model.PageRequest), args["order"].(*model.OrderByInput)), true + return e.ComplexityRoot.Query.Jobs(childComplexity, args["filter"].([]*model.JobFilter), args["page"].(*model.PageRequest), args["order"].(*model.OrderByInput)), true case "Query.jobsFootprints": - if e.complexity.Query.JobsFootprints == nil { + if e.ComplexityRoot.Query.JobsFootprints == nil { break } @@ -1706,9 +1691,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.JobsFootprints(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true + return e.ComplexityRoot.Query.JobsFootprints(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true case "Query.jobsMetricStats": - if e.complexity.Query.JobsMetricStats == nil { + if e.ComplexityRoot.Query.JobsMetricStats == nil { break } @@ -1717,9 +1702,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.JobsMetricStats(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true + return e.ComplexityRoot.Query.JobsMetricStats(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string)), true case "Query.jobsStatistics": - if e.complexity.Query.JobsStatistics == nil { + if e.ComplexityRoot.Query.JobsStatistics == nil { break } @@ -1728,9 +1713,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate), args["numDurationBins"].(*string), args["numMetricBins"].(*int)), true + return e.ComplexityRoot.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate), args["numDurationBins"].(*string), args["numMetricBins"].(*int)), true case "Query.node": - if e.complexity.Query.Node == nil { + if e.ComplexityRoot.Query.Node == nil { break } @@ -1739,9 +1724,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.Node(childComplexity, args["id"].(string)), true + return e.ComplexityRoot.Query.Node(childComplexity, args["id"].(string)), true case "Query.nodeMetrics": - if e.complexity.Query.NodeMetrics == nil { + if e.ComplexityRoot.Query.NodeMetrics == nil { break } @@ -1750,9 +1735,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.NodeMetrics(childComplexity, args["cluster"].(string), args["nodes"].([]string), args["scopes"].([]schema.MetricScope), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true + return e.ComplexityRoot.Query.NodeMetrics(childComplexity, args["cluster"].(string), args["nodes"].([]string), args["scopes"].([]schema.MetricScope), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true case "Query.nodeMetricsList": - if e.complexity.Query.NodeMetricsList == nil { + if e.ComplexityRoot.Query.NodeMetricsList == nil { break } @@ -1761,9 +1746,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.NodeMetricsList(childComplexity, args["cluster"].(string), args["subCluster"].(string), args["stateFilter"].(string), args["nodeFilter"].(string), args["scopes"].([]schema.MetricScope), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time), args["page"].(*model.PageRequest), args["resolution"].(*int)), true + return e.ComplexityRoot.Query.NodeMetricsList(childComplexity, args["cluster"].(string), args["subCluster"].(string), args["stateFilter"].(string), args["nodeFilter"].(string), args["scopes"].([]schema.MetricScope), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time), args["page"].(*model.PageRequest), args["resolution"].(*int)), true case "Query.nodeStates": - if e.complexity.Query.NodeStates == nil { + if e.ComplexityRoot.Query.NodeStates == nil { break } @@ -1772,9 +1757,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.NodeStates(childComplexity, args["filter"].([]*model.NodeFilter)), true + return e.ComplexityRoot.Query.NodeStates(childComplexity, args["filter"].([]*model.NodeFilter)), true case "Query.nodeStatesTimed": - if e.complexity.Query.NodeStatesTimed == nil { + if e.ComplexityRoot.Query.NodeStatesTimed == nil { break } @@ -1783,9 +1768,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.NodeStatesTimed(childComplexity, args["filter"].([]*model.NodeFilter), args["type"].(string)), true + return e.ComplexityRoot.Query.NodeStatesTimed(childComplexity, args["filter"].([]*model.NodeFilter), args["type"].(string)), true case "Query.nodes": - if e.complexity.Query.Nodes == nil { + if e.ComplexityRoot.Query.Nodes == nil { break } @@ -1794,9 +1779,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.Nodes(childComplexity, args["filter"].([]*model.NodeFilter), args["order"].(*model.OrderByInput)), true + return e.ComplexityRoot.Query.Nodes(childComplexity, args["filter"].([]*model.NodeFilter), args["order"].(*model.OrderByInput)), true case "Query.nodesWithMeta": - if e.complexity.Query.NodesWithMeta == nil { + if e.ComplexityRoot.Query.NodesWithMeta == nil { break } @@ -1805,9 +1790,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.NodesWithMeta(childComplexity, args["filter"].([]*model.NodeFilter), args["order"].(*model.OrderByInput)), true + return e.ComplexityRoot.Query.NodesWithMeta(childComplexity, args["filter"].([]*model.NodeFilter), args["order"].(*model.OrderByInput)), true case "Query.rooflineHeatmap": - if e.complexity.Query.RooflineHeatmap == nil { + if e.ComplexityRoot.Query.RooflineHeatmap == nil { break } @@ -1816,9 +1801,9 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.RooflineHeatmap(childComplexity, args["filter"].([]*model.JobFilter), args["rows"].(int), args["cols"].(int), args["minX"].(float64), args["minY"].(float64), args["maxX"].(float64), args["maxY"].(float64)), true + return e.ComplexityRoot.Query.RooflineHeatmap(childComplexity, args["filter"].([]*model.JobFilter), args["rows"].(int), args["cols"].(int), args["minX"].(float64), args["minY"].(float64), args["maxX"].(float64), args["maxY"].(float64)), true case "Query.scopedJobStats": - if e.complexity.Query.ScopedJobStats == nil { + if e.ComplexityRoot.Query.ScopedJobStats == nil { break } @@ -1827,15 +1812,15 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.ScopedJobStats(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope)), true + return e.ComplexityRoot.Query.ScopedJobStats(childComplexity, args["id"].(string), args["metrics"].([]string), args["scopes"].([]schema.MetricScope)), true case "Query.tags": - if e.complexity.Query.Tags == nil { + if e.ComplexityRoot.Query.Tags == nil { break } - return e.complexity.Query.Tags(childComplexity), true + return e.ComplexityRoot.Query.Tags(childComplexity), true case "Query.user": - if e.complexity.Query.User == nil { + if e.ComplexityRoot.Query.User == nil { break } @@ -1844,349 +1829,349 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Query.User(childComplexity, args["username"].(string)), true + return e.ComplexityRoot.Query.User(childComplexity, args["username"].(string)), true case "Resource.accelerators": - if e.complexity.Resource.Accelerators == nil { + if e.ComplexityRoot.Resource.Accelerators == nil { break } - return e.complexity.Resource.Accelerators(childComplexity), true + return e.ComplexityRoot.Resource.Accelerators(childComplexity), true case "Resource.configuration": - if e.complexity.Resource.Configuration == nil { + if e.ComplexityRoot.Resource.Configuration == nil { break } - return e.complexity.Resource.Configuration(childComplexity), true + return e.ComplexityRoot.Resource.Configuration(childComplexity), true case "Resource.hwthreads": - if e.complexity.Resource.HWThreads == nil { + if e.ComplexityRoot.Resource.HWThreads == nil { break } - return e.complexity.Resource.HWThreads(childComplexity), true + return e.ComplexityRoot.Resource.HWThreads(childComplexity), true case "Resource.hostname": - if e.complexity.Resource.Hostname == nil { + if e.ComplexityRoot.Resource.Hostname == nil { break } - return e.complexity.Resource.Hostname(childComplexity), true + return e.ComplexityRoot.Resource.Hostname(childComplexity), true case "ScopedStats.data": - if e.complexity.ScopedStats.Data == nil { + if e.ComplexityRoot.ScopedStats.Data == nil { break } - return e.complexity.ScopedStats.Data(childComplexity), true + return e.ComplexityRoot.ScopedStats.Data(childComplexity), true case "ScopedStats.hostname": - if e.complexity.ScopedStats.Hostname == nil { + if e.ComplexityRoot.ScopedStats.Hostname == nil { break } - return e.complexity.ScopedStats.Hostname(childComplexity), true + return e.ComplexityRoot.ScopedStats.Hostname(childComplexity), true case "ScopedStats.id": - if e.complexity.ScopedStats.ID == nil { + if e.ComplexityRoot.ScopedStats.ID == nil { break } - return e.complexity.ScopedStats.ID(childComplexity), true + return e.ComplexityRoot.ScopedStats.ID(childComplexity), true case "Series.data": - if e.complexity.Series.Data == nil { + if e.ComplexityRoot.Series.Data == nil { break } - return e.complexity.Series.Data(childComplexity), true + return e.ComplexityRoot.Series.Data(childComplexity), true case "Series.hostname": - if e.complexity.Series.Hostname == nil { + if e.ComplexityRoot.Series.Hostname == nil { break } - return e.complexity.Series.Hostname(childComplexity), true + return e.ComplexityRoot.Series.Hostname(childComplexity), true case "Series.id": - if e.complexity.Series.ID == nil { + if e.ComplexityRoot.Series.ID == nil { break } - return e.complexity.Series.ID(childComplexity), true + return e.ComplexityRoot.Series.ID(childComplexity), true case "Series.statistics": - if e.complexity.Series.Statistics == nil { + if e.ComplexityRoot.Series.Statistics == nil { break } - return e.complexity.Series.Statistics(childComplexity), true + return e.ComplexityRoot.Series.Statistics(childComplexity), true case "StatsSeries.max": - if e.complexity.StatsSeries.Max == nil { + if e.ComplexityRoot.StatsSeries.Max == nil { break } - return e.complexity.StatsSeries.Max(childComplexity), true + return e.ComplexityRoot.StatsSeries.Max(childComplexity), true case "StatsSeries.mean": - if e.complexity.StatsSeries.Mean == nil { + if e.ComplexityRoot.StatsSeries.Mean == nil { break } - return e.complexity.StatsSeries.Mean(childComplexity), true + return e.ComplexityRoot.StatsSeries.Mean(childComplexity), true case "StatsSeries.median": - if e.complexity.StatsSeries.Median == nil { + if e.ComplexityRoot.StatsSeries.Median == nil { break } - return e.complexity.StatsSeries.Median(childComplexity), true + return e.ComplexityRoot.StatsSeries.Median(childComplexity), true case "StatsSeries.min": - if e.complexity.StatsSeries.Min == nil { + if e.ComplexityRoot.StatsSeries.Min == nil { break } - return e.complexity.StatsSeries.Min(childComplexity), true + return e.ComplexityRoot.StatsSeries.Min(childComplexity), true case "SubCluster.coresPerSocket": - if e.complexity.SubCluster.CoresPerSocket == nil { + if e.ComplexityRoot.SubCluster.CoresPerSocket == nil { break } - return e.complexity.SubCluster.CoresPerSocket(childComplexity), true + return e.ComplexityRoot.SubCluster.CoresPerSocket(childComplexity), true case "SubCluster.flopRateScalar": - if e.complexity.SubCluster.FlopRateScalar == nil { + if e.ComplexityRoot.SubCluster.FlopRateScalar == nil { break } - return e.complexity.SubCluster.FlopRateScalar(childComplexity), true + return e.ComplexityRoot.SubCluster.FlopRateScalar(childComplexity), true case "SubCluster.flopRateSimd": - if e.complexity.SubCluster.FlopRateSimd == nil { + if e.ComplexityRoot.SubCluster.FlopRateSimd == nil { break } - return e.complexity.SubCluster.FlopRateSimd(childComplexity), true + return e.ComplexityRoot.SubCluster.FlopRateSimd(childComplexity), true case "SubCluster.footprint": - if e.complexity.SubCluster.Footprint == nil { + if e.ComplexityRoot.SubCluster.Footprint == nil { break } - return e.complexity.SubCluster.Footprint(childComplexity), true + return e.ComplexityRoot.SubCluster.Footprint(childComplexity), true case "SubCluster.memoryBandwidth": - if e.complexity.SubCluster.MemoryBandwidth == nil { + if e.ComplexityRoot.SubCluster.MemoryBandwidth == nil { break } - return e.complexity.SubCluster.MemoryBandwidth(childComplexity), true + return e.ComplexityRoot.SubCluster.MemoryBandwidth(childComplexity), true case "SubCluster.metricConfig": - if e.complexity.SubCluster.MetricConfig == nil { + if e.ComplexityRoot.SubCluster.MetricConfig == nil { break } - return e.complexity.SubCluster.MetricConfig(childComplexity), true + return e.ComplexityRoot.SubCluster.MetricConfig(childComplexity), true case "SubCluster.name": - if e.complexity.SubCluster.Name == nil { + if e.ComplexityRoot.SubCluster.Name == nil { break } - return e.complexity.SubCluster.Name(childComplexity), true + return e.ComplexityRoot.SubCluster.Name(childComplexity), true case "SubCluster.nodes": - if e.complexity.SubCluster.Nodes == nil { + if e.ComplexityRoot.SubCluster.Nodes == nil { break } - return e.complexity.SubCluster.Nodes(childComplexity), true + return e.ComplexityRoot.SubCluster.Nodes(childComplexity), true case "SubCluster.numberOfNodes": - if e.complexity.SubCluster.NumberOfNodes == nil { + if e.ComplexityRoot.SubCluster.NumberOfNodes == nil { break } - return e.complexity.SubCluster.NumberOfNodes(childComplexity), true + return e.ComplexityRoot.SubCluster.NumberOfNodes(childComplexity), true case "SubCluster.processorType": - if e.complexity.SubCluster.ProcessorType == nil { + if e.ComplexityRoot.SubCluster.ProcessorType == nil { break } - return e.complexity.SubCluster.ProcessorType(childComplexity), true + return e.ComplexityRoot.SubCluster.ProcessorType(childComplexity), true case "SubCluster.socketsPerNode": - if e.complexity.SubCluster.SocketsPerNode == nil { + if e.ComplexityRoot.SubCluster.SocketsPerNode == nil { break } - return e.complexity.SubCluster.SocketsPerNode(childComplexity), true + return e.ComplexityRoot.SubCluster.SocketsPerNode(childComplexity), true case "SubCluster.threadsPerCore": - if e.complexity.SubCluster.ThreadsPerCore == nil { + if e.ComplexityRoot.SubCluster.ThreadsPerCore == nil { break } - return e.complexity.SubCluster.ThreadsPerCore(childComplexity), true + return e.ComplexityRoot.SubCluster.ThreadsPerCore(childComplexity), true case "SubCluster.topology": - if e.complexity.SubCluster.Topology == nil { + if e.ComplexityRoot.SubCluster.Topology == nil { break } - return e.complexity.SubCluster.Topology(childComplexity), true + return e.ComplexityRoot.SubCluster.Topology(childComplexity), true case "SubClusterConfig.alert": - if e.complexity.SubClusterConfig.Alert == nil { + if e.ComplexityRoot.SubClusterConfig.Alert == nil { break } - return e.complexity.SubClusterConfig.Alert(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Alert(childComplexity), true case "SubClusterConfig.caution": - if e.complexity.SubClusterConfig.Caution == nil { + if e.ComplexityRoot.SubClusterConfig.Caution == nil { break } - return e.complexity.SubClusterConfig.Caution(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Caution(childComplexity), true case "SubClusterConfig.name": - if e.complexity.SubClusterConfig.Name == nil { + if e.ComplexityRoot.SubClusterConfig.Name == nil { break } - return e.complexity.SubClusterConfig.Name(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Name(childComplexity), true case "SubClusterConfig.normal": - if e.complexity.SubClusterConfig.Normal == nil { + if e.ComplexityRoot.SubClusterConfig.Normal == nil { break } - return e.complexity.SubClusterConfig.Normal(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Normal(childComplexity), true case "SubClusterConfig.peak": - if e.complexity.SubClusterConfig.Peak == nil { + if e.ComplexityRoot.SubClusterConfig.Peak == nil { break } - return e.complexity.SubClusterConfig.Peak(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Peak(childComplexity), true case "SubClusterConfig.remove": - if e.complexity.SubClusterConfig.Remove == nil { + if e.ComplexityRoot.SubClusterConfig.Remove == nil { break } - return e.complexity.SubClusterConfig.Remove(childComplexity), true + return e.ComplexityRoot.SubClusterConfig.Remove(childComplexity), true case "Tag.id": - if e.complexity.Tag.ID == nil { + if e.ComplexityRoot.Tag.ID == nil { break } - return e.complexity.Tag.ID(childComplexity), true + return e.ComplexityRoot.Tag.ID(childComplexity), true case "Tag.name": - if e.complexity.Tag.Name == nil { + if e.ComplexityRoot.Tag.Name == nil { break } - return e.complexity.Tag.Name(childComplexity), true + return e.ComplexityRoot.Tag.Name(childComplexity), true case "Tag.scope": - if e.complexity.Tag.Scope == nil { + if e.ComplexityRoot.Tag.Scope == nil { break } - return e.complexity.Tag.Scope(childComplexity), true + return e.ComplexityRoot.Tag.Scope(childComplexity), true case "Tag.type": - if e.complexity.Tag.Type == nil { + if e.ComplexityRoot.Tag.Type == nil { break } - return e.complexity.Tag.Type(childComplexity), true + return e.ComplexityRoot.Tag.Type(childComplexity), true case "TimeRangeOutput.from": - if e.complexity.TimeRangeOutput.From == nil { + if e.ComplexityRoot.TimeRangeOutput.From == nil { break } - return e.complexity.TimeRangeOutput.From(childComplexity), true + return e.ComplexityRoot.TimeRangeOutput.From(childComplexity), true case "TimeRangeOutput.range": - if e.complexity.TimeRangeOutput.Range == nil { + if e.ComplexityRoot.TimeRangeOutput.Range == nil { break } - return e.complexity.TimeRangeOutput.Range(childComplexity), true + return e.ComplexityRoot.TimeRangeOutput.Range(childComplexity), true case "TimeRangeOutput.to": - if e.complexity.TimeRangeOutput.To == nil { + if e.ComplexityRoot.TimeRangeOutput.To == nil { break } - return e.complexity.TimeRangeOutput.To(childComplexity), true + return e.ComplexityRoot.TimeRangeOutput.To(childComplexity), true case "TimeWeights.accHours": - if e.complexity.TimeWeights.AccHours == nil { + if e.ComplexityRoot.TimeWeights.AccHours == nil { break } - return e.complexity.TimeWeights.AccHours(childComplexity), true + return e.ComplexityRoot.TimeWeights.AccHours(childComplexity), true case "TimeWeights.coreHours": - if e.complexity.TimeWeights.CoreHours == nil { + if e.ComplexityRoot.TimeWeights.CoreHours == nil { break } - return e.complexity.TimeWeights.CoreHours(childComplexity), true + return e.ComplexityRoot.TimeWeights.CoreHours(childComplexity), true case "TimeWeights.nodeHours": - if e.complexity.TimeWeights.NodeHours == nil { + if e.ComplexityRoot.TimeWeights.NodeHours == nil { break } - return e.complexity.TimeWeights.NodeHours(childComplexity), true + return e.ComplexityRoot.TimeWeights.NodeHours(childComplexity), true case "Topology.accelerators": - if e.complexity.Topology.Accelerators == nil { + if e.ComplexityRoot.Topology.Accelerators == nil { break } - return e.complexity.Topology.Accelerators(childComplexity), true + return e.ComplexityRoot.Topology.Accelerators(childComplexity), true case "Topology.core": - if e.complexity.Topology.Core == nil { + if e.ComplexityRoot.Topology.Core == nil { break } - return e.complexity.Topology.Core(childComplexity), true + return e.ComplexityRoot.Topology.Core(childComplexity), true case "Topology.die": - if e.complexity.Topology.Die == nil { + if e.ComplexityRoot.Topology.Die == nil { break } - return e.complexity.Topology.Die(childComplexity), true + return e.ComplexityRoot.Topology.Die(childComplexity), true case "Topology.memoryDomain": - if e.complexity.Topology.MemoryDomain == nil { + if e.ComplexityRoot.Topology.MemoryDomain == nil { break } - return e.complexity.Topology.MemoryDomain(childComplexity), true + return e.ComplexityRoot.Topology.MemoryDomain(childComplexity), true case "Topology.node": - if e.complexity.Topology.Node == nil { + if e.ComplexityRoot.Topology.Node == nil { break } - return e.complexity.Topology.Node(childComplexity), true + return e.ComplexityRoot.Topology.Node(childComplexity), true case "Topology.socket": - if e.complexity.Topology.Socket == nil { + if e.ComplexityRoot.Topology.Socket == nil { break } - return e.complexity.Topology.Socket(childComplexity), true + return e.ComplexityRoot.Topology.Socket(childComplexity), true case "Unit.base": - if e.complexity.Unit.Base == nil { + if e.ComplexityRoot.Unit.Base == nil { break } - return e.complexity.Unit.Base(childComplexity), true + return e.ComplexityRoot.Unit.Base(childComplexity), true case "Unit.prefix": - if e.complexity.Unit.Prefix == nil { + if e.ComplexityRoot.Unit.Prefix == nil { break } - return e.complexity.Unit.Prefix(childComplexity), true + return e.ComplexityRoot.Unit.Prefix(childComplexity), true case "User.email": - if e.complexity.User.Email == nil { + if e.ComplexityRoot.User.Email == nil { break } - return e.complexity.User.Email(childComplexity), true + return e.ComplexityRoot.User.Email(childComplexity), true case "User.name": - if e.complexity.User.Name == nil { + if e.ComplexityRoot.User.Name == nil { break } - return e.complexity.User.Name(childComplexity), true + return e.ComplexityRoot.User.Name(childComplexity), true case "User.username": - if e.complexity.User.Username == nil { + if e.ComplexityRoot.User.Username == nil { break } - return e.complexity.User.Username(childComplexity), true + return e.ComplexityRoot.User.Username(childComplexity), true } return 0, false @@ -2194,7 +2179,7 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { opCtx := graphql.GetOperationContext(ctx) - ec := executionContext{opCtx, e, 0, 0, make(chan graphql.DeferredResult)} + ec := newExecutionContext(opCtx, e, make(chan graphql.DeferredResult)) inputUnmarshalMap := graphql.BuildUnmarshalerMap( ec.unmarshalInputFloatRange, ec.unmarshalInputIntRange, @@ -2218,9 +2203,9 @@ func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { ctx = graphql.WithUnmarshalerMap(ctx, inputUnmarshalMap) data = ec._Query(ctx, opCtx.Operation.SelectionSet) } else { - if atomic.LoadInt32(&ec.pendingDeferred) > 0 { - result := <-ec.deferredResults - atomic.AddInt32(&ec.pendingDeferred, -1) + if atomic.LoadInt32(&ec.PendingDeferred) > 0 { + result := <-ec.DeferredResults + atomic.AddInt32(&ec.PendingDeferred, -1) data = result.Result response.Path = result.Path response.Label = result.Label @@ -2232,8 +2217,8 @@ func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { var buf bytes.Buffer data.MarshalGQL(&buf) response.Data = buf.Bytes() - if atomic.LoadInt32(&ec.deferred) > 0 { - hasNext := atomic.LoadInt32(&ec.pendingDeferred) > 0 + if atomic.LoadInt32(&ec.Deferred) > 0 { + hasNext := atomic.LoadInt32(&ec.PendingDeferred) > 0 response.HasNext = &hasNext } @@ -2261,44 +2246,22 @@ func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { } type executionContext struct { - *graphql.OperationContext - *executableSchema - deferred int32 - pendingDeferred int32 - deferredResults chan graphql.DeferredResult + *graphql.ExecutionContextState[ResolverRoot, DirectiveRoot, ComplexityRoot] } -func (ec *executionContext) processDeferredGroup(dg graphql.DeferredGroup) { - atomic.AddInt32(&ec.pendingDeferred, 1) - go func() { - ctx := graphql.WithFreshResponseContext(dg.Context) - dg.FieldSet.Dispatch(ctx) - ds := graphql.DeferredResult{ - Path: dg.Path, - Label: dg.Label, - Result: dg.FieldSet, - Errors: graphql.GetErrors(ctx), - } - // null fields should bubble up - if dg.FieldSet.Invalids > 0 { - ds.Result = graphql.Null - } - ec.deferredResults <- ds - }() -} - -func (ec *executionContext) introspectSchema() (*introspection.Schema, error) { - if ec.DisableIntrospection { - return nil, errors.New("introspection disabled") +func newExecutionContext( + opCtx *graphql.OperationContext, + execSchema *executableSchema, + deferredResults chan graphql.DeferredResult, +) executionContext { + return executionContext{ + ExecutionContextState: graphql.NewExecutionContextState[ResolverRoot, DirectiveRoot, ComplexityRoot]( + opCtx, + (*graphql.ExecutableSchemaState[ResolverRoot, DirectiveRoot, ComplexityRoot])(execSchema), + parsedSchema, + deferredResults, + ), } - return introspection.WrapSchema(ec.Schema()), nil -} - -func (ec *executionContext) introspectType(name string) (*introspection.Type, error) { - if ec.DisableIntrospection { - return nil, errors.New("introspection disabled") - } - return introspection.WrapTypeFromDef(ec.Schema(), ec.Schema().Types[name]), nil } var sources = []*ast.Source{ @@ -3571,7 +3534,7 @@ func (ec *executionContext) _Cluster_partitions(ctx context.Context, field graph field, ec.fieldContext_Cluster_partitions, func(ctx context.Context) (any, error) { - return ec.resolvers.Cluster().Partitions(ctx, obj) + return ec.Resolvers.Cluster().Partitions(ctx, obj) }, nil, ec.marshalNString2ᚕstringᚄ, @@ -4656,7 +4619,7 @@ func (ec *executionContext) _Job_startTime(ctx context.Context, field graphql.Co field, ec.fieldContext_Job_startTime, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().StartTime(ctx, obj) + return ec.Resolvers.Job().StartTime(ctx, obj) }, nil, ec.marshalNTime2ᚖtimeᚐTime, @@ -5033,7 +4996,7 @@ func (ec *executionContext) _Job_tags(ctx context.Context, field graphql.Collect field, ec.fieldContext_Job_tags, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().Tags(ctx, obj) + return ec.Resolvers.Job().Tags(ctx, obj) }, nil, ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, @@ -5111,7 +5074,7 @@ func (ec *executionContext) _Job_concurrentJobs(ctx context.Context, field graph field, ec.fieldContext_Job_concurrentJobs, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().ConcurrentJobs(ctx, obj) + return ec.Resolvers.Job().ConcurrentJobs(ctx, obj) }, nil, ec.marshalOJobLinkResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobLinkResultList, @@ -5148,7 +5111,7 @@ func (ec *executionContext) _Job_footprint(ctx context.Context, field graphql.Co field, ec.fieldContext_Job_footprint, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().Footprint(ctx, obj) + return ec.Resolvers.Job().Footprint(ctx, obj) }, nil, ec.marshalOFootprintValue2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprintValue, @@ -5185,7 +5148,7 @@ func (ec *executionContext) _Job_energyFootprint(ctx context.Context, field grap field, ec.fieldContext_Job_energyFootprint, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().EnergyFootprint(ctx, obj) + return ec.Resolvers.Job().EnergyFootprint(ctx, obj) }, nil, ec.marshalOEnergyFootprintValue2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐEnergyFootprintValue, @@ -5222,7 +5185,7 @@ func (ec *executionContext) _Job_metaData(ctx context.Context, field graphql.Col field, ec.fieldContext_Job_metaData, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().MetaData(ctx, obj) + return ec.Resolvers.Job().MetaData(ctx, obj) }, nil, ec.marshalOAny2interface, @@ -5251,7 +5214,7 @@ func (ec *executionContext) _Job_userData(ctx context.Context, field graphql.Col field, ec.fieldContext_Job_userData, func(ctx context.Context) (any, error) { - return ec.resolvers.Job().UserData(ctx, obj) + return ec.Resolvers.Job().UserData(ctx, obj) }, nil, ec.marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐUser, @@ -7455,7 +7418,7 @@ func (ec *executionContext) _MetricValue_name(ctx context.Context, field graphql field, ec.fieldContext_MetricValue_name, func(ctx context.Context) (any, error) { - return ec.resolvers.MetricValue().Name(ctx, obj) + return ec.Resolvers.MetricValue().Name(ctx, obj) }, nil, ec.marshalOString2ᚖstring, @@ -7549,7 +7512,7 @@ func (ec *executionContext) _Mutation_createTag(ctx context.Context, field graph ec.fieldContext_Mutation_createTag, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().CreateTag(ctx, fc.Args["type"].(string), fc.Args["name"].(string), fc.Args["scope"].(string)) + return ec.Resolvers.Mutation().CreateTag(ctx, fc.Args["type"].(string), fc.Args["name"].(string), fc.Args["scope"].(string)) }, nil, ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag, @@ -7600,7 +7563,7 @@ func (ec *executionContext) _Mutation_deleteTag(ctx context.Context, field graph ec.fieldContext_Mutation_deleteTag, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().DeleteTag(ctx, fc.Args["id"].(string)) + return ec.Resolvers.Mutation().DeleteTag(ctx, fc.Args["id"].(string)) }, nil, ec.marshalNID2string, @@ -7641,7 +7604,7 @@ func (ec *executionContext) _Mutation_addTagsToJob(ctx context.Context, field gr ec.fieldContext_Mutation_addTagsToJob, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().AddTagsToJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) + return ec.Resolvers.Mutation().AddTagsToJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) }, nil, ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, @@ -7692,7 +7655,7 @@ func (ec *executionContext) _Mutation_removeTagsFromJob(ctx context.Context, fie ec.fieldContext_Mutation_removeTagsFromJob, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().RemoveTagsFromJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) + return ec.Resolvers.Mutation().RemoveTagsFromJob(ctx, fc.Args["job"].(string), fc.Args["tagIds"].([]string)) }, nil, ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, @@ -7743,7 +7706,7 @@ func (ec *executionContext) _Mutation_removeTagFromList(ctx context.Context, fie ec.fieldContext_Mutation_removeTagFromList, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().RemoveTagFromList(ctx, fc.Args["tagIds"].([]string)) + return ec.Resolvers.Mutation().RemoveTagFromList(ctx, fc.Args["tagIds"].([]string)) }, nil, ec.marshalNInt2ᚕintᚄ, @@ -7784,7 +7747,7 @@ func (ec *executionContext) _Mutation_updateConfiguration(ctx context.Context, f ec.fieldContext_Mutation_updateConfiguration, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Mutation().UpdateConfiguration(ctx, fc.Args["name"].(string), fc.Args["value"].(string)) + return ec.Resolvers.Mutation().UpdateConfiguration(ctx, fc.Args["name"].(string), fc.Args["value"].(string)) }, nil, ec.marshalOString2ᚖstring, @@ -7985,7 +7948,7 @@ func (ec *executionContext) _Node_id(ctx context.Context, field graphql.Collecte field, ec.fieldContext_Node_id, func(ctx context.Context) (any, error) { - return ec.resolvers.Node().ID(ctx, obj) + return ec.Resolvers.Node().ID(ctx, obj) }, nil, ec.marshalNID2string, @@ -8217,7 +8180,7 @@ func (ec *executionContext) _Node_schedulerState(ctx context.Context, field grap field, ec.fieldContext_Node_schedulerState, func(ctx context.Context) (any, error) { - return ec.resolvers.Node().SchedulerState(ctx, obj) + return ec.Resolvers.Node().SchedulerState(ctx, obj) }, nil, ec.marshalNSchedulerState2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSchedulerState, @@ -8246,7 +8209,7 @@ func (ec *executionContext) _Node_healthState(ctx context.Context, field graphql field, ec.fieldContext_Node_healthState, func(ctx context.Context) (any, error) { - return ec.resolvers.Node().HealthState(ctx, obj) + return ec.Resolvers.Node().HealthState(ctx, obj) }, nil, ec.marshalNMonitoringState2string, @@ -8275,7 +8238,7 @@ func (ec *executionContext) _Node_metaData(ctx context.Context, field graphql.Co field, ec.fieldContext_Node_metaData, func(ctx context.Context) (any, error) { - return ec.resolvers.Node().MetaData(ctx, obj) + return ec.Resolvers.Node().MetaData(ctx, obj) }, nil, ec.marshalOAny2interface, @@ -8304,7 +8267,7 @@ func (ec *executionContext) _Node_healthData(ctx context.Context, field graphql. field, ec.fieldContext_Node_healthData, func(ctx context.Context) (any, error) { - return ec.resolvers.Node().HealthData(ctx, obj) + return ec.Resolvers.Node().HealthData(ctx, obj) }, nil, ec.marshalOAny2interface, @@ -8870,7 +8833,7 @@ func (ec *executionContext) _Query_clusters(ctx context.Context, field graphql.C field, ec.fieldContext_Query_clusters, func(ctx context.Context) (any, error) { - return ec.resolvers.Query().Clusters(ctx) + return ec.Resolvers.Query().Clusters(ctx) }, nil, ec.marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterᚄ, @@ -8907,7 +8870,7 @@ func (ec *executionContext) _Query_tags(ctx context.Context, field graphql.Colle field, ec.fieldContext_Query_tags, func(ctx context.Context) (any, error) { - return ec.resolvers.Query().Tags(ctx) + return ec.Resolvers.Query().Tags(ctx) }, nil, ec.marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ, @@ -8946,7 +8909,7 @@ func (ec *executionContext) _Query_globalMetrics(ctx context.Context, field grap field, ec.fieldContext_Query_globalMetrics, func(ctx context.Context) (any, error) { - return ec.resolvers.Query().GlobalMetrics(ctx) + return ec.Resolvers.Query().GlobalMetrics(ctx) }, nil, ec.marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItemᚄ, @@ -8988,7 +8951,7 @@ func (ec *executionContext) _Query_user(ctx context.Context, field graphql.Colle ec.fieldContext_Query_user, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().User(ctx, fc.Args["username"].(string)) + return ec.Resolvers.Query().User(ctx, fc.Args["username"].(string)) }, nil, ec.marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐUser, @@ -9037,7 +9000,7 @@ func (ec *executionContext) _Query_allocatedNodes(ctx context.Context, field gra ec.fieldContext_Query_allocatedNodes, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().AllocatedNodes(ctx, fc.Args["cluster"].(string)) + return ec.Resolvers.Query().AllocatedNodes(ctx, fc.Args["cluster"].(string)) }, nil, ec.marshalNCount2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐCountᚄ, @@ -9084,7 +9047,7 @@ func (ec *executionContext) _Query_node(ctx context.Context, field graphql.Colle ec.fieldContext_Query_node, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().Node(ctx, fc.Args["id"].(string)) + return ec.Resolvers.Query().Node(ctx, fc.Args["id"].(string)) }, nil, ec.marshalONode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode, @@ -9151,7 +9114,7 @@ func (ec *executionContext) _Query_nodes(ctx context.Context, field graphql.Coll ec.fieldContext_Query_nodes, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().Nodes(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["order"].(*model.OrderByInput)) + return ec.Resolvers.Query().Nodes(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["order"].(*model.OrderByInput)) }, nil, ec.marshalNNodeStateResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStateResultList, @@ -9198,7 +9161,7 @@ func (ec *executionContext) _Query_nodesWithMeta(ctx context.Context, field grap ec.fieldContext_Query_nodesWithMeta, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().NodesWithMeta(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["order"].(*model.OrderByInput)) + return ec.Resolvers.Query().NodesWithMeta(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["order"].(*model.OrderByInput)) }, nil, ec.marshalNNodeStateResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStateResultList, @@ -9245,7 +9208,7 @@ func (ec *executionContext) _Query_nodeStates(ctx context.Context, field graphql ec.fieldContext_Query_nodeStates, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().NodeStates(ctx, fc.Args["filter"].([]*model.NodeFilter)) + return ec.Resolvers.Query().NodeStates(ctx, fc.Args["filter"].([]*model.NodeFilter)) }, nil, ec.marshalNNodeStates2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesᚄ, @@ -9292,7 +9255,7 @@ func (ec *executionContext) _Query_nodeStatesTimed(ctx context.Context, field gr ec.fieldContext_Query_nodeStatesTimed, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().NodeStatesTimed(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["type"].(string)) + return ec.Resolvers.Query().NodeStatesTimed(ctx, fc.Args["filter"].([]*model.NodeFilter), fc.Args["type"].(string)) }, nil, ec.marshalNNodeStatesTimed2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesTimedᚄ, @@ -9341,7 +9304,7 @@ func (ec *executionContext) _Query_job(ctx context.Context, field graphql.Collec ec.fieldContext_Query_job, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().Job(ctx, fc.Args["id"].(string)) + return ec.Resolvers.Query().Job(ctx, fc.Args["id"].(string)) }, nil, ec.marshalOJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob, @@ -9436,7 +9399,7 @@ func (ec *executionContext) _Query_jobMetrics(ctx context.Context, field graphql ec.fieldContext_Query_jobMetrics, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().JobMetrics(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["resolution"].(*int)) + return ec.Resolvers.Query().JobMetrics(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["resolution"].(*int)) }, nil, ec.marshalNJobMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricWithNameᚄ, @@ -9485,7 +9448,7 @@ func (ec *executionContext) _Query_jobStats(ctx context.Context, field graphql.C ec.fieldContext_Query_jobStats, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().JobStats(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string)) + return ec.Resolvers.Query().JobStats(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string)) }, nil, ec.marshalNNamedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsᚄ, @@ -9532,7 +9495,7 @@ func (ec *executionContext) _Query_scopedJobStats(ctx context.Context, field gra ec.fieldContext_Query_scopedJobStats, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().ScopedJobStats(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string), fc.Args["scopes"].([]schema.MetricScope)) + return ec.Resolvers.Query().ScopedJobStats(ctx, fc.Args["id"].(string), fc.Args["metrics"].([]string), fc.Args["scopes"].([]schema.MetricScope)) }, nil, ec.marshalNNamedStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScopeᚄ, @@ -9581,7 +9544,7 @@ func (ec *executionContext) _Query_jobs(ctx context.Context, field graphql.Colle ec.fieldContext_Query_jobs, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().Jobs(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["page"].(*model.PageRequest), fc.Args["order"].(*model.OrderByInput)) + return ec.Resolvers.Query().Jobs(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["page"].(*model.PageRequest), fc.Args["order"].(*model.OrderByInput)) }, nil, ec.marshalNJobResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobResultList, @@ -9634,7 +9597,7 @@ func (ec *executionContext) _Query_jobsStatistics(ctx context.Context, field gra ec.fieldContext_Query_jobsStatistics, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().JobsStatistics(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate), fc.Args["numDurationBins"].(*string), fc.Args["numMetricBins"].(*int)) + return ec.Resolvers.Query().JobsStatistics(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate), fc.Args["numDurationBins"].(*string), fc.Args["numMetricBins"].(*int)) }, nil, ec.marshalNJobsStatistics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatisticsᚄ, @@ -9713,7 +9676,7 @@ func (ec *executionContext) _Query_jobsMetricStats(ctx context.Context, field gr ec.fieldContext_Query_jobsMetricStats, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().JobsMetricStats(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) + return ec.Resolvers.Query().JobsMetricStats(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) }, nil, ec.marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ, @@ -9776,7 +9739,7 @@ func (ec *executionContext) _Query_jobsFootprints(ctx context.Context, field gra ec.fieldContext_Query_jobsFootprints, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().JobsFootprints(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) + return ec.Resolvers.Query().JobsFootprints(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string)) }, nil, ec.marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprints, @@ -9823,7 +9786,7 @@ func (ec *executionContext) _Query_rooflineHeatmap(ctx context.Context, field gr ec.fieldContext_Query_rooflineHeatmap, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().RooflineHeatmap(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["rows"].(int), fc.Args["cols"].(int), fc.Args["minX"].(float64), fc.Args["minY"].(float64), fc.Args["maxX"].(float64), fc.Args["maxY"].(float64)) + return ec.Resolvers.Query().RooflineHeatmap(ctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["rows"].(int), fc.Args["cols"].(int), fc.Args["minX"].(float64), fc.Args["minY"].(float64), fc.Args["maxX"].(float64), fc.Args["maxY"].(float64)) }, nil, ec.marshalNFloat2ᚕᚕfloat64ᚄ, @@ -9864,7 +9827,7 @@ func (ec *executionContext) _Query_nodeMetrics(ctx context.Context, field graphq ec.fieldContext_Query_nodeMetrics, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().NodeMetrics(ctx, fc.Args["cluster"].(string), fc.Args["nodes"].([]string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) + return ec.Resolvers.Query().NodeMetrics(ctx, fc.Args["cluster"].(string), fc.Args["nodes"].([]string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) }, nil, ec.marshalNNodeMetrics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeMetricsᚄ, @@ -9915,7 +9878,7 @@ func (ec *executionContext) _Query_nodeMetricsList(ctx context.Context, field gr ec.fieldContext_Query_nodeMetricsList, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().NodeMetricsList(ctx, fc.Args["cluster"].(string), fc.Args["subCluster"].(string), fc.Args["stateFilter"].(string), fc.Args["nodeFilter"].(string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time), fc.Args["page"].(*model.PageRequest), fc.Args["resolution"].(*int)) + return ec.Resolvers.Query().NodeMetricsList(ctx, fc.Args["cluster"].(string), fc.Args["subCluster"].(string), fc.Args["stateFilter"].(string), fc.Args["nodeFilter"].(string), fc.Args["scopes"].([]schema.MetricScope), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time), fc.Args["page"].(*model.PageRequest), fc.Args["resolution"].(*int)) }, nil, ec.marshalNNodesResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodesResultList, @@ -9970,7 +9933,7 @@ func (ec *executionContext) _Query_clusterMetrics(ctx context.Context, field gra ec.fieldContext_Query_clusterMetrics, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.resolvers.Query().ClusterMetrics(ctx, fc.Args["cluster"].(string), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) + return ec.Resolvers.Query().ClusterMetrics(ctx, fc.Args["cluster"].(string), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) }, nil, ec.marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics, @@ -10017,7 +9980,7 @@ func (ec *executionContext) _Query___type(ctx context.Context, field graphql.Col ec.fieldContext_Query___type, func(ctx context.Context) (any, error) { fc := graphql.GetFieldContext(ctx) - return ec.introspectType(fc.Args["name"].(string)) + return ec.IntrospectType(fc.Args["name"].(string)) }, nil, ec.marshalO__Type2ᚖgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐType, @@ -10081,7 +10044,7 @@ func (ec *executionContext) _Query___schema(ctx context.Context, field graphql.C field, ec.fieldContext_Query___schema, func(ctx context.Context) (any, error) { - return ec.introspectSchema() + return ec.IntrospectSchema() }, nil, ec.marshalO__Schema2ᚖgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐSchema, @@ -10633,7 +10596,7 @@ func (ec *executionContext) _SubCluster_numberOfNodes(ctx context.Context, field field, ec.fieldContext_SubCluster_numberOfNodes, func(ctx context.Context) (any, error) { - return ec.resolvers.SubCluster().NumberOfNodes(ctx, obj) + return ec.Resolvers.SubCluster().NumberOfNodes(ctx, obj) }, nil, ec.marshalNInt2int, @@ -13274,7 +13237,6 @@ func (ec *executionContext) unmarshalInputFloatRange(ctx context.Context, obj an it.To = data } } - return it, nil } @@ -13308,7 +13270,6 @@ func (ec *executionContext) unmarshalInputIntRange(ctx context.Context, obj any) it.To = data } } - return it, nil } @@ -13482,7 +13443,6 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj any it.Node = data } } - return it, nil } @@ -13516,7 +13476,6 @@ func (ec *executionContext) unmarshalInputMetricStatItem(ctx context.Context, ob it.Range = data } } - return it, nil } @@ -13578,7 +13537,6 @@ func (ec *executionContext) unmarshalInputNodeFilter(ctx context.Context, obj an it.TimeStart = data } } - return it, nil } @@ -13623,7 +13581,6 @@ func (ec *executionContext) unmarshalInputOrderByInput(ctx context.Context, obj it.Order = data } } - return it, nil } @@ -13657,7 +13614,6 @@ func (ec *executionContext) unmarshalInputPageRequest(ctx context.Context, obj a it.Page = data } } - return it, nil } @@ -13719,7 +13675,6 @@ func (ec *executionContext) unmarshalInputStringInput(ctx context.Context, obj a it.In = data } } - return it, nil } @@ -13760,7 +13715,6 @@ func (ec *executionContext) unmarshalInputTimeRange(ctx context.Context, obj any it.To = data } } - return it, nil } @@ -13807,10 +13761,10 @@ func (ec *executionContext) _Accelerator(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -13887,10 +13841,10 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -13938,10 +13892,10 @@ func (ec *executionContext) _ClusterMetricWithName(ctx context.Context, sel ast. return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -13982,10 +13936,10 @@ func (ec *executionContext) _ClusterMetrics(ctx context.Context, sel ast.Selecti return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14026,10 +13980,10 @@ func (ec *executionContext) _ClusterSupport(ctx context.Context, sel ast.Selecti return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14070,10 +14024,10 @@ func (ec *executionContext) _Count(ctx context.Context, sel ast.SelectionSet, ob return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14119,10 +14073,10 @@ func (ec *executionContext) _EnergyFootprintValue(ctx context.Context, sel ast.S return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14168,10 +14122,10 @@ func (ec *executionContext) _FootprintValue(ctx context.Context, sel ast.Selecti return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14212,10 +14166,10 @@ func (ec *executionContext) _Footprints(ctx context.Context, sel ast.SelectionSe return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14268,10 +14222,10 @@ func (ec *executionContext) _GlobalMetricListItem(ctx context.Context, sel ast.S return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14312,10 +14266,10 @@ func (ec *executionContext) _HistoPoint(ctx context.Context, sel ast.SelectionSe return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14356,10 +14310,10 @@ func (ec *executionContext) _IntRangeOutput(ctx context.Context, sel ast.Selecti return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14722,10 +14676,10 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14766,10 +14720,10 @@ func (ec *executionContext) _JobLink(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14809,10 +14763,10 @@ func (ec *executionContext) _JobLinkResultList(ctx context.Context, sel ast.Sele return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14854,10 +14808,10 @@ func (ec *executionContext) _JobMetric(ctx context.Context, sel ast.SelectionSet return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14903,10 +14857,10 @@ func (ec *executionContext) _JobMetricWithName(ctx context.Context, sel ast.Sele return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -14950,10 +14904,10 @@ func (ec *executionContext) _JobResultList(ctx context.Context, sel ast.Selectio return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15028,10 +14982,10 @@ func (ec *executionContext) _JobStats(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15152,10 +15106,10 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15235,10 +15189,10 @@ func (ec *executionContext) _MetricConfig(ctx context.Context, sel ast.Selection return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15279,10 +15233,10 @@ func (ec *executionContext) _MetricFootprints(ctx context.Context, sel ast.Selec return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15324,10 +15278,10 @@ func (ec *executionContext) _MetricHistoPoint(ctx context.Context, sel ast.Selec return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15372,10 +15326,10 @@ func (ec *executionContext) _MetricHistoPoints(ctx context.Context, sel ast.Sele return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15421,10 +15375,10 @@ func (ec *executionContext) _MetricStatistics(ctx context.Context, sel ast.Selec return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15498,10 +15452,10 @@ func (ec *executionContext) _MetricValue(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15579,10 +15533,10 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15623,10 +15577,10 @@ func (ec *executionContext) _NamedStats(ctx context.Context, sel ast.SelectionSe return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15672,10 +15626,10 @@ func (ec *executionContext) _NamedStatsWithScope(ctx context.Context, sel ast.Se return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15906,10 +15860,10 @@ func (ec *executionContext) _Node(ctx context.Context, sel ast.SelectionSet, obj return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -15960,10 +15914,10 @@ func (ec *executionContext) _NodeMetrics(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16001,10 +15955,10 @@ func (ec *executionContext) _NodeStateResultList(ctx context.Context, sel ast.Se return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16045,10 +15999,10 @@ func (ec *executionContext) _NodeStates(ctx context.Context, sel ast.SelectionSe return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16094,10 +16048,10 @@ func (ec *executionContext) _NodeStatesTimed(ctx context.Context, sel ast.Select return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16143,10 +16097,10 @@ func (ec *executionContext) _NodesResultList(ctx context.Context, sel ast.Select return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16665,10 +16619,10 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16710,10 +16664,10 @@ func (ec *executionContext) _Resource(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16756,10 +16710,10 @@ func (ec *executionContext) _ScopedStats(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16804,10 +16758,10 @@ func (ec *executionContext) _Series(ctx context.Context, sel ast.SelectionSet, o return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16858,10 +16812,10 @@ func (ec *executionContext) _StatsSeries(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -16988,10 +16942,10 @@ func (ec *executionContext) _SubCluster(ctx context.Context, sel ast.SelectionSe return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17037,10 +16991,10 @@ func (ec *executionContext) _SubClusterConfig(ctx context.Context, sel ast.Selec return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17091,10 +17045,10 @@ func (ec *executionContext) _Tag(ctx context.Context, sel ast.SelectionSet, obj return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17137,10 +17091,10 @@ func (ec *executionContext) _TimeRangeOutput(ctx context.Context, sel ast.Select return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17186,10 +17140,10 @@ func (ec *executionContext) _TimeWeights(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17232,10 +17186,10 @@ func (ec *executionContext) _Topology(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17273,10 +17227,10 @@ func (ec *executionContext) _Unit(ctx context.Context, sel ast.SelectionSet, obj return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17322,10 +17276,10 @@ func (ec *executionContext) _User(ctx context.Context, sel ast.SelectionSet, obj return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17378,10 +17332,10 @@ func (ec *executionContext) ___Directive(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17426,10 +17380,10 @@ func (ec *executionContext) ___EnumValue(ctx context.Context, sel ast.SelectionS return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17484,10 +17438,10 @@ func (ec *executionContext) ___Field(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17539,10 +17493,10 @@ func (ec *executionContext) ___InputValue(ctx context.Context, sel ast.Selection return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17594,10 +17548,10 @@ func (ec *executionContext) ___Schema(ctx context.Context, sel ast.SelectionSet, return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17653,10 +17607,10 @@ func (ec *executionContext) ___Type(ctx context.Context, sel ast.SelectionSet, o return graphql.Null } - atomic.AddInt32(&ec.deferred, int32(len(deferred))) + atomic.AddInt32(&ec.Deferred, int32(len(deferred))) for label, dfs := range deferred { - ec.processDeferredGroup(graphql.DeferredGroup{ + ec.ProcessDeferredGroup(graphql.DeferredGroup{ Label: label, Path: graphql.GetPath(ctx), FieldSet: dfs, @@ -17698,39 +17652,11 @@ func (ec *executionContext) marshalNBoolean2bool(ctx context.Context, sel ast.Se } func (ec *executionContext) marshalNCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Cluster) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐCluster(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐCluster(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -17752,39 +17678,11 @@ func (ec *executionContext) marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋc } func (ec *executionContext) marshalNClusterMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.ClusterMetricWithName) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -17824,39 +17722,11 @@ func (ec *executionContext) marshalNClusterSupport2githubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupportᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.ClusterSupport) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupport(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐClusterSupport(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -17868,39 +17738,11 @@ func (ec *executionContext) marshalNClusterSupport2ᚕgithubᚗcomᚋClusterCock } func (ec *executionContext) marshalNCount2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐCountᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.Count) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNCount2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐCount(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNCount2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐCount(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18003,39 +17845,11 @@ func (ec *executionContext) unmarshalNFloatRange2ᚖgithubᚗcomᚋClusterCockpi } func (ec *executionContext) marshalNGlobalMetricListItem2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItemᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.GlobalMetricListItem) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItem(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐGlobalMetricListItem(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18057,39 +17871,11 @@ func (ec *executionContext) marshalNGlobalMetricListItem2ᚖgithubᚗcomᚋClust } func (ec *executionContext) marshalNHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.HistoPoint) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNHistoPoint2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPoint(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNHistoPoint2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPoint(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18325,39 +18111,11 @@ func (ec *executionContext) marshalNInt2ᚖint(ctx context.Context, sel ast.Sele } func (ec *executionContext) marshalNJob2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJobᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Job) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐJob(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18399,39 +18157,11 @@ func (ec *executionContext) unmarshalNJobFilter2ᚖgithubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNJobLink2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobLinkᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobLink) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobLink2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobLink(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNJobLink2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobLink(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18463,39 +18193,11 @@ func (ec *executionContext) marshalNJobMetric2ᚖgithubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNJobMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobMetricWithName) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricWithName(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNJobMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobMetricWithName(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18541,39 +18243,11 @@ func (ec *executionContext) marshalNJobState2githubᚗcomᚋClusterCockpitᚋcc } func (ec *executionContext) marshalNJobStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobStats) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStats(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobStats(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18595,39 +18269,11 @@ func (ec *executionContext) marshalNJobStats2ᚖgithubᚗcomᚋClusterCockpitᚋ } func (ec *executionContext) marshalNJobsStatistics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatisticsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.JobsStatistics) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNJobsStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatistics(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNJobsStatistics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobsStatistics(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18649,39 +18295,11 @@ func (ec *executionContext) marshalNJobsStatistics2ᚖgithubᚗcomᚋClusterCock } func (ec *executionContext) marshalNMetricConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.MetricConfig) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18703,39 +18321,11 @@ func (ec *executionContext) marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpi } func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricFootprintsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.MetricFootprints) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricFootprints(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNMetricFootprints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricFootprints(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18767,39 +18357,11 @@ func (ec *executionContext) marshalNMetricHistoPoint2ᚖgithubᚗcomᚋClusterCo } func (ec *executionContext) marshalNMetricHistoPoints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPointsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.MetricHistoPoints) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNMetricHistoPoints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoints(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNMetricHistoPoints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoints(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18866,39 +18428,11 @@ func (ec *executionContext) marshalNMonitoringState2string(ctx context.Context, } func (ec *executionContext) marshalNNamedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NamedStats) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNamedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStats(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNamedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStats(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18920,39 +18454,11 @@ func (ec *executionContext) marshalNNamedStats2ᚖgithubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNNamedStatsWithScope2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScopeᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NamedStatsWithScope) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScope(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNamedStatsWithScope(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -18974,39 +18480,11 @@ func (ec *executionContext) marshalNNamedStatsWithScope2ᚖgithubᚗcomᚋCluste } func (ec *executionContext) marshalNNode2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNodeᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Node) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNode2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐNode(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19033,39 +18511,11 @@ func (ec *executionContext) unmarshalNNodeFilter2ᚖgithubᚗcomᚋClusterCockpi } func (ec *executionContext) marshalNNodeMetrics2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeMetricsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NodeMetrics) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNodeMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeMetrics(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNodeMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeMetrics(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19101,39 +18551,11 @@ func (ec *executionContext) marshalNNodeStateResultList2ᚖgithubᚗcomᚋCluste } func (ec *executionContext) marshalNNodeStates2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NodeStates) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNodeStates2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStates(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNodeStates2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStates(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19155,39 +18577,11 @@ func (ec *executionContext) marshalNNodeStates2ᚖgithubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNNodeStatesTimed2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesTimedᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.NodeStatesTimed) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNNodeStatesTimed2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesTimed(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNNodeStatesTimed2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐNodeStatesTimed(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19263,39 +18657,11 @@ func (ec *executionContext) marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockp } func (ec *executionContext) marshalNResource2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResourceᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Resource) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResource(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNResource2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐResource(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19334,39 +18700,11 @@ func (ec *executionContext) marshalNSchedulerState2githubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNScopedStats2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStatsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.ScopedStats) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNScopedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStats(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNScopedStats2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐScopedStats(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19448,39 +18786,11 @@ func (ec *executionContext) marshalNString2ᚕstringᚄ(ctx context.Context, sel } func (ec *executionContext) marshalNSubCluster2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubCluster) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubCluster(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubCluster(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19502,39 +18812,11 @@ func (ec *executionContext) marshalNSubCluster2ᚖgithubᚗcomᚋClusterCockpit } func (ec *executionContext) marshalNSubClusterConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.SubClusterConfig) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfig(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNSubClusterConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSubClusterConfig(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19560,39 +18842,11 @@ func (ec *executionContext) marshalNTag2githubᚗcomᚋClusterCockpitᚋccᚑlib } func (ec *executionContext) marshalNTag2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTagᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.Tag) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNTag2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐTag(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19674,39 +18928,11 @@ func (ec *executionContext) marshalN__Directive2githubᚗcomᚋ99designsᚋgqlge } func (ec *executionContext) marshalN__Directive2ᚕgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐDirectiveᚄ(ctx context.Context, sel ast.SelectionSet, v []introspection.Directive) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__Directive2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐDirective(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__Directive2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐDirective(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19749,39 +18975,11 @@ func (ec *executionContext) unmarshalN__DirectiveLocation2ᚕstringᚄ(ctx conte } func (ec *executionContext) marshalN__DirectiveLocation2ᚕstringᚄ(ctx context.Context, sel ast.SelectionSet, v []string) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__DirectiveLocation2string(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__DirectiveLocation2string(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19805,39 +19003,11 @@ func (ec *executionContext) marshalN__InputValue2githubᚗcomᚋ99designsᚋgqlg } func (ec *executionContext) marshalN__InputValue2ᚕgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐInputValueᚄ(ctx context.Context, sel ast.SelectionSet, v []introspection.InputValue) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__InputValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐInputValue(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__InputValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐInputValue(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19853,39 +19023,11 @@ func (ec *executionContext) marshalN__Type2githubᚗcomᚋ99designsᚋgqlgenᚋg } func (ec *executionContext) marshalN__Type2ᚕgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐTypeᚄ(ctx context.Context, sel ast.SelectionSet, v []introspection.Type) graphql.Marshaler { - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__Type2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐType(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__Type2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐType(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -19926,39 +19068,11 @@ func (ec *executionContext) marshalOAccelerator2ᚕᚖgithubᚗcomᚋClusterCock if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAccelerator(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐAccelerator(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20037,39 +19151,11 @@ func (ec *executionContext) marshalOEnergyFootprintValue2ᚕᚖgithubᚗcomᚋCl if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalOEnergyFootprintValue2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐEnergyFootprintValue(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalOEnergyFootprintValue2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐEnergyFootprintValue(ctx, sel, v[i]) + }) return ret } @@ -20104,39 +19190,11 @@ func (ec *executionContext) marshalOFootprintValue2ᚕᚖgithubᚗcomᚋClusterC if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalOFootprintValue2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprintValue(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalOFootprintValue2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐFootprintValue(ctx, sel, v[i]) + }) return ret } @@ -20409,39 +19467,11 @@ func (ec *executionContext) marshalOMetricHistoPoint2ᚕᚖgithubᚗcomᚋCluste if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNMetricHistoPoint2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoint(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNMetricHistoPoint2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoint(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20592,39 +19622,11 @@ func (ec *executionContext) marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋcc if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeries(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalNSeries2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐSeries(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20780,39 +19782,11 @@ func (ec *executionContext) marshalO__EnumValue2ᚕgithubᚗcomᚋ99designsᚋgq if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__EnumValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐEnumValue(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__EnumValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐEnumValue(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20827,39 +19801,11 @@ func (ec *executionContext) marshalO__Field2ᚕgithubᚗcomᚋ99designsᚋgqlgen if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__Field2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐField(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__Field2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐField(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20874,39 +19820,11 @@ func (ec *executionContext) marshalO__InputValue2ᚕgithubᚗcomᚋ99designsᚋg if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__InputValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐInputValue(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__InputValue2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐInputValue(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { @@ -20928,39 +19846,11 @@ func (ec *executionContext) marshalO__Type2ᚕgithubᚗcomᚋ99designsᚋgqlgen if v == nil { return graphql.Null } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalN__Type2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐType(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() + ret := graphql.MarshalSliceConcurrently(ctx, len(v), 0, false, func(ctx context.Context, i int) graphql.Marshaler { + fc := graphql.GetFieldContext(ctx) + fc.Result = &v[i] + return ec.marshalN__Type2githubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐType(ctx, sel, v[i]) + }) for _, e := range ret { if e == graphql.Null { diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 0d56b02c..4c398ee3 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -3,7 +3,7 @@ package graph // This file will be automatically regenerated based on the schema, any resolver // implementations // will be copied through when generating and any unknown code will be moved to the end. -// Code generated by github.com/99designs/gqlgen version v0.17.85 +// Code generated by github.com/99designs/gqlgen version v0.17.87 import ( "context" diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index ea4bdcc0..6062ee54 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@rollup/plugin-replace": "^6.0.3", "@sveltestrap/sveltestrap": "^7.1.0", - "@urql/svelte": "^5.0.0", + "@urql/svelte": "^4.2.3", "chart.js": "^4.5.1", "date-fns": "^4.1.0", "graphql": "^16.13.0", @@ -27,7 +27,7 @@ "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.53.6" + "svelte": "^5.53.7" } }, "node_modules/@0no-co/graphql.web": { @@ -655,9 +655,9 @@ "license": "MIT" }, "node_modules/@urql/core": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@urql/core/-/core-6.0.1.tgz", - "integrity": "sha512-FZDiQk6jxbj5hixf2rEPv0jI+IZz0EqqGW8mJBEug68/zHTtT+f34guZDmyjJZyiWbj0vL165LoMr/TkeDHaug==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.2.0.tgz", + "integrity": "sha512-/n0ieD0mvvDnVAXEQgX/7qJiVcvYvNkOHeBvkwtylfjydar123caCXcl58PXFY11oU1oquJocVXHxLAbtv4x1A==", "license": "MIT", "dependencies": { "@0no-co/graphql.web": "^1.0.13", @@ -665,16 +665,16 @@ } }, "node_modules/@urql/svelte": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-5.0.0.tgz", - "integrity": "sha512-tHYEyFZwWsBW9GfpXbK+GImWhyZO1TJkhHsquosza0D0qOZyL+wGp/qT74WPUBJaF4gkUSXOQtUidDI7uvnuoQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@urql/svelte/-/svelte-4.2.3.tgz", + "integrity": "sha512-v3eArfymhdjaM5VQFp3QZxq9veYPadmDfX7ueid/kD4DlRplIycPakJ2FrKigh46SXa5mWqJ3QWuWyRKVu61sw==", "license": "MIT", "dependencies": { - "@urql/core": "^6.0.0", + "@urql/core": "^5.1.1", "wonka": "^6.3.2" }, "peerDependencies": { - "@urql/core": "^6.0.0", + "@urql/core": "^5.0.0", "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" } }, @@ -1224,9 +1224,9 @@ } }, "node_modules/svelte": { - "version": "5.53.6", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.6.tgz", - "integrity": "sha512-lP5DGF3oDDI9fhHcSpaBiJEkFLuS16h92DhM1L5K1lFm0WjOmUh1i2sNkBBk8rkxJRpob0dBE75jRfUzGZUOGA==", + "version": "5.53.7", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.7.tgz", + "integrity": "sha512-uxck1KI7JWtlfP3H6HOWi/94soAl23jsGJkBzN2BAWcQng0+lTrRNhxActFqORgnO9BHVd1hKJhG+ljRuIUWfQ==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", diff --git a/web/frontend/package.json b/web/frontend/package.json index 077126ba..5a81042a 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -14,12 +14,12 @@ "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.53.6" + "svelte": "^5.53.7" }, "dependencies": { "@rollup/plugin-replace": "^6.0.3", "@sveltestrap/sveltestrap": "^7.1.0", - "@urql/svelte": "^5.0.0", + "@urql/svelte": "^4.2.3", "chart.js": "^4.5.1", "date-fns": "^4.1.0", "graphql": "^16.13.0", From 84fda9c8e2a023964d10ebac3837758902424880 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 3 Mar 2026 19:08:16 +0100 Subject: [PATCH 321/341] optimize resource filter by only using range query if required, review filter component --- internal/routerConfig/routes.go | 60 ++++++++-- web/frontend/src/generic/Filters.svelte | 52 ++++++-- .../src/generic/filters/Resources.svelte | 112 ++++++++++++++---- .../generic/select/DoubleRangeSlider.svelte | 24 +++- 4 files changed, 197 insertions(+), 51 deletions(-) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index b6137ddb..218384e0 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -305,30 +305,66 @@ func buildFilterPresets(query url.Values) map[string]any { if query.Get("numNodes") != "" { parts := strings.Split(query.Get("numNodes"), "-") if len(parts) == 2 { - a, e1 := strconv.Atoi(parts[0]) - b, e2 := strconv.Atoi(parts[1]) - if e1 == nil && e2 == nil { - filterPresets["numNodes"] = map[string]int{"from": a, "to": b} + if parts[0] == "lessthan" { + lt, lte := strconv.Atoi(parts[1]) + if lte == nil { + filterPresets["numNodes"] = map[string]int{"from": 1, "to": lt} + } + } else if parts[0] == "morethan" { + mt, mte := strconv.Atoi(parts[1]) + if mte == nil { + filterPresets["numNodes"] = map[string]int{"from": mt, "to": 0} + } + } else { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["numNodes"] = map[string]int{"from": a, "to": b} + } } } } if query.Get("numHWThreads") != "" { parts := strings.Split(query.Get("numHWThreads"), "-") if len(parts) == 2 { - a, e1 := strconv.Atoi(parts[0]) - b, e2 := strconv.Atoi(parts[1]) - if e1 == nil && e2 == nil { - filterPresets["numHWThreads"] = map[string]int{"from": a, "to": b} + if parts[0] == "lessthan" { + lt, lte := strconv.Atoi(parts[1]) + if lte == nil { + filterPresets["numHWThreads"] = map[string]int{"from": 1, "to": lt} + } + } else if parts[0] == "morethan" { + mt, mte := strconv.Atoi(parts[1]) + if mte == nil { + filterPresets["numHWThreads"] = map[string]int{"from": mt, "to": 0} + } + } else { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["numHWThreads"] = map[string]int{"from": a, "to": b} + } } } } if query.Get("numAccelerators") != "" { parts := strings.Split(query.Get("numAccelerators"), "-") if len(parts) == 2 { - a, e1 := strconv.Atoi(parts[0]) - b, e2 := strconv.Atoi(parts[1]) - if e1 == nil && e2 == nil { - filterPresets["numAccelerators"] = map[string]int{"from": a, "to": b} + if parts[0] == "lessthan" { + lt, lte := strconv.Atoi(parts[1]) + if lte == nil { + filterPresets["numAccelerators"] = map[string]int{"from": 1, "to": lt} + } + } else if parts[0] == "morethan" { + mt, mte := strconv.Atoi(parts[1]) + if mte == nil { + filterPresets["numAccelerators"] = map[string]int{"from": mt, "to": 0} + } + } else { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["numAccelerators"] = map[string]int{"from": a, "to": b} + } } } } diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index 4031d0a5..162082c0 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -280,12 +280,24 @@ opts.push(`duration=morethan-${filters.duration.moreThan}`); if (filters.tags.length != 0) for (let tag of filters.tags) opts.push(`tag=${tag}`); - if (filters.numNodes.from && filters.numNodes.to) + if (filters.numNodes.from > 1 && filters.numNodes.to > 0) opts.push(`numNodes=${filters.numNodes.from}-${filters.numNodes.to}`); - if (filters.numHWThreads.from && filters.numHWThreads.to) + else if (filters.numNodes.from > 1 && filters.numNodes.to == 0) + opts.push(`numNodes=morethan-${filters.numNodes.from}`); + else if (filters.numNodes.from == 1 && filters.numNodes.to > 0) + opts.push(`numNodes=lessthan-${filters.numNodes.to}`); + if (filters.numHWThreads.from > 1 && filters.numHWThreads.to > 0) opts.push(`numHWThreads=${filters.numHWThreads.from}-${filters.numHWThreads.to}`); + else if (filters.numHWThreads.from > 1 && filters.numHWThreads.to == 0) + opts.push(`numHWThreads=morethan-${filters.numHWThreads.from}`); + else if (filters.numHWThreads.from == 1 && filters.numHWThreads.to > 0) + opts.push(`numHWThreads=lessthan-${filters.numHWThreads.to}`); if (filters.numAccelerators.from && filters.numAccelerators.to) opts.push(`numAccelerators=${filters.numAccelerators.from}-${filters.numAccelerators.to}`); + else if (filters.numAccelerators.from > 1 && filters.numAccelerators.to == 0) + opts.push(`numAccelerators=morethan-${filters.numAccelerators.from}`); + else if (filters.numAccelerators.from == 1 && filters.numAccelerators.to > 0) + opts.push(`numAccelerators=lessthan-${filters.numAccelerators.to}`); if (filters.node) opts.push(`node=${filters.node}`); if (filters.node && filters.nodeMatch != "eq") // "eq" is default-case opts.push(`nodeMatch=${filters.nodeMatch}`); @@ -490,21 +502,45 @@ {/if} - {#if filters.numNodes.from != null || filters.numNodes.to != null} + {#if filters.numNodes.from > 1 && filters.numNodes.to > 0} (isResourcesOpen = true)}> - Nodes: {filters.numNodes.from} - {filters.numNodes.to} + Nodes: {filters.numNodes.from} - {filters.numNodes.to} + + {:else if filters.numNodes.from > 1 && filters.numNodes.to == 0} + (isResourcesOpen = true)}> +  ≥ {filters.numNodes.from} Node(s) + + {:else if filters.numNodes.from == 1 && filters.numNodes.to > 0} + (isResourcesOpen = true)}> +  ≤ {filters.numNodes.to} Node(s) {/if} - {#if filters.numHWThreads.from != null || filters.numHWThreads.to != null} + {#if filters.numHWThreads.from > 1 && filters.numHWThreads.to > 0} (isResourcesOpen = true)}> - HWThreads: {filters.numHWThreads.from} - {filters.numHWThreads.to} + HWThreads: {filters.numHWThreads.from} - {filters.numHWThreads.to} + + {:else if filters.numHWThreads.from > 1 && filters.numHWThreads.to == 0} + (isResourcesOpen = true)}> +  ≥ {filters.numHWThreads.from} HWThread(s) + + {:else if filters.numHWThreads.from == 1 && filters.numHWThreads.to > 0} + (isResourcesOpen = true)}> +  ≤ {filters.numHWThreads.to} HWThread(s) {/if} - {#if filters.numAccelerators.from != null || filters.numAccelerators.to != null} + {#if filters.numAccelerators.from > 1 && filters.numAccelerators.to > 0} (isResourcesOpen = true)}> - Accelerators: {filters.numAccelerators.from} - {filters.numAccelerators.to} + Accelerators: {filters.numAccelerators.from} - {filters.numAccelerators.to} + + {:else if filters.numAccelerators.from > 1 && filters.numAccelerators.to == 0} + (isResourcesOpen = true)}> +  ≥ {filters.numAccelerators.from} Acc(s) + + {:else if filters.numAccelerators.from == 1 && filters.numAccelerators.to > 0} + (isResourcesOpen = true)}> +  ≤ {filters.numAccelerators.to} Acc(s) {/if} diff --git a/web/frontend/src/generic/filters/Resources.svelte b/web/frontend/src/generic/filters/Resources.svelte index d8cdbc65..be55bcc3 100644 --- a/web/frontend/src/generic/filters/Resources.svelte +++ b/web/frontend/src/generic/filters/Resources.svelte @@ -20,7 +20,9 @@ ModalBody, ModalHeader, ModalFooter, - Input + Input, + Tooltip, + Icon } from "@sveltestrap/sveltestrap"; import DoubleRangeSlider from "../select/DoubleRangeSlider.svelte"; @@ -42,6 +44,19 @@ contains: "Contains", } + const findMaxNumNodes = (infos) => + infos.reduce( + (max, cluster) => + Math.max( + max, + cluster.subClusters.reduce( + (max, sc) => Math.max(max, sc.numberOfNodes || 0), + 0, + ), + ), + 0, + ); + const findMaxNumAccels = (infos) => infos.reduce( (max, cluster) => @@ -75,29 +90,46 @@ /* State Init*/ // Counts - let minNumNodes = $state(1); - let maxNumNodes = $state(128); - let maxNumHWThreads = $state(0); - let maxNumAccelerators = $state(0); + let maxNumNodes = $state(1); + let maxNumHWThreads = $state(1); + let maxNumAccelerators = $state(1); /* Derived States */ // Pending - let pendingNumNodes = $derived(presetNumNodes); - let pendingNumHWThreads = $derived(presetNumHWThreads); - let pendingNumAccelerators = $derived(presetNumAccelerators); + let pendingNumNodes = $derived({ + from: presetNumNodes.from, + to: (presetNumNodes.to == 0) ? maxNumNodes : presetNumNodes.to + }); + let pendingNumHWThreads = $derived({ + from: presetNumHWThreads.from, + to: (presetNumHWThreads.to == 0) ? maxNumHWThreads : presetNumHWThreads.to + }); + let pendingNumAccelerators = $derived({ + from: presetNumAccelerators.from, + to: (presetNumAccelerators.to == 0) ? maxNumAccelerators : presetNumAccelerators.to + }); let pendingNamedNode = $derived(presetNamedNode); let pendingNodeMatch = $derived(presetNodeMatch); // Changable States - let nodesState = $derived(presetNumNodes); - let threadState = $derived(presetNumHWThreads); - let accState = $derived(presetNumAccelerators); + let nodesState = $derived({ + from: presetNumNodes.from, + to: (presetNumNodes.to == 0) ? maxNumNodes : presetNumNodes.to + }); + let threadState = $derived({ + from: presetNumHWThreads.from, + to: (presetNumHWThreads.to == 0) ? maxNumHWThreads : presetNumHWThreads.to + }); + let accState = $derived({ + from: presetNumAccelerators.from, + to: (presetNumAccelerators.to == 0) ? maxNumAccelerators : presetNumAccelerators.to + }); const initialized = $derived(getContext("initialized") || false); const clusterInfos = $derived($initialized ? getContext("clusters") : null); // Is Selection Active const nodesActive = $derived(!(JSON.stringify(nodesState) === JSON.stringify({ from: 1, to: maxNumNodes }))); const threadActive = $derived(!(JSON.stringify(threadState) === JSON.stringify({ from: 1, to: maxNumHWThreads }))); - const accActive = $derived(!(JSON.stringify(accState) === JSON.stringify({ from: 0, to: maxNumAccelerators }))); + const accActive = $derived(!(JSON.stringify(accState) === JSON.stringify({ from: 1, to: maxNumAccelerators }))); // Block Apply if null const disableApply = $derived( nodesState.from === null || nodesState.to === null || @@ -110,11 +142,13 @@ if ($initialized) { if (activeCluster != null) { const { subClusters } = clusterInfos.find((c) => c.name == activeCluster); - maxNumAccelerators = findMaxNumAccels([{ subClusters }]); + maxNumNodes = findMaxNumNodes([{ subClusters }]); maxNumHWThreads = findMaxNumHWThreadsPerNode([{ subClusters }]); + maxNumAccelerators = findMaxNumAccels([{ subClusters }]); } else if (clusterInfos.length > 0) { - maxNumAccelerators = findMaxNumAccels(clusterInfos); + maxNumNodes = findMaxNumNodes(clusterInfos); maxNumHWThreads = findMaxNumHWThreadsPerNode(clusterInfos); + maxNumAccelerators = findMaxNumAccels(clusterInfos); } } }); @@ -145,26 +179,35 @@ pendingNumAccelerators.from == null && pendingNumAccelerators.to == null ) { - accState = { from: 0, to: maxNumAccelerators }; + accState = { from: 1, to: maxNumAccelerators }; } }); /* Functions */ function setResources() { if (nodesActive) { - pendingNumNodes = {...nodesState}; + pendingNumNodes = { + from: nodesState.from, + to: (nodesState.to == maxNumNodes) ? 0 : nodesState.to + }; } else { - pendingNumNodes = { from: null, to: null }; + pendingNumNodes = { from: null, to: null}; }; if (threadActive) { - pendingNumHWThreads = {...threadState}; + pendingNumHWThreads = { + from: threadState.from, + to: (threadState.to == maxNumHWThreads) ? 0 : threadState.to + }; } else { - pendingNumHWThreads = { from: null, to: null }; + pendingNumHWThreads = { from: null, to: null}; }; if (accActive) { - pendingNumAccelerators = {...accState}; + pendingNumAccelerators = { + from: accState.from, + to: (accState.to == maxNumAccelerators) ? 0 : accState.to + }; } else { - pendingNumAccelerators = { from: null, to: null }; + pendingNumAccelerators = { from: null, to: null}; }; }; @@ -195,13 +238,18 @@
    -
    Number of Nodes
    +
    Number of Nodes + +
    + + Preset maximum is for whole cluster. + { nodesState.from = detail[0]; nodesState.to = detail[1]; }} - sliderMin={minNumNodes} + sliderMin={1} sliderMax={maxNumNodes} fromPreset={nodesState.from} toPreset={nodesState.to} @@ -209,7 +257,13 @@
    -
    Number of HWThreads (Use for Single-Node Jobs)
    +
    + Number of HWThreads + +
    + + Presets for a single node. Can be changed to higher values. + { threadState.from = detail[0]; @@ -223,13 +277,19 @@
    {#if maxNumAccelerators != null && maxNumAccelerators > 1}
    -
    Number of Accelerators
    +
    + Number of Accelerators + +
    + + Presets for a single node. Can be changed to higher values. + { accState.from = detail[0]; accState.to = detail[1]; }} - sliderMin={0} + sliderMin={1} sliderMax={maxNumAccelerators} fromPreset={accState.from} toPreset={accState.to} diff --git a/web/frontend/src/generic/select/DoubleRangeSlider.svelte b/web/frontend/src/generic/select/DoubleRangeSlider.svelte index 12ca7449..c655087f 100644 --- a/web/frontend/src/generic/select/DoubleRangeSlider.svelte +++ b/web/frontend/src/generic/select/DoubleRangeSlider.svelte @@ -34,8 +34,8 @@ let pendingValues = $derived([fromPreset, toPreset]); let sliderFrom = $derived(Math.max(((fromPreset == null ? sliderMin : fromPreset) - sliderMin) / (sliderMax - sliderMin), 0.)); let sliderTo = $derived(Math.min(((toPreset == null ? sliderMin : toPreset) - sliderMin) / (sliderMax - sliderMin), 1.)); - let inputFieldFrom = $derived(fromPreset.toString()); - let inputFieldTo = $derived(toPreset.toString()); + let inputFieldFrom = $derived(fromPreset ? fromPreset.toString() : null); + let inputFieldTo = $derived(toPreset ? toPreset.toString() : null); /* Var Init */ let timeoutId = null; @@ -160,12 +160,26 @@
    inputChanged(e, 'from')} /> + oninput={(e) => { + inputChanged(e, 'from'); + }} + /> - Full Range: {sliderMin} - {sliderMax} + {#if inputFieldFrom != "1" && inputFieldTo != sliderMax?.toString() } + Selected: Range {inputFieldFrom} - {inputFieldTo} + {:else if inputFieldFrom != "1" && inputFieldTo == sliderMax?.toString() } + Selected: More than {inputFieldFrom} + {:else if inputFieldFrom == "1" && inputFieldTo != sliderMax?.toString() } + Selected: Less than {inputFieldTo} + {:else} + No Selection + {/if} inputChanged(e, 'to')} /> + oninput={(e) => { + inputChanged(e, 'to'); + }} + />
    From 39635ea123f9b2d2ae9771e3e02c129a7a532477 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 10:37:43 +0100 Subject: [PATCH 322/341] Cleanup metricstore options Entire-Checkpoint: 2f9a4e1c2e87 --- configs/config-demo.json | 2 -- configs/config.json | 2 -- pkg/metricstore/archive.go | 8 ++------ pkg/metricstore/checkpoint.go | 16 ++++++---------- pkg/metricstore/config.go | 29 ++++++++++++----------------- pkg/metricstore/configSchema.go | 23 +++++++---------------- 6 files changed, 27 insertions(+), 53 deletions(-) diff --git a/configs/config-demo.json b/configs/config-demo.json index 50dff298..4a69ecd2 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -21,8 +21,6 @@ ], "metric-store": { "checkpoints": { - "file-format": "wal", - "interval": "12h" }, "retention-in-memory": "48h", "memory-cap": 100 diff --git a/configs/config.json b/configs/config.json index c2361a1c..f880b8d8 100644 --- a/configs/config.json +++ b/configs/config.json @@ -74,14 +74,12 @@ ], "metric-store": { "checkpoints": { - "interval": "12h", "directory": "./var/checkpoints" }, "memory-cap": 100, "retention-in-memory": "48h", "cleanup": { "mode": "archive", - "interval": "48h", "directory": "./var/archive" }, "nats-subscriptions": [ diff --git a/pkg/metricstore/archive.go b/pkg/metricstore/archive.go index 77f4264a..916736d0 100644 --- a/pkg/metricstore/archive.go +++ b/pkg/metricstore/archive.go @@ -24,19 +24,15 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) { if Keys.Cleanup.Mode == "archive" { // Run as Archiver cleanUpWorker(wg, ctx, - Keys.Cleanup.Interval, + Keys.RetentionInMemory, "archiving", Keys.Cleanup.RootDir, false, ) } else { - if Keys.Cleanup.Interval == "" { - Keys.Cleanup.Interval = Keys.RetentionInMemory - } - // Run as Deleter cleanUpWorker(wg, ctx, - Keys.Cleanup.Interval, + Keys.RetentionInMemory, "deleting", "", true, diff --git a/pkg/metricstore/checkpoint.go b/pkg/metricstore/checkpoint.go index 45b2bc2a..ba1f7ba0 100644 --- a/pkg/metricstore/checkpoint.go +++ b/pkg/metricstore/checkpoint.go @@ -86,9 +86,11 @@ var ( // Checkpointing starts a background worker that periodically saves metric data to disk. // +// Checkpoints are written every 12 hours (hardcoded). +// // Format behaviour: -// - "json": Periodic checkpointing based on Keys.Checkpoints.Interval -// - "wal": Periodic binary snapshots + WAL rotation at Keys.Checkpoints.Interval +// - "json": Periodic checkpointing every checkpointInterval +// - "wal": Periodic binary snapshots + WAL rotation every checkpointInterval func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { lastCheckpointMu.Lock() lastCheckpoint = time.Now() @@ -98,14 +100,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { wg.Go(func() { - d, err := time.ParseDuration(Keys.Checkpoints.Interval) - if err != nil { - cclog.Fatalf("[METRICSTORE]> invalid checkpoint interval '%s': %s", Keys.Checkpoints.Interval, err.Error()) - } - if d <= 0 { - cclog.Warnf("[METRICSTORE]> checkpoint interval is zero or negative (%s), checkpointing disabled", d) - return - } + const checkpointInterval = 12 * time.Hour + d := checkpointInterval ticker := time.NewTicker(d) defer ticker.Stop() diff --git a/pkg/metricstore/config.go b/pkg/metricstore/config.go index 53716967..3b6be529 100644 --- a/pkg/metricstore/config.go +++ b/pkg/metricstore/config.go @@ -11,15 +11,13 @@ // // MetricStoreConfig (Keys) // ├─ NumWorkers: Parallel checkpoint/archive workers -// ├─ RetentionInMemory: How long to keep data in RAM +// ├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval) // ├─ MemoryCap: Memory limit in bytes (triggers forceFree) // ├─ Checkpoints: Persistence configuration -// │ ├─ FileFormat: "json" or "wal" -// │ ├─ Interval: How often to save (e.g., "1h") +// │ ├─ FileFormat: "json" or "wal" (default: "wal") // │ └─ RootDir: Checkpoint storage path -// ├─ Cleanup: Long-term storage configuration -// │ ├─ Interval: How often to delete/archive -// │ ├─ RootDir: Archive storage path +// ├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory) +// │ ├─ RootDir: Archive storage path (archive mode only) // │ └─ Mode: "delete" or "archive" // ├─ Debug: Development/debugging options // └─ Subscriptions: NATS topic subscriptions for metric ingestion @@ -61,12 +59,10 @@ const ( // Checkpoints configures periodic persistence of in-memory metric data. // // Fields: -// - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe) -// - Interval: Duration string (e.g., "1h", "30m") between checkpoint saves +// - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal" // - RootDir: Filesystem path for checkpoint files (created if missing) type Checkpoints struct { FileFormat string `json:"file-format"` - Interval string `json:"interval"` RootDir string `json:"directory"` } @@ -80,18 +76,17 @@ type Debug struct { EnableGops bool `json:"gops"` } -// Archive configures long-term storage of old metric data. +// Cleanup configures long-term storage of old metric data. // // Data older than RetentionInMemory is archived to disk or deleted. +// The cleanup interval is always RetentionInMemory. // // Fields: -// - ArchiveInterval: Duration string (e.g., "24h") between archive operations -// - RootDir: Filesystem path for archived data (created if missing) -// - DeleteInstead: If true, delete old data instead of archiving (saves disk space) +// - RootDir: Filesystem path for archived data (used in "archive" mode) +// - Mode: "delete" (discard old data) or "archive" (write to RootDir) type Cleanup struct { - Interval string `json:"interval"` - RootDir string `json:"directory"` - Mode string `json:"mode"` + RootDir string `json:"directory"` + Mode string `json:"mode"` } // Subscriptions defines NATS topics to subscribe to for metric ingestion. @@ -141,7 +136,7 @@ type MetricStoreConfig struct { // Accessed by Init(), Checkpointing(), and other lifecycle functions. var Keys MetricStoreConfig = MetricStoreConfig{ Checkpoints: Checkpoints{ - FileFormat: "json", + FileFormat: "wal", RootDir: "./var/checkpoints", }, Cleanup: &Cleanup{ diff --git a/pkg/metricstore/configSchema.go b/pkg/metricstore/configSchema.go index 67f30976..ed9bccaa 100644 --- a/pkg/metricstore/configSchema.go +++ b/pkg/metricstore/configSchema.go @@ -18,35 +18,26 @@ const configSchema = `{ "type": "object", "properties": { "file-format": { - "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'json'.", - "type": "string" - }, - "interval": { - "description": "Interval at which the metrics should be checkpointed.", + "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.", "type": "string" }, "directory": { "description": "Path in which the checkpointed files should be placed.", "type": "string" } - }, - "required": ["interval"] + } }, "cleanup": { - "description": "Configuration for the cleanup process.", + "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.", "type": "object", "properties": { "mode": { "description": "The operation mode (e.g., 'archive' or 'delete').", "type": "string", - "enum": ["archive", "delete"] - }, - "interval": { - "description": "Interval at which the cleanup runs.", - "type": "string" + "enum": ["archive", "delete"] }, "directory": { - "description": "Target directory for operations.", + "description": "Target directory for archive operations.", "type": "string" } }, @@ -56,7 +47,7 @@ const configSchema = `{ } }, "then": { - "required": ["interval", "directory"] + "required": ["directory"] } }, "retention-in-memory": { @@ -86,5 +77,5 @@ const configSchema = `{ } } }, - "required": ["checkpoints", "retention-in-memory", "memory-cap"] + "required": ["retention-in-memory", "memory-cap"] }` From 87425c0b09bbbe9321fef0c4050f18624c140946 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 10:41:52 +0100 Subject: [PATCH 323/341] Cleanup and update example config files --- configs/config-demo.json | 4 +--- configs/config.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/config-demo.json b/configs/config-demo.json index 4a69ecd2..8c72e37f 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -20,9 +20,7 @@ } ], "metric-store": { - "checkpoints": { - }, - "retention-in-memory": "48h", + "retention-in-memory": "24h", "memory-cap": 100 } } diff --git a/configs/config.json b/configs/config.json index f880b8d8..6b654e4f 100644 --- a/configs/config.json +++ b/configs/config.json @@ -77,7 +77,7 @@ "directory": "./var/checkpoints" }, "memory-cap": 100, - "retention-in-memory": "48h", + "retention-in-memory": "24h", "cleanup": { "mode": "archive", "directory": "./var/archive" From db625239eac21fb4fc3b979ff7677410aa8631e8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 4 Mar 2026 14:18:30 +0100 Subject: [PATCH 324/341] apply updated rangefilter logic to energy and stats --- internal/routerConfig/routes.go | 58 +++++++++++++---- web/frontend/src/generic/Filters.svelte | 47 +++++++++++--- .../src/generic/filters/Energy.svelte | 62 +++++++++++++++---- .../src/generic/filters/InfoBox.svelte | 2 +- .../src/generic/filters/Resources.svelte | 4 +- web/frontend/src/generic/filters/Stats.svelte | 27 ++++++-- .../generic/select/DoubleRangeSlider.svelte | 6 +- web/frontend/src/generic/utils.js | 20 +++--- 8 files changed, 171 insertions(+), 55 deletions(-) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 218384e0..e24038e2 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -405,10 +405,22 @@ func buildFilterPresets(query url.Values) map[string]any { if query.Get("energy") != "" { parts := strings.Split(query.Get("energy"), "-") if len(parts) == 2 { - a, e1 := strconv.Atoi(parts[0]) - b, e2 := strconv.Atoi(parts[1]) - if e1 == nil && e2 == nil { - filterPresets["energy"] = map[string]int{"from": a, "to": b} + if parts[0] == "lessthan" { + lt, lte := strconv.Atoi(parts[1]) + if lte == nil { + filterPresets["energy"] = map[string]int{"from": 1, "to": lt} + } + } else if parts[0] == "morethan" { + mt, mte := strconv.Atoi(parts[1]) + if mte == nil { + filterPresets["energy"] = map[string]int{"from": mt, "to": 0} + } + } else { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["energy"] = map[string]int{"from": a, "to": b} + } } } } @@ -417,15 +429,37 @@ func buildFilterPresets(query url.Values) map[string]any { for _, statEntry := range query["stat"] { parts := strings.Split(statEntry, "-") if len(parts) == 3 { // Metric Footprint Stat Field, from - to - a, e1 := strconv.ParseInt(parts[1], 10, 64) - b, e2 := strconv.ParseInt(parts[2], 10, 64) - if e1 == nil && e2 == nil { - statEntry := map[string]any{ - "field": parts[0], - "from": a, - "to": b, + if parts[1] == "lessthan" { + lt, lte := strconv.ParseInt(parts[2], 10, 64) + if lte == nil { + statEntry := map[string]any{ + "field": parts[0], + "from": 1, + "to": lt, + } + statList = append(statList, statEntry) + } + } else if parts[1] == "morethan" { + mt, mte := strconv.ParseInt(parts[2], 10, 64) + if mte == nil { + statEntry := map[string]any{ + "field": parts[0], + "from": mt, + "to": 0, + } + statList = append(statList, statEntry) + } + } else { + a, e1 := strconv.ParseInt(parts[1], 10, 64) + b, e2 := strconv.ParseInt(parts[2], 10, 64) + if e1 == nil && e2 == nil { + statEntry := map[string]any{ + "field": parts[0], + "from": a, + "to": b, + } + statList = append(statList, statEntry) } - statList = append(statList, statEntry) } } } diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index 162082c0..02f801a0 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -206,7 +206,7 @@ items.push({ duration: { to: filters.duration.lessThan, from: 0 } }); if (filters.duration.moreThan) items.push({ duration: { to: 0, from: filters.duration.moreThan } }); - if (filters.energy.from || filters.energy.to) + if (filters.energy.from != null || filters.energy.to != null) items.push({ energy: { from: filters.energy.from, to: filters.energy.to }, }); @@ -301,11 +301,20 @@ if (filters.node) opts.push(`node=${filters.node}`); if (filters.node && filters.nodeMatch != "eq") // "eq" is default-case opts.push(`nodeMatch=${filters.nodeMatch}`); - if (filters.energy.from && filters.energy.to) + if (filters.energy.from > 1 && filters.energy.to > 0) opts.push(`energy=${filters.energy.from}-${filters.energy.to}`); - if (filters.stats.length != 0) + else if (filters.energy.from > 1 && filters.energy.to == 0) + opts.push(`energy=morethan-${filters.energy.from}`); + else if (filters.energy.from == 1 && filters.energy.to > 0) + opts.push(`energy=lessthan-${filters.energy.to}`); + if (filters.stats.length > 0) for (let stat of filters.stats) { + if (stat.from > 1 && stat.to > 0) opts.push(`stat=${stat.field}-${stat.from}-${stat.to}`); + else if (stat.from > 1 && stat.to == 0) + opts.push(`stat=${stat.field}-morethan-${stat.from}`); + else if (stat.from == 1 && stat.to > 0) + opts.push(`stat=${stat.field}-lessthan-${stat.to}`); } // Build && Return if (opts.length == 0 && window.location.search.length <= 1) return; @@ -550,18 +559,36 @@ {/if} - {#if filters.energy.from || filters.energy.to} + {#if filters.energy.from > 1 && filters.energy.to > 0} (isEnergyOpen = true)}> - Total Energy: {filters.energy.from} - {filters.energy.to} + Total Energy: {filters.energy.from} - {filters.energy.to} kWh + + {:else if filters.energy.from > 1 && filters.energy.to == 0} + (isEnergyOpen = true)}> + Total Energy ≥ {filters.energy.from} kWh + + {:else if filters.energy.from == 1 && filters.energy.to > 0} + (isEnergyOpen = true)}> + Total Energy ≤ {filters.energy.to} kWh {/if} {#if filters.stats.length > 0} - (isStatsOpen = true)}> - {filters.stats - .map((stat) => `${stat.field}: ${stat.from} - ${stat.to}`) - .join(", ")} - + {#each filters.stats as stat} + {#if stat.from > 1 && stat.to > 0} + (isStatsOpen = true)}> + {stat.field}: {stat.from} - {stat.to} {stat.unit} +   + {:else if stat.from > 1 && stat.to == 0} + (isStatsOpen = true)}> + {stat.field} ≥ {stat.from} {stat.unit} +   + {:else if stat.from == 1 && stat.to > 0} + (isStatsOpen = true)}> + {stat.field} ≤ {stat.to} {stat.unit} +   + {/if} + {/each} {/if} {/if} diff --git a/web/frontend/src/generic/filters/Energy.svelte b/web/frontend/src/generic/filters/Energy.svelte index 4d542add..648fdb4d 100644 --- a/web/frontend/src/generic/filters/Energy.svelte +++ b/web/frontend/src/generic/filters/Energy.svelte @@ -15,54 +15,90 @@ ModalBody, ModalHeader, ModalFooter, + Tooltip, + Icon } from "@sveltestrap/sveltestrap"; import DoubleRangeSlider from "../select/DoubleRangeSlider.svelte"; /* Svelte 5 Props */ let { isOpen = $bindable(false), - presetEnergy = { - from: null, - to: null - }, + presetEnergy = { from: null, to: null }, setFilter, } = $props(); + /* Const */ + const minEnergyPreset = 1; + const maxEnergyPreset = 1000; + /* Derived */ - let energyState = $derived(presetEnergy); + // Pending + let pendingEnergyState = $derived({ + from: presetEnergy?.from ? presetEnergy.from : minEnergyPreset, + to: !(presetEnergy.to == null || presetEnergy.to == 0) ? presetEnergy.to : maxEnergyPreset, + }); + // Changable + let energyState = $derived({ + from: presetEnergy?.from ? presetEnergy.from : minEnergyPreset, + to: !(presetEnergy.to == null || presetEnergy.to == 0) ? presetEnergy.to : maxEnergyPreset, + }); + + const energyActive = $derived(!(JSON.stringify(energyState) === JSON.stringify({ from: minEnergyPreset, to: maxEnergyPreset }))); + // Block Apply if null + const disableApply = $derived(energyState.from === null || energyState.to === null); + + /* Function */ + function setEnergy() { + if (energyActive) { + pendingEnergyState = { + from: energyState.from, + to: (energyState.to == maxEnergyPreset) ? 0 : energyState.to + }; + } else { + pendingEnergyState = { from: null, to: null}; + }; + } (isOpen = !isOpen)}> Filter based on energy
    -
    Total Job Energy (kWh)
    +
    + Total Job Energy (kWh) + +
    + + Generalized Presets. Use input fields to change to higher values. + { energyState.from = detail[0]; energyState.to = detail[1]; }} - sliderMin={0.0} - sliderMax={1000.0} - fromPreset={energyState?.from? energyState.from : 0.0} - toPreset={energyState?.to? energyState.to : 1000.0} + sliderMin={minEnergyPreset} + sliderMax={maxEnergyPreset} + fromPreset={energyState.from} + toPreset={energyState.to} />
    diff --git a/web/frontend/src/generic/filters/InfoBox.svelte b/web/frontend/src/generic/filters/InfoBox.svelte index 0c249980..35af4635 100644 --- a/web/frontend/src/generic/filters/InfoBox.svelte +++ b/web/frontend/src/generic/filters/InfoBox.svelte @@ -20,7 +20,7 @@ } = $props(); -
    - Presets for a single node. Can be changed to higher values. + Presets for a single node. Use input fields to change to higher values. { @@ -282,7 +282,7 @@
    - Presets for a single node. Can be changed to higher values. + Presets for a single node. Use input fields to change to higher values. { diff --git a/web/frontend/src/generic/filters/Stats.svelte b/web/frontend/src/generic/filters/Stats.svelte index fef112c7..96d4ecaf 100644 --- a/web/frontend/src/generic/filters/Stats.svelte +++ b/web/frontend/src/generic/filters/Stats.svelte @@ -15,13 +15,15 @@ ModalBody, ModalHeader, ModalFooter, + Tooltip, + Icon } from "@sveltestrap/sveltestrap"; import DoubleRangeSlider from "../select/DoubleRangeSlider.svelte"; /* Svelte 5 Props */ let { isOpen = $bindable(), - presetStats, + presetStats = [], setFilter } = $props(); @@ -29,10 +31,18 @@ const availableStats = $derived(getStatsItems(presetStats)); /* Functions */ + function setRanges() { + for (let as of availableStats) { + if (as.enabled) { + as.to = (as.to == as.peak) ? 0 : as.to + } + }; + } + function resetRanges() { for (let as of availableStats) { as.enabled = false - as.from = 0 + as.from = 1 as.to = as.peak }; } @@ -45,18 +55,24 @@ {#each availableStats as aStat}
    -
    {aStat.text}
    +
    + {aStat.text} ({aStat.unit}) + +
    + + Peak Threshold Preset. Use input fields to change to higher values. + { aStat.from = detail[0]; aStat.to = detail[1]; - if (aStat.from == 0 && aStat.to == aStat.peak) { + if (aStat.from == 1 && aStat.to == aStat.peak) { aStat.enabled = false; } else { aStat.enabled = true; } }} - sliderMin={0.0} + sliderMin={1} sliderMax={aStat.peak} fromPreset={aStat.from} toPreset={aStat.to} @@ -69,6 +85,7 @@ color="primary" onclick={() => { isOpen = false; + setRanges(); setFilter({ stats: [...availableStats.filter((as) => as.enabled)] }); }}>Close & Apply diff --git a/web/frontend/src/generic/select/DoubleRangeSlider.svelte b/web/frontend/src/generic/select/DoubleRangeSlider.svelte index c655087f..958db598 100644 --- a/web/frontend/src/generic/select/DoubleRangeSlider.svelte +++ b/web/frontend/src/generic/select/DoubleRangeSlider.svelte @@ -165,11 +165,11 @@ }} /> - {#if inputFieldFrom != "1" && inputFieldTo != sliderMax?.toString() } + {#if inputFieldFrom != sliderMin?.toString() && inputFieldTo != sliderMax?.toString() } Selected: Range {inputFieldFrom} - {inputFieldTo} - {:else if inputFieldFrom != "1" && inputFieldTo == sliderMax?.toString() } + {:else if inputFieldFrom != sliderMin?.toString() && inputFieldTo == sliderMax?.toString() } Selected: More than {inputFieldFrom} - {:else if inputFieldFrom == "1" && inputFieldTo != sliderMax?.toString() } + {:else if inputFieldFrom == sliderMin?.toString() && inputFieldTo != sliderMax?.toString() } Selected: Less than {inputFieldTo} {:else} No Selection diff --git a/web/frontend/src/generic/utils.js b/web/frontend/src/generic/utils.js index 09239ec8..82da21b8 100644 --- a/web/frontend/src/generic/utils.js +++ b/web/frontend/src/generic/utils.js @@ -341,26 +341,28 @@ export function getStatsItems(presetStats = []) { if (gm?.footprint) { const mc = getMetricConfigDeep(gm.name, null, null) if (mc) { - const presetEntry = presetStats.find((s) => s?.field === (gm.name + '_' + gm.footprint)) + const presetEntry = presetStats.find((s) => s.field == `${gm.name}_${gm.footprint}`) if (presetEntry) { return { - field: gm.name + '_' + gm.footprint, - text: gm.name + ' (' + gm.footprint + ')', + field: presetEntry.field, + text: `${gm.name} (${gm.footprint})`, metric: gm.name, from: presetEntry.from, - to: presetEntry.to, + to: (presetEntry.to == 0) ? mc.peak : presetEntry.to, peak: mc.peak, - enabled: true + enabled: true, + unit: `${gm?.unit?.prefix ? gm.unit.prefix : ''}${gm.unit.base}` } } else { return { - field: gm.name + '_' + gm.footprint, - text: gm.name + ' (' + gm.footprint + ')', + field: `${gm.name}_${gm.footprint}`, + text: `${gm.name} (${gm.footprint})`, metric: gm.name, - from: 0, + from: 1, to: mc.peak, peak: mc.peak, - enabled: false + enabled: false, + unit: `${gm?.unit?.prefix ? gm.unit.prefix : ''}${gm.unit.base}` } } } From cc0403e2a4f4c7cc0879d4004c7a141c8148dd1b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 14:32:40 +0100 Subject: [PATCH 325/341] Fix goreleaser config Entire-Checkpoint: a204a44fa885 --- .goreleaser.yaml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 3edcb7d6..f861e3c2 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,4 @@ +version: 2 before: hooks: - go mod tidy @@ -34,6 +35,19 @@ builds: main: ./tools/archive-manager tags: - static_build + - env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + goamd64: + - v3 + id: "archive-migration" + binary: archive-migration + main: ./tools/archive-migration + tags: + - static_build - env: - CGO_ENABLED=0 goos: @@ -48,7 +62,7 @@ builds: tags: - static_build archives: - - format: tar.gz + - formats: tar.gz # this name template makes the OS and Arch compatible with the results of uname. name_template: >- {{ .ProjectName }}_ @@ -59,7 +73,7 @@ archives: checksum: name_template: "checksums.txt" snapshot: - name_template: "{{ incpatch .Version }}-next" + version_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: @@ -87,7 +101,7 @@ changelog: release: draft: false footer: | - Supports job archive version 2 and database version 8. + Supports job archive version 3 and database version 10. Please check out the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md) for further details on breaking changes. # vim: set ts=2 sw=2 tw=0 fo=cnqoj From 33ec755422ae2ea72bca982299c108a264958477 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 15:04:53 +0100 Subject: [PATCH 326/341] Fix typo in job high memory tagger --- configs/tagger/jobclasses/highMemoryUsage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/tagger/jobclasses/highMemoryUsage.json b/configs/tagger/jobclasses/highMemoryUsage.json index f241457d..878cd669 100644 --- a/configs/tagger/jobclasses/highMemoryUsage.json +++ b/configs/tagger/jobclasses/highMemoryUsage.json @@ -16,6 +16,6 @@ "expr": "mem_used.max / mem_used.limits.peak * 100.0" } ], - "rule": "mem_used.max > memory_used.limits.alert", + "rule": "mem_used.max > mem_used.limits.alert", "hint": "This job used high memory: peak memory usage {{.mem_used.max}} GB ({{.memory_usage_pct}}% of {{.mem_used.limits.peak}} GB node capacity), exceeding the {{.highmemoryusage_threshold_factor}} utilization threshold. Risk of out-of-memory conditions." } From 67a17b530690239e7d00f8149aa20f78c37a0db5 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 15:14:35 +0100 Subject: [PATCH 327/341] Reduce noise in info log --- internal/tagger/classifyJob.go | 9 +++------ internal/tagger/detectApp.go | 10 +++++----- pkg/metricstore/healthcheck.go | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/internal/tagger/classifyJob.go b/internal/tagger/classifyJob.go index f8751047..6a53fae8 100644 --- a/internal/tagger/classifyJob.go +++ b/internal/tagger/classifyJob.go @@ -309,7 +309,7 @@ func (t *JobClassTagger) Register() error { func (t *JobClassTagger) Match(job *schema.Job) { jobStats, err := t.getStatistics(job) metricsList := t.getMetricConfig(job.Cluster, job.SubCluster) - cclog.Infof("Enter match rule with %d rules for job %d", len(t.rules), job.JobID) + cclog.Debugf("Enter match rule with %d rules for job %d", len(t.rules), job.JobID) if err != nil { cclog.Errorf("job classification failed for job %d: %#v", job.JobID, err) return @@ -321,7 +321,7 @@ func (t *JobClassTagger) Match(job *schema.Job) { for tag, ri := range t.rules { env := make(map[string]any) maps.Copy(env, ri.env) - cclog.Infof("Try to match rule %s for job %d", tag, job.JobID) + cclog.Debugf("Try to match rule %s for job %d", tag, job.JobID) // Initialize environment env["job"] = map[string]any{ @@ -369,7 +369,7 @@ func (t *JobClassTagger) Match(job *schema.Job) { break } if !ok.(bool) { - cclog.Infof("requirement for rule %s not met", tag) + cclog.Debugf("requirement for rule %s not met", tag) requirementsMet = false break } @@ -399,7 +399,6 @@ func (t *JobClassTagger) Match(job *schema.Job) { continue } if match.(bool) { - cclog.Info("Rule matches!") if !t.repo.HasTag(id, t.tagType, tag) { if _, err := t.repo.AddTagOrCreateDirect(id, t.tagType, tag); err != nil { cclog.Errorf("failed to add tag '%s' to job %d: %v", tag, id, err) @@ -414,8 +413,6 @@ func (t *JobClassTagger) Match(job *schema.Job) { continue } messages = append(messages, msg.String()) - } else { - cclog.Info("Rule does not match!") } } diff --git a/internal/tagger/detectApp.go b/internal/tagger/detectApp.go index 54626eff..97b9d6b0 100644 --- a/internal/tagger/detectApp.go +++ b/internal/tagger/detectApp.go @@ -178,24 +178,24 @@ func (t *AppTagger) Match(job *schema.Job) { metadata, err := r.FetchMetadata(job) if err != nil { - cclog.Infof("AppTagger: cannot fetch metadata for job %d on %s: %v", job.JobID, job.Cluster, err) + cclog.Debugf("AppTagger: cannot fetch metadata for job %d on %s: %v", job.JobID, job.Cluster, err) return } if metadata == nil { - cclog.Infof("AppTagger: metadata is nil for job %d on %s", job.JobID, job.Cluster) + cclog.Debugf("AppTagger: metadata is nil for job %d on %s", job.JobID, job.Cluster) return } jobscript, ok := metadata["jobScript"] if !ok { - cclog.Infof("AppTagger: no 'jobScript' key in metadata for job %d on %s (keys: %v)", + cclog.Debugf("AppTagger: no 'jobScript' key in metadata for job %d on %s (keys: %v)", job.JobID, job.Cluster, metadataKeys(metadata)) return } if len(jobscript) == 0 { - cclog.Infof("AppTagger: empty jobScript for job %d on %s", job.JobID, job.Cluster) + cclog.Debugf("AppTagger: empty jobScript for job %d on %s", job.JobID, job.Cluster) return } @@ -210,7 +210,7 @@ func (t *AppTagger) Match(job *schema.Job) { if r.HasTag(id, t.tagType, a.tag) { cclog.Debugf("AppTagger: job %d already has tag %s:%s, skipping", id, t.tagType, a.tag) } else { - cclog.Infof("AppTagger: pattern '%s' matched for app '%s' on job %d", re.String(), a.tag, id) + cclog.Debugf("AppTagger: pattern '%s' matched for app '%s' on job %d", re.String(), a.tag, id) if _, err := r.AddTagOrCreateDirect(id, t.tagType, a.tag); err != nil { cclog.Errorf("AppTagger: failed to add tag '%s' to job %d: %v", a.tag, id, err) } diff --git a/pkg/metricstore/healthcheck.go b/pkg/metricstore/healthcheck.go index 73973ab0..b3470a14 100644 --- a/pkg/metricstore/healthcheck.go +++ b/pkg/metricstore/healthcheck.go @@ -166,10 +166,10 @@ func (m *MemoryStore) HealthCheck(cluster string, healthyCount := len(expectedMetrics) - degradedCount - missingCount if degradedCount > 0 { - cclog.ComponentInfo("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList) + cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList) } if missingCount > 0 { - cclog.ComponentInfo("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList) + cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList) } var state schema.MonitoringState From 9672903d416a65304173757948924b4bb0d2fb69 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 4 Mar 2026 15:54:08 +0100 Subject: [PATCH 328/341] fix panic caused by concurrent map writes --- pkg/archive/clusterConfig.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 64851365..3e27e415 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -126,6 +126,9 @@ func initClusterConfig() error { if newMetric.Energy != "" { sc.EnergyFootprint = append(sc.EnergyFootprint, newMetric.Name) } + + // Init Topology Lookup Maps Once Per Subcluster + sc.Topology.InitTopologyMaps() } item := metricLookup[mc.Name] From 26982088c33ce8efdd8b5acf0b6e99f11cba3656 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 16:43:05 +0100 Subject: [PATCH 329/341] Consolidate code for external and internal ccms buildQueries function Entire-Checkpoint: fc3be444ef4c --- .../cc-metric-store-queries.go | 324 +---------- internal/metricstoreclient/cc-metric-store.go | 26 +- pkg/metricstore/metricstore.go | 2 +- pkg/metricstore/query.go | 543 ++---------------- pkg/metricstore/scopequery.go | 314 ++++++++++ pkg/metricstore/scopequery_test.go | 273 +++++++++ 6 files changed, 676 insertions(+), 806 deletions(-) create mode 100644 pkg/metricstore/scopequery.go create mode 100644 pkg/metricstore/scopequery_test.go diff --git a/internal/metricstoreclient/cc-metric-store-queries.go b/internal/metricstoreclient/cc-metric-store-queries.go index 7a04efc4..b8e3a94a 100644 --- a/internal/metricstoreclient/cc-metric-store-queries.go +++ b/internal/metricstoreclient/cc-metric-store-queries.go @@ -37,23 +37,13 @@ package metricstoreclient import ( "fmt" - "strconv" "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/metricstore" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" ) -// Scope string constants used in API queries. -// Pre-converted to avoid repeated allocations during query building. -var ( - hwthreadString = string(schema.MetricScopeHWThread) - coreString = string(schema.MetricScopeCore) - memoryDomainString = string(schema.MetricScopeMemoryDomain) - socketString = string(schema.MetricScopeSocket) - acceleratorString = string(schema.MetricScopeAccelerator) -) - // buildQueries constructs API queries for job-specific metric data. // It iterates through metrics, scopes, and job resources to build the complete query set. // @@ -126,21 +116,27 @@ func (ccms *CCMetricStore) buildQueries( hwthreads = topology.Node } - // Note: Expected exceptions will return as empty slices -> Continue - hostQueries, hostScopes := buildScopeQueries( + scopeResults, ok := metricstore.BuildScopeQueries( nativeScope, requestedScope, remoteName, host.Hostname, topology, hwthreads, host.Accelerators, - resolution, ) - // Note: Unexpected errors, such as unhandled cases, will return as nils -> Error - if hostQueries == nil && hostScopes == nil { + if !ok { return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } - queries = append(queries, hostQueries...) - assignedScope = append(assignedScope, hostScopes...) + for _, sr := range scopeResults { + queries = append(queries, APIQuery{ + Metric: sr.Metric, + Hostname: sr.Hostname, + Aggregate: sr.Aggregate, + Type: sr.Type, + TypeIds: sr.TypeIds, + Resolution: resolution, + }) + assignedScope = append(assignedScope, sr.Scope) + } } } } @@ -231,19 +227,27 @@ func (ccms *CCMetricStore) buildNodeQueries( continue scopesLoop } - nodeQueries, nodeScopes := buildScopeQueries( + scopeResults, ok := metricstore.BuildScopeQueries( nativeScope, requestedScope, remoteName, hostname, topology, topology.Node, acceleratorIds, - resolution, ) - if len(nodeQueries) == 0 && len(nodeScopes) == 0 { + if !ok { return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } - queries = append(queries, nodeQueries...) - assignedScope = append(assignedScope, nodeScopes...) + for _, sr := range scopeResults { + queries = append(queries, APIQuery{ + Metric: sr.Metric, + Hostname: sr.Hostname, + Aggregate: sr.Aggregate, + Type: sr.Type, + TypeIds: sr.TypeIds, + Resolution: resolution, + }) + assignedScope = append(assignedScope, sr.Scope) + } } } } @@ -251,277 +255,3 @@ func (ccms *CCMetricStore) buildNodeQueries( return queries, assignedScope, nil } -// buildScopeQueries generates API queries for a given scope transformation. -// It returns a slice of queries and corresponding assigned scopes. -// Some transformations (e.g., HWThread -> Core/Socket) may generate multiple queries. -func buildScopeQueries( - nativeScope, requestedScope schema.MetricScope, - metric, hostname string, - topology *schema.Topology, - hwthreads []int, - accelerators []string, - resolution int, -) ([]APIQuery, []schema.MetricScope) { - scope := nativeScope.Max(requestedScope) - queries := []APIQuery{} - scopes := []schema.MetricScope{} - - hwthreadsStr := intToStringSlice(hwthreads) - - // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) - if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { - if scope != schema.MetricScopeAccelerator { - // Expected Exception -> Continue -> Return Empty Slices - return queries, scopes - } - - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &acceleratorString, - TypeIds: accelerators, - Resolution: resolution, - }) - scopes = append(scopes, schema.MetricScopeAccelerator) - return queries, scopes - } - - // Accelerator -> Node - if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { - if len(accelerators) == 0 { - // Expected Exception -> Continue -> Return Empty Slices - return queries, scopes - } - - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &acceleratorString, - TypeIds: accelerators, - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // HWThread -> HWThread - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &hwthreadString, - TypeIds: hwthreadsStr, - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // HWThread -> Core - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - for _, core := range cores { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Core[core]), - Resolution: resolution, - }) - scopes = append(scopes, scope) - } - return queries, scopes - } - - // HWThread -> Socket - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - scopes = append(scopes, scope) - } - return queries, scopes - } - - // HWThread -> Node - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: hwthreadsStr, - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // Core -> Core - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // Core -> Socket - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromCores(hwthreads) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - scopes = append(scopes, scope) - } - return queries, scopes - } - - // Core -> Node - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // MemoryDomain -> MemoryDomain - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { - memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &memoryDomainString, - TypeIds: intToStringSlice(memDomains), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // MemoryDomain -> Node - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { - memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(memDomains), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // MemoryDomain -> Socket - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket { - memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains) - if err != nil { - cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err) - // Rare Error Case -> Still Continue -> Return Empty Slices - return queries, scopes - } - - // Create a query for each socket - for _, domains := range socketToDomains { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(domains), - Resolution: resolution, - }) - // Add scope for each query, not just once - scopes = append(scopes, scope) - } - return queries, scopes - } - - // Socket -> Socket - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // Socket -> Node - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // Node -> Node - if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Resolution: resolution, - }) - scopes = append(scopes, scope) - return queries, scopes - } - - // Unhandled Case -> Error -> Return nils - return nil, nil -} - -// intToStringSlice converts a slice of integers to a slice of strings. -// Used to convert hardware IDs (core IDs, socket IDs, etc.) to the string format required by the API. -func intToStringSlice(is []int) []string { - ss := make([]string, len(is)) - for i, x := range is { - ss[i] = strconv.Itoa(x) - } - return ss -} diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go index e2a84466..2f13ade6 100644 --- a/internal/metricstoreclient/cc-metric-store.go +++ b/internal/metricstoreclient/cc-metric-store.go @@ -63,7 +63,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" - "github.com/ClusterCockpit/cc-backend/pkg/metricstore" + ms "github.com/ClusterCockpit/cc-backend/pkg/metricstore" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" ) @@ -331,7 +331,7 @@ func (ccms *CCMetricStore) LoadData( } } - sanitizeStats(&res.Avg, &res.Min, &res.Max) + ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) jobMetric.Series = append(jobMetric.Series, schema.Series{ Hostname: query.Hostname, @@ -494,7 +494,7 @@ func (ccms *CCMetricStore) LoadScopedStats( } } - sanitizeStats(&res.Avg, &res.Min, &res.Max) + ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ Hostname: query.Hostname, @@ -584,7 +584,7 @@ func (ccms *CCMetricStore) LoadNodeData( errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) } - sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max) + ms.SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max) hostdata, ok := data[query.Hostname] if !ok { @@ -756,7 +756,7 @@ func (ccms *CCMetricStore) LoadNodeListData( } } - sanitizeStats(&res.Avg, &res.Min, &res.Max) + ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) scopeData.Series = append(scopeData.Series, schema.Series{ Hostname: query.Hostname, @@ -784,8 +784,8 @@ func (ccms *CCMetricStore) LoadNodeListData( // returns the per-node health check results. func (ccms *CCMetricStore) HealthCheck(cluster string, nodes []string, metrics []string, -) (map[string]metricstore.HealthCheckResult, error) { - req := metricstore.HealthCheckReq{ +) (map[string]ms.HealthCheckResult, error) { + req := ms.HealthCheckReq{ Cluster: cluster, Nodes: nodes, MetricNames: metrics, @@ -818,7 +818,7 @@ func (ccms *CCMetricStore) HealthCheck(cluster string, return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status) } - var results map[string]metricstore.HealthCheckResult + var results map[string]ms.HealthCheckResult if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil { cclog.Errorf("Error while decoding health check response: %s", err.Error()) return nil, err @@ -827,16 +827,6 @@ func (ccms *CCMetricStore) HealthCheck(cluster string, return results, nil } -// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling. -// Regular float64 values cannot be JSONed when NaN. -func sanitizeStats(avg, min, max *schema.Float) { - if avg.IsNaN() || min.IsNaN() || max.IsNaN() { - *avg = schema.Float(0) - *min = schema.Float(0) - *max = schema.Float(0) - } -} - // hasNaNStats returns true if any of the statistics contain NaN values. func hasNaNStats(avg, min, max schema.Float) bool { return avg.IsNaN() || min.IsNaN() || max.IsNaN() diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go index 6d49624a..b6fbb51a 100644 --- a/pkg/metricstore/metricstore.go +++ b/pkg/metricstore/metricstore.go @@ -235,7 +235,7 @@ func InitMetrics(metrics map[string]MetricConfig) { // This function is safe for concurrent use after initialization. func GetMemoryStore() *MemoryStore { if msInstance == nil { - cclog.Fatalf("[METRICSTORE]> MemoryStore not initialized!") + cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!") } return msInstance diff --git a/pkg/metricstore/query.go b/pkg/metricstore/query.go index 735c45d6..ed55521f 100644 --- a/pkg/metricstore/query.go +++ b/pkg/metricstore/query.go @@ -29,7 +29,6 @@ package metricstore import ( "context" "fmt" - "strconv" "strings" "time" @@ -186,7 +185,7 @@ func (ccms *InternalMetricStore) LoadData( } } - sanitizeStats(&res) + SanitizeStats(&res.Avg, &res.Min, &res.Max) jobMetric.Series = append(jobMetric.Series, schema.Series{ Hostname: query.Hostname, @@ -216,18 +215,6 @@ func (ccms *InternalMetricStore) LoadData( return jobData, nil } -// Pre-converted scope strings avoid repeated string(MetricScope) allocations during -// query construction. These are used in APIQuery.Type field throughout buildQueries -// and buildNodeQueries functions. Converting once at package initialization improves -// performance for high-volume query building. -var ( - hwthreadString = string(schema.MetricScopeHWThread) - coreString = string(schema.MetricScopeCore) - memoryDomainString = string(schema.MetricScopeMemoryDomain) - socketString = string(schema.MetricScopeSocket) - acceleratorString = string(schema.MetricScopeAccelerator) -) - // buildQueries constructs APIQuery structures with automatic scope transformation for a job. // // This function implements the core scope transformation logic, handling all combinations of @@ -293,9 +280,10 @@ func buildQueries( } } - // Avoid duplicates using map for O(1) lookup - handledScopes := make(map[schema.MetricScope]bool, 3) + // Avoid duplicates... + handledScopes := make([]schema.MetricScope, 0, 3) + scopesLoop: for _, requestedScope := range scopes { nativeScope := mc.Scope if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 { @@ -303,10 +291,12 @@ func buildQueries( } scope := nativeScope.Max(requestedScope) - if handledScopes[scope] { - continue + for _, s := range handledScopes { + if scope == s { + continue scopesLoop + } } - handledScopes[scope] = true + handledScopes = append(handledScopes, scope) for _, host := range job.Resources { hwthreads := host.HWThreads @@ -314,224 +304,27 @@ func buildQueries( hwthreads = topology.Node } - // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) - if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { - if scope != schema.MetricScopeAccelerator { - // Skip all other catched cases - continue - } + scopeResults, ok := BuildScopeQueries( + nativeScope, requestedScope, + metric, host.Hostname, + &topology, hwthreads, host.Accelerators, + ) + if !ok { + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + } + + for _, sr := range scopeResults { queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: false, - Type: &acceleratorString, - TypeIds: host.Accelerators, + Metric: sr.Metric, + Hostname: sr.Hostname, + Aggregate: sr.Aggregate, + Type: sr.Type, + TypeIds: sr.TypeIds, Resolution: resolution, }) - assignedScope = append(assignedScope, schema.MetricScopeAccelerator) - continue + assignedScope = append(assignedScope, sr.Scope) } - - // Accelerator -> Node - if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { - if len(host.Accelerators) == 0 { - continue - } - - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &acceleratorString, - TypeIds: host.Accelerators, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> HWThread - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: false, - Type: &hwthreadString, - TypeIds: intToStringSlice(hwthreads), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> Core - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - for _, core := range cores { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Core[core]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Socket - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Node - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(hwthreads), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Core - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: false, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Socket - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromCores(hwthreads) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // Core -> Node - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { - cores, _ := topology.GetCoresFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> MemoryDomain - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: false, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> Node - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Socket - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: false, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Node - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { - sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Aggregate: true, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Node -> Node - if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: host.Hostname, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } } } @@ -695,7 +488,7 @@ func (ccms *InternalMetricStore) LoadScopedStats( } } - sanitizeStats(&res) + SanitizeStats(&res.Avg, &res.Min, &res.Max) scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{ Hostname: query.Hostname, @@ -796,7 +589,7 @@ func (ccms *InternalMetricStore) LoadNodeData( errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) } - sanitizeStats(&qdata) + SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max) hostdata, ok := data[query.Hostname] if !ok { @@ -977,7 +770,7 @@ func (ccms *InternalMetricStore) LoadNodeListData( } } - sanitizeStats(&res) + SanitizeStats(&res.Avg, &res.Min, &res.Max) scopeData.Series = append(scopeData.Series, schema.Series{ Hostname: query.Hostname, @@ -1060,17 +853,20 @@ func buildNodeQueries( } } - // Avoid duplicates using map for O(1) lookup - handledScopes := make(map[schema.MetricScope]bool, 3) + // Avoid duplicates... + handledScopes := make([]schema.MetricScope, 0, 3) + nodeScopesLoop: for _, requestedScope := range scopes { nativeScope := mc.Scope scope := nativeScope.Max(requestedScope) - if handledScopes[scope] { - continue + for _, s := range handledScopes { + if scope == s { + continue nodeScopesLoop + } } - handledScopes[scope] = true + handledScopes = append(handledScopes, scope) for _, hostname := range nodes { @@ -1086,8 +882,7 @@ func buildNodeQueries( } } - // Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable - // Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable + // Always full node hwthread id list, no partial queries expected topology := subClusterTopol.Topology acceleratorIds := topology.GetAcceleratorIDs() @@ -1096,262 +891,30 @@ func buildNodeQueries( continue } - // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) - if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { - if scope != schema.MetricScopeAccelerator { - // Skip all other catched cases - continue - } + scopeResults, ok := BuildScopeQueries( + nativeScope, requestedScope, + metric, hostname, + &topology, topology.Node, acceleratorIds, + ) + if !ok { + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + } + + for _, sr := range scopeResults { queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &acceleratorString, - TypeIds: acceleratorIds, + Metric: sr.Metric, + Hostname: sr.Hostname, + Aggregate: sr.Aggregate, + Type: sr.Type, + TypeIds: sr.TypeIds, Resolution: resolution, }) - assignedScope = append(assignedScope, schema.MetricScopeAccelerator) - continue + assignedScope = append(assignedScope, sr.Scope) } - - // Accelerator -> Node - if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { - if len(acceleratorIds) == 0 { - continue - } - - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &acceleratorString, - TypeIds: acceleratorIds, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> HWThread - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Node), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // HWThread -> Core - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - for _, core := range cores { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Core[core]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Socket - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // HWThread -> Node - if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &hwthreadString, - TypeIds: intToStringSlice(topology.Node), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Core - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Core -> Socket - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromCores(topology.Node) - for _, socket := range sockets { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(topology.Socket[socket]), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - } - continue - } - - // Core -> Node - if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { - cores, _ := topology.GetCoresFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &coreString, - TypeIds: intToStringSlice(cores), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> MemoryDomain - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // MemoryDomain -> Node - if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { - sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &memoryDomainString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Socket - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: false, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Socket -> Node - if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { - sockets, _ := topology.GetSocketsFromHWThreads(topology.Node) - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Aggregate: true, - Type: &socketString, - TypeIds: intToStringSlice(sockets), - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - // Node -> Node - if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { - queries = append(queries, APIQuery{ - Metric: metric, - Hostname: hostname, - Resolution: resolution, - }) - assignedScope = append(assignedScope, scope) - continue - } - - return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } } } return queries, assignedScope, nil } - -// sanitizeStats converts NaN statistics to zero for JSON compatibility. -// -// schema.Float with NaN values cannot be properly JSON-encoded, so we convert -// NaN to 0. This loses the distinction between "no data" and "zero value", -// but maintains API compatibility. -func sanitizeStats(data *APIMetricData) { - if data.Avg.IsNaN() { - data.Avg = schema.Float(0) - } - if data.Min.IsNaN() { - data.Min = schema.Float(0) - } - if data.Max.IsNaN() { - data.Max = schema.Float(0) - } -} - -// intToStringSlice converts a slice of integers to a slice of strings. -// Used to convert hardware thread/core/socket IDs from topology (int) to APIQuery TypeIds (string). -// -// Optimized to reuse a byte buffer for string conversion, reducing allocations. -func intToStringSlice(is []int) []string { - if len(is) == 0 { - return nil - } - - ss := make([]string, len(is)) - buf := make([]byte, 0, 16) // Reusable buffer for integer conversion - for i, x := range is { - buf = strconv.AppendInt(buf[:0], int64(x), 10) - ss[i] = string(buf) - } - return ss -} diff --git a/pkg/metricstore/scopequery.go b/pkg/metricstore/scopequery.go new file mode 100644 index 00000000..a414b794 --- /dev/null +++ b/pkg/metricstore/scopequery.go @@ -0,0 +1,314 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// This file contains shared scope transformation logic used by both the internal +// metric store (pkg/metricstore) and the external cc-metric-store client +// (internal/metricstoreclient). It extracts the common algorithm for mapping +// between native metric scopes and requested scopes based on cluster topology. +package metricstore + +import ( + "strconv" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +// Pre-converted scope strings avoid repeated string(MetricScope) allocations +// during query construction. Used in ScopeQueryResult.Type field. +var ( + HWThreadString = string(schema.MetricScopeHWThread) + CoreString = string(schema.MetricScopeCore) + MemoryDomainString = string(schema.MetricScopeMemoryDomain) + SocketString = string(schema.MetricScopeSocket) + AcceleratorString = string(schema.MetricScopeAccelerator) +) + +// ScopeQueryResult is a package-independent intermediate type returned by +// BuildScopeQueries. Each consumer converts it to their own APIQuery type +// (adding Resolution and any other package-specific fields). +type ScopeQueryResult struct { + Type *string + Metric string + Hostname string + TypeIds []string + Scope schema.MetricScope + Aggregate bool +} + +// BuildScopeQueries generates scope query results for a given scope transformation. +// It returns a slice of results and a boolean indicating success. +// An empty slice means an expected exception (skip this combination). +// ok=false means an unhandled case (caller should return an error). +func BuildScopeQueries( + nativeScope, requestedScope schema.MetricScope, + metric, hostname string, + topology *schema.Topology, + hwthreads []int, + accelerators []string, +) ([]ScopeQueryResult, bool) { + scope := nativeScope.Max(requestedScope) + results := []ScopeQueryResult{} + + hwthreadsStr := IntToStringSlice(hwthreads) + + // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) + if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { + if scope != schema.MetricScopeAccelerator { + // Expected Exception -> Return Empty Slice + return results, true + } + + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: false, + Type: &AcceleratorString, + TypeIds: accelerators, + Scope: schema.MetricScopeAccelerator, + }) + return results, true + } + + // Accelerator -> Node + if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode { + if len(accelerators) == 0 { + // Expected Exception -> Return Empty Slice + return results, true + } + + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &AcceleratorString, + TypeIds: accelerators, + Scope: scope, + }) + return results, true + } + + // HWThread -> HWThread + if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: false, + Type: &HWThreadString, + TypeIds: hwthreadsStr, + Scope: scope, + }) + return results, true + } + + // HWThread -> Core + if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore { + cores, _ := topology.GetCoresFromHWThreads(hwthreads) + for _, core := range cores { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &HWThreadString, + TypeIds: IntToStringSlice(topology.Core[core]), + Scope: scope, + }) + } + return results, true + } + + // HWThread -> Socket + if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket { + sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) + for _, socket := range sockets { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &HWThreadString, + TypeIds: IntToStringSlice(topology.Socket[socket]), + Scope: scope, + }) + } + return results, true + } + + // HWThread -> Node + if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &HWThreadString, + TypeIds: hwthreadsStr, + Scope: scope, + }) + return results, true + } + + // Core -> Core + if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore { + cores, _ := topology.GetCoresFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: false, + Type: &CoreString, + TypeIds: IntToStringSlice(cores), + Scope: scope, + }) + return results, true + } + + // Core -> Socket + if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket { + sockets, _ := topology.GetSocketsFromCores(hwthreads) + for _, socket := range sockets { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &CoreString, + TypeIds: IntToStringSlice(topology.Socket[socket]), + Scope: scope, + }) + } + return results, true + } + + // Core -> Node + if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode { + cores, _ := topology.GetCoresFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &CoreString, + TypeIds: IntToStringSlice(cores), + Scope: scope, + }) + return results, true + } + + // MemoryDomain -> MemoryDomain + if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain { + memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: false, + Type: &MemoryDomainString, + TypeIds: IntToStringSlice(memDomains), + Scope: scope, + }) + return results, true + } + + // MemoryDomain -> Socket + if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket { + memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) + socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains) + if err != nil { + cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err) + // Rare Error Case -> Still Continue -> Return Empty Slice + return results, true + } + + // Create a query for each socket + for _, domains := range socketToDomains { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &MemoryDomainString, + TypeIds: IntToStringSlice(domains), + Scope: scope, + }) + } + return results, true + } + + // MemoryDomain -> Node + if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode { + memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &MemoryDomainString, + TypeIds: IntToStringSlice(memDomains), + Scope: scope, + }) + return results, true + } + + // Socket -> Socket + if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { + sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: false, + Type: &SocketString, + TypeIds: IntToStringSlice(sockets), + Scope: scope, + }) + return results, true + } + + // Socket -> Node + if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { + sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Aggregate: true, + Type: &SocketString, + TypeIds: IntToStringSlice(sockets), + Scope: scope, + }) + return results, true + } + + // Node -> Node + if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode { + results = append(results, ScopeQueryResult{ + Metric: metric, + Hostname: hostname, + Scope: scope, + }) + return results, true + } + + // Unhandled Case + return nil, false +} + +// IntToStringSlice converts a slice of integers to a slice of strings. +// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string). +// Optimized to reuse a byte buffer for string conversion, reducing allocations. +func IntToStringSlice(is []int) []string { + if len(is) == 0 { + return nil + } + + ss := make([]string, len(is)) + buf := make([]byte, 0, 16) // Reusable buffer for integer conversion + for i, x := range is { + buf = strconv.AppendInt(buf[:0], int64(x), 10) + ss[i] = string(buf) + } + return ss +} + +// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling. +// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency. +func SanitizeStats(avg, min, max *schema.Float) { + if avg.IsNaN() || min.IsNaN() || max.IsNaN() { + *avg = schema.Float(0) + *min = schema.Float(0) + *max = schema.Float(0) + } +} diff --git a/pkg/metricstore/scopequery_test.go b/pkg/metricstore/scopequery_test.go new file mode 100644 index 00000000..4cdfca78 --- /dev/null +++ b/pkg/metricstore/scopequery_test.go @@ -0,0 +1,273 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package metricstore + +import ( + "testing" + + "github.com/ClusterCockpit/cc-lib/v2/schema" +) + +// makeTopology creates a simple 2-socket, 4-core, 8-hwthread topology for testing. +// Socket 0: cores 0,1 with hwthreads 0,1,2,3 +// Socket 1: cores 2,3 with hwthreads 4,5,6,7 +// MemoryDomain 0: hwthreads 0,1,2,3 (socket 0) +// MemoryDomain 1: hwthreads 4,5,6,7 (socket 1) +func makeTopology() schema.Topology { + topo := schema.Topology{ + Node: []int{0, 1, 2, 3, 4, 5, 6, 7}, + Socket: [][]int{{0, 1, 2, 3}, {4, 5, 6, 7}}, + MemoryDomain: [][]int{{0, 1, 2, 3}, {4, 5, 6, 7}}, + Core: [][]int{{0, 1}, {2, 3}, {4, 5}, {6, 7}}, + Accelerators: []*schema.Accelerator{ + {ID: "gpu0"}, + {ID: "gpu1"}, + }, + } + return topo +} + +func TestBuildScopeQueries(t *testing.T) { + topo := makeTopology() + topo.InitTopologyMaps() + accIds := topo.GetAcceleratorIDs() + + tests := []struct { + name string + nativeScope schema.MetricScope + requestedScope schema.MetricScope + expectOk bool + expectLen int // expected number of results + expectAgg bool + expectScope schema.MetricScope + }{ + // Same-scope cases + { + name: "HWThread->HWThread", nativeScope: schema.MetricScopeHWThread, + requestedScope: schema.MetricScopeHWThread, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeHWThread, + }, + { + name: "Core->Core", nativeScope: schema.MetricScopeCore, + requestedScope: schema.MetricScopeCore, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeCore, + }, + { + name: "Socket->Socket", nativeScope: schema.MetricScopeSocket, + requestedScope: schema.MetricScopeSocket, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeSocket, + }, + { + name: "MemoryDomain->MemoryDomain", nativeScope: schema.MetricScopeMemoryDomain, + requestedScope: schema.MetricScopeMemoryDomain, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeMemoryDomain, + }, + { + name: "Node->Node", nativeScope: schema.MetricScopeNode, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeNode, + }, + { + name: "Accelerator->Accelerator", nativeScope: schema.MetricScopeAccelerator, + requestedScope: schema.MetricScopeAccelerator, expectOk: true, expectLen: 1, + expectAgg: false, expectScope: schema.MetricScopeAccelerator, + }, + // Aggregation cases + { + name: "HWThread->Core", nativeScope: schema.MetricScopeHWThread, + requestedScope: schema.MetricScopeCore, expectOk: true, expectLen: 4, // 4 cores + expectAgg: true, expectScope: schema.MetricScopeCore, + }, + { + name: "HWThread->Socket", nativeScope: schema.MetricScopeHWThread, + requestedScope: schema.MetricScopeSocket, expectOk: true, expectLen: 2, // 2 sockets + expectAgg: true, expectScope: schema.MetricScopeSocket, + }, + { + name: "HWThread->Node", nativeScope: schema.MetricScopeHWThread, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: true, expectScope: schema.MetricScopeNode, + }, + { + name: "Core->Socket", nativeScope: schema.MetricScopeCore, + requestedScope: schema.MetricScopeSocket, expectOk: true, expectLen: 2, // 2 sockets + expectAgg: true, expectScope: schema.MetricScopeSocket, + }, + { + name: "Core->Node", nativeScope: schema.MetricScopeCore, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: true, expectScope: schema.MetricScopeNode, + }, + { + name: "Socket->Node", nativeScope: schema.MetricScopeSocket, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: true, expectScope: schema.MetricScopeNode, + }, + { + name: "MemoryDomain->Node", nativeScope: schema.MetricScopeMemoryDomain, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: true, expectScope: schema.MetricScopeNode, + }, + { + name: "MemoryDomain->Socket", nativeScope: schema.MetricScopeMemoryDomain, + requestedScope: schema.MetricScopeSocket, expectOk: true, expectLen: 2, // 2 sockets + expectAgg: true, expectScope: schema.MetricScopeSocket, + }, + { + name: "Accelerator->Node", nativeScope: schema.MetricScopeAccelerator, + requestedScope: schema.MetricScopeNode, expectOk: true, expectLen: 1, + expectAgg: true, expectScope: schema.MetricScopeNode, + }, + // Expected exception: Accelerator scope requested but non-accelerator scope in between + { + name: "Accelerator->Core (exception)", nativeScope: schema.MetricScopeAccelerator, + requestedScope: schema.MetricScopeCore, expectOk: true, expectLen: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results, ok := BuildScopeQueries( + tt.nativeScope, tt.requestedScope, + "test_metric", "node001", + &topo, topo.Node, accIds, + ) + + if ok != tt.expectOk { + t.Fatalf("expected ok=%v, got ok=%v", tt.expectOk, ok) + } + + if len(results) != tt.expectLen { + t.Fatalf("expected %d results, got %d", tt.expectLen, len(results)) + } + + if tt.expectLen > 0 { + for _, r := range results { + if r.Scope != tt.expectScope { + t.Errorf("expected scope %s, got %s", tt.expectScope, r.Scope) + } + if r.Aggregate != tt.expectAgg { + t.Errorf("expected aggregate=%v, got %v", tt.expectAgg, r.Aggregate) + } + if r.Metric != "test_metric" { + t.Errorf("expected metric 'test_metric', got '%s'", r.Metric) + } + if r.Hostname != "node001" { + t.Errorf("expected hostname 'node001', got '%s'", r.Hostname) + } + } + } + }) + } +} + +func TestBuildScopeQueries_UnhandledCase(t *testing.T) { + topo := makeTopology() + topo.InitTopologyMaps() + + // Node native with HWThread requested => scope.Max = Node, but let's try an invalid combination + // Actually all valid combinations are handled. An unhandled case would be something like + // a scope that doesn't exist in the if-chain. Since all real scopes are covered, + // we test with a synthetic unhandled combination by checking the bool return. + // The function should return ok=false for truly unhandled cases. + + // For now, verify all known combinations return ok=true + scopes := []schema.MetricScope{ + schema.MetricScopeHWThread, schema.MetricScopeCore, + schema.MetricScopeSocket, schema.MetricScopeNode, + } + + for _, native := range scopes { + for _, requested := range scopes { + results, ok := BuildScopeQueries( + native, requested, + "m", "h", &topo, topo.Node, nil, + ) + if !ok { + t.Errorf("unexpected unhandled case: native=%s, requested=%s", native, requested) + } + if results == nil { + t.Errorf("results should not be nil for native=%s, requested=%s", native, requested) + } + } + } +} + +func TestIntToStringSlice(t *testing.T) { + tests := []struct { + input []int + expected []string + }{ + {nil, nil}, + {[]int{}, nil}, + {[]int{0}, []string{"0"}}, + {[]int{1, 2, 3}, []string{"1", "2", "3"}}, + {[]int{10, 100, 1000}, []string{"10", "100", "1000"}}, + } + + for _, tt := range tests { + result := IntToStringSlice(tt.input) + if len(result) != len(tt.expected) { + t.Errorf("IntToStringSlice(%v): expected len %d, got %d", tt.input, len(tt.expected), len(result)) + continue + } + for i := range result { + if result[i] != tt.expected[i] { + t.Errorf("IntToStringSlice(%v)[%d]: expected %s, got %s", tt.input, i, tt.expected[i], result[i]) + } + } + } +} + +func TestSanitizeStats(t *testing.T) { + // Test: all valid - should remain unchanged + avg, min, max := schema.Float(1.0), schema.Float(0.5), schema.Float(2.0) + SanitizeStats(&avg, &min, &max) + if avg != 1.0 || min != 0.5 || max != 2.0 { + t.Errorf("SanitizeStats should not change valid values") + } + + // Test: one NaN - all should be zeroed + avg, min, max = schema.Float(1.0), schema.Float(0.5), schema.NaN + SanitizeStats(&avg, &min, &max) + if avg != 0 || min != 0 || max != 0 { + t.Errorf("SanitizeStats should zero all when any is NaN, got avg=%v min=%v max=%v", avg, min, max) + } + + // Test: all NaN + avg, min, max = schema.NaN, schema.NaN, schema.NaN + SanitizeStats(&avg, &min, &max) + if avg != 0 || min != 0 || max != 0 { + t.Errorf("SanitizeStats should zero all NaN values") + } +} + +func TestNodeToNodeQuery(t *testing.T) { + topo := makeTopology() + topo.InitTopologyMaps() + + results, ok := BuildScopeQueries( + schema.MetricScopeNode, schema.MetricScopeNode, + "cpu_load", "node001", + &topo, topo.Node, nil, + ) + + if !ok { + t.Fatal("expected ok=true for Node->Node") + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + if r.Type != nil { + t.Error("Node->Node should have nil Type") + } + if r.TypeIds != nil { + t.Error("Node->Node should have nil TypeIds") + } + if r.Aggregate { + t.Error("Node->Node should not aggregate") + } +} From 845d0111af29376086aeb93fee082c4fbb6e9efc Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 17:31:36 +0100 Subject: [PATCH 330/341] Further consolidate and improve ccms query builder Entire-Checkpoint: d10e6221ee4f --- .../cc-metric-store-queries.go | 30 ++---- internal/metricstoreclient/cc-metric-store.go | 65 +++--------- pkg/metricstore/query.go | 99 +++++-------------- pkg/metricstore/scopequery.go | 27 +++++ 4 files changed, 69 insertions(+), 152 deletions(-) diff --git a/internal/metricstoreclient/cc-metric-store-queries.go b/internal/metricstoreclient/cc-metric-store-queries.go index b8e3a94a..1119d70c 100644 --- a/internal/metricstoreclient/cc-metric-store-queries.go +++ b/internal/metricstoreclient/cc-metric-store-queries.go @@ -79,17 +79,8 @@ func (ccms *CCMetricStore) buildQueries( } // Skip if metric is removed for subcluster - if len(mc.SubClusters) != 0 { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == job.SubCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } + if len(mc.SubClusters) != 0 && metricstore.IsMetricRemovedForSubCluster(mc, job.SubCluster) { + continue } // Avoid duplicates... @@ -123,7 +114,7 @@ func (ccms *CCMetricStore) buildQueries( ) if !ok { - return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } for _, sr := range scopeResults { @@ -175,17 +166,8 @@ func (ccms *CCMetricStore) buildNodeQueries( } // Skip if metric is removed for subcluster - if mc.SubClusters != nil { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == subCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } + if mc.SubClusters != nil && metricstore.IsMetricRemovedForSubCluster(mc, subCluster) { + continue } // Avoid duplicates... @@ -234,7 +216,7 @@ func (ccms *CCMetricStore) buildNodeQueries( ) if !ok { - return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } for _, sr := range scopeResults { diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go index 2f13ade6..55dc7fb5 100644 --- a/internal/metricstoreclient/cc-metric-store.go +++ b/internal/metricstoreclient/cc-metric-store.go @@ -275,13 +275,6 @@ func (ccms *CCMetricStore) LoadData( } for i, row := range resBody.Results { - // Safety check to prevent index out of range errors - if i >= len(req.Queries) || i >= len(assignedScope) { - cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d", - i, len(req.Queries), len(assignedScope)) - continue - } - query := req.Queries[i] metric := query.Metric scope := assignedScope[i] @@ -318,18 +311,7 @@ func (ccms *CCMetricStore) LoadData( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ms.ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) @@ -393,6 +375,10 @@ func (ccms *CCMetricStore) LoadStats( stats := make(map[string]map[string]schema.MetricStatistics, len(metrics)) for i, res := range resBody.Results { + if i >= len(req.Queries) { + cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries)) + break + } if len(res) == 0 { // No Data Found For Metric, Logged in FetchData to Warn continue @@ -481,18 +467,7 @@ func (ccms *CCMetricStore) LoadScopedStats( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ms.ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) @@ -582,6 +557,13 @@ func (ccms *CCMetricStore) LoadNodeData( qdata := res[0] if qdata.Error != nil { errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) + continue + } + + mc := archive.GetMetricConfig(cluster, metric) + if mc == nil { + cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster) + continue } ms.SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max) @@ -592,7 +574,6 @@ func (ccms *CCMetricStore) LoadNodeData( data[query.Hostname] = hostdata } - mc := archive.GetMetricConfig(cluster, metric) hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ Unit: mc.Unit, Timestep: mc.Timestep, @@ -680,13 +661,6 @@ func (ccms *CCMetricStore) LoadNodeListData( } for i, row := range resBody.Results { - // Safety check to prevent index out of range errors - if i >= len(req.Queries) || i >= len(assignedScope) { - cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d", - i, len(req.Queries), len(assignedScope)) - continue - } - var query APIQuery if resBody.Queries != nil { if i < len(resBody.Queries) { @@ -743,18 +717,7 @@ func (ccms *CCMetricStore) LoadNodeListData( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ms.ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) ms.SanitizeStats(&res.Avg, &res.Min, &res.Max) diff --git a/pkg/metricstore/query.go b/pkg/metricstore/query.go index ed55521f..8a349b5a 100644 --- a/pkg/metricstore/query.go +++ b/pkg/metricstore/query.go @@ -129,13 +129,6 @@ func (ccms *InternalMetricStore) LoadData( } for i, row := range resBody.Results { - // Safety check to prevent index out of range errors - if i >= len(req.Queries) || i >= len(assignedScope) { - cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d", - i, len(req.Queries), len(assignedScope)) - continue - } - query := req.Queries[i] metric := query.Metric scope := assignedScope[i] @@ -172,18 +165,7 @@ func (ccms *InternalMetricStore) LoadData( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) SanitizeStats(&res.Avg, &res.Min, &res.Max) @@ -251,7 +233,7 @@ func buildQueries( } queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources)) - assignedScope := []schema.MetricScope{} + assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources)) subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) if scerr != nil { @@ -267,17 +249,8 @@ func buildQueries( } // Skip if metric is removed for subcluster - if len(mc.SubClusters) != 0 { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == job.SubCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } + if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) { + continue } // Avoid duplicates... @@ -311,7 +284,7 @@ func buildQueries( ) if !ok { - return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } for _, sr := range scopeResults { @@ -374,6 +347,10 @@ func (ccms *InternalMetricStore) LoadStats( stats := make(map[string]map[string]schema.MetricStatistics, len(metrics)) for i, res := range resBody.Results { + if i >= len(req.Queries) { + cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries)) + break + } if len(res) == 0 { // No Data Found For Metric, Logged in FetchData to Warn continue @@ -475,18 +452,7 @@ func (ccms *InternalMetricStore) LoadScopedStats( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) SanitizeStats(&res.Avg, &res.Min, &res.Max) @@ -587,6 +553,13 @@ func (ccms *InternalMetricStore) LoadNodeData( qdata := res[0] if qdata.Error != nil { errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) + continue + } + + mc := archive.GetMetricConfig(cluster, metric) + if mc == nil { + cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster) + continue } SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max) @@ -597,7 +570,6 @@ func (ccms *InternalMetricStore) LoadNodeData( data[query.Hostname] = hostdata } - mc := archive.GetMetricConfig(cluster, metric) hostdata[metric] = append(hostdata[metric], &schema.JobMetric{ Unit: mc.Unit, Timestep: mc.Timestep, @@ -694,13 +666,6 @@ func (ccms *InternalMetricStore) LoadNodeListData( } for i, row := range resBody.Results { - // Safety check to prevent index out of range errors - if i >= len(req.Queries) || i >= len(assignedScope) { - cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d", - i, len(req.Queries), len(assignedScope)) - continue - } - var query APIQuery if resBody.Queries != nil { if i < len(resBody.Queries) { @@ -757,18 +722,7 @@ func (ccms *InternalMetricStore) LoadNodeListData( continue } - id := (*string)(nil) - if query.Type != nil { - // Check if ndx is within the bounds of TypeIds slice - if ndx < len(query.TypeIds) { - id = new(string) - *id = query.TypeIds[ndx] - } else { - // Log the error but continue processing - cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", - ndx, len(query.TypeIds), query.Metric, query.Hostname) - } - } + id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname) SanitizeStats(&res.Avg, &res.Min, &res.Max) @@ -819,7 +773,7 @@ func buildNodeQueries( resolution int64, ) ([]APIQuery, []schema.MetricScope, error) { queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes)) - assignedScope := []schema.MetricScope{} + assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes)) // Get Topol before loop if subCluster given var subClusterTopol *schema.SubCluster @@ -840,17 +794,8 @@ func buildNodeQueries( } // Skip if metric is removed for subcluster - if mc.SubClusters != nil { - isRemoved := false - for _, scConfig := range mc.SubClusters { - if scConfig.Name == subCluster && scConfig.Remove { - isRemoved = true - break - } - } - if isRemoved { - continue - } + if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) { + continue } // Avoid duplicates... @@ -898,7 +843,7 @@ func buildNodeQueries( ) if !ok { - return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } for _, sr := range scopeResults { diff --git a/pkg/metricstore/scopequery.go b/pkg/metricstore/scopequery.go index a414b794..a01a9cc6 100644 --- a/pkg/metricstore/scopequery.go +++ b/pkg/metricstore/scopequery.go @@ -303,6 +303,33 @@ func IntToStringSlice(is []int) []string { return ss } +// ExtractTypeID returns the type ID at the given index from a query's TypeIds slice. +// Returns nil if queryType is nil (no type filtering). Logs a warning and returns nil +// if the index is out of range. +func ExtractTypeID(queryType *string, typeIds []string, ndx int, metric, hostname string) *string { + if queryType == nil { + return nil + } + if ndx < len(typeIds) { + id := typeIds[ndx] + return &id + } + cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s", + ndx, len(typeIds), metric, hostname) + return nil +} + +// IsMetricRemovedForSubCluster checks whether a metric is marked as removed +// for the given subcluster in its per-subcluster configuration. +func IsMetricRemovedForSubCluster(mc *schema.MetricConfig, subCluster string) bool { + for _, scConfig := range mc.SubClusters { + if scConfig.Name == subCluster && scConfig.Remove { + return true + } + } + return false +} + // SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling. // If ANY of avg/min/max is NaN, ALL three are zeroed for consistency. func SanitizeStats(avg, min, max *schema.Float) { From 47181330e9017c2a03373093f4664b53e434113f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 17:39:46 +0100 Subject: [PATCH 331/341] Update to latest cc-lib --- go.mod | 3 +-- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2e72e342..b03e7e0b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ tool ( require ( github.com/99designs/gqlgen v0.17.87 - github.com/ClusterCockpit/cc-lib/v2 v2.7.0 + github.com/ClusterCockpit/cc-lib/v2 v2.8.0 github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 github.com/Masterminds/squirrel v1.5.4 github.com/aws/aws-sdk-go-v2 v1.41.2 @@ -111,7 +111,6 @@ require ( github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect diff --git a/go.sum b/go.sum index 70f98fc8..812e4b83 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/99designs/gqlgen v0.17.87 h1:pSnCIMhBQezAE8bc1GNmfdLXFmnWtWl1GRDFEE/n github.com/99designs/gqlgen v0.17.87/go.mod h1:fK05f1RqSNfQpd4CfW5qk/810Tqi4/56Wf6Nem0khAg= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= -github.com/ClusterCockpit/cc-lib/v2 v2.7.0 h1:EMTShk6rMTR1wlfmQ8SVCawH1OdltUbD3kVQmaW+5pE= -github.com/ClusterCockpit/cc-lib/v2 v2.7.0/go.mod h1:0Etx8WMs0lYZ4tiOQizY18CQop+2i3WROvU9rMUxHA4= +github.com/ClusterCockpit/cc-lib/v2 v2.8.0 h1:ROduRzRuusi+6kLB991AAu3Pp2AHOasQJFJc7JU/n/E= +github.com/ClusterCockpit/cc-lib/v2 v2.8.0/go.mod h1:FwD8vnTIbBM3ngeLNKmCvp9FoSjQZm7xnuaVxEKR23o= github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q= github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= @@ -307,8 +307,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= From ddda341e10b906347ac7be124b787e316eccc8ec Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 5 Mar 2026 10:37:33 +0100 Subject: [PATCH 332/341] Safeguard metricstore shutdown if internal metricstore is not initialized --- cmd/cc-backend/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 91f8360f..8687db63 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -424,7 +424,11 @@ func (s *Server) Shutdown(ctx context.Context) { } // Archive all the metric store data - metricstore.Shutdown() + ms := metricstore.GetMemoryStore() + + if ms != nil { + metricstore.Shutdown() + } // Shutdown archiver with 10 second timeout for fast shutdown if err := archiver.Shutdown(10 * time.Second); err != nil { From 2c519ab2dc97e7f52583cd0c271a697f3c9fb75f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 5 Mar 2026 12:23:00 +0100 Subject: [PATCH 333/341] bump frontend dependencies - fixes CVE-2020-7660 in @rollup/plugin-terser --- web/frontend/package-lock.json | 69 ++++++++++------------------------ web/frontend/package.json | 6 +-- 2 files changed, 22 insertions(+), 53 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 6062ee54..08b87ea3 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -14,15 +14,15 @@ "@urql/svelte": "^4.2.3", "chart.js": "^4.5.1", "date-fns": "^4.1.0", - "graphql": "^16.13.0", + "graphql": "^16.13.1", "mathjs": "^15.1.1", "uplot": "^1.6.32", "wonka": "^6.3.5" }, "devDependencies": { - "@rollup/plugin-commonjs": "^29.0.0", + "@rollup/plugin-commonjs": "^29.0.1", "@rollup/plugin-node-resolve": "^16.0.3", - "@rollup/plugin-terser": "^0.4.4", + "@rollup/plugin-terser": "^1.0.0", "@timohausmann/quadtree-js": "^1.2.6", "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", @@ -126,9 +126,9 @@ } }, "node_modules/@rollup/plugin-commonjs": { - "version": "29.0.0", - "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.0.tgz", - "integrity": "sha512-U2YHaxR2cU/yAiwKJtJRhnyLk7cifnQw0zUpISsocBDoHDJn+HTV74ABqnwr5bEgWUwFZC9oFL6wLe21lHu5eQ==", + "version": "29.0.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.1.tgz", + "integrity": "sha512-VUEHINN2rQEWPfNUR3mzidRObM1XZKXMQsaG6qBlDqd6M1qyw91nDZvcSozgyjt3x/QKrgKBc5MdxfdxAy6tdg==", "dev": true, "license": "MIT", "dependencies": { @@ -199,18 +199,18 @@ } }, "node_modules/@rollup/plugin-terser": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/@rollup/plugin-terser/-/plugin-terser-0.4.4.tgz", - "integrity": "sha512-XHeJC5Bgvs8LfukDwWZp7yeqin6ns8RTl2B9avbejt6tZqsqvVoWI7ZTQrcNsfKEDWBTnTxM8nMDkO2IFFbd0A==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-terser/-/plugin-terser-1.0.0.tgz", + "integrity": "sha512-FnCxhTBx6bMOYQrar6C8h3scPt8/JwIzw3+AJ2K++6guogH5fYaIFia+zZuhqv0eo1RN7W1Pz630SyvLbDjhtQ==", "dev": true, "license": "MIT", "dependencies": { - "serialize-javascript": "^6.0.1", + "serialize-javascript": "^7.0.3", "smob": "^1.0.0", "terser": "^5.17.4" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.0.0" }, "peerDependencies": { "rollup": "^2.0.0||^3.0.0||^4.0.0" @@ -878,9 +878,9 @@ } }, "node_modules/graphql": { - "version": "16.13.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.13.0.tgz", - "integrity": "sha512-uSisMYERbaB9bkA9M4/4dnqyktaEkf1kMHNKq/7DHyxVeWqHQ2mBmVqm5u6/FVHwF3iCNalKcg82Zfl+tffWoA==", + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.13.1.tgz", + "integrity": "sha512-gGgrVCoDKlIZ8fIqXBBb0pPKqDgki0Z/FSKNiQzSGj2uEYHr1tq5wmBegGwJx6QB5S5cM0khSBpi/JFHMCvsmQ==", "license": "MIT", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" @@ -995,16 +995,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/resolve": { "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", @@ -1142,27 +1132,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/seedrandom": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", @@ -1170,13 +1139,13 @@ "license": "MIT" }, "node_modules/serialize-javascript": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", - "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.4.tgz", + "integrity": "sha512-DuGdB+Po43Q5Jxwpzt1lhyFSYKryqoNjQSA9M92tyw0lyHIOur+XCalOUe0KTJpyqzT8+fQ5A0Jf7vCx/NKmIg==", "dev": true, "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" + "engines": { + "node": ">=20.0.0" } }, "node_modules/smob": { diff --git a/web/frontend/package.json b/web/frontend/package.json index 5a81042a..fe535fcb 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -7,9 +7,9 @@ "dev": "rollup -c -w" }, "devDependencies": { - "@rollup/plugin-commonjs": "^29.0.0", + "@rollup/plugin-commonjs": "^29.0.1", "@rollup/plugin-node-resolve": "^16.0.3", - "@rollup/plugin-terser": "^0.4.4", + "@rollup/plugin-terser": "^1.0.0", "@timohausmann/quadtree-js": "^1.2.6", "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", @@ -22,7 +22,7 @@ "@urql/svelte": "^4.2.3", "chart.js": "^4.5.1", "date-fns": "^4.1.0", - "graphql": "^16.13.0", + "graphql": "^16.13.1", "mathjs": "^15.1.1", "uplot": "^1.6.32", "wonka": "^6.3.5" From d74465215d85207e7024381ae8ab01c8fadea6f8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 6 Mar 2026 10:09:44 +0100 Subject: [PATCH 334/341] simplify and fix adaptive threshold logic --- .../src/generic/joblist/JobListRow.svelte | 2 -- .../src/generic/plots/MetricPlot.svelte | 28 +++++-------------- web/frontend/src/job/Metric.svelte | 4 --- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 3963708f..e9382bee 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -234,8 +234,6 @@ cluster={clusterInfos.find((c) => c.name == job.cluster)} subCluster={job.subCluster} isShared={job.shared != "none"} - numhwthreads={job.numHWThreads} - numaccs={job.numAcc} zoomState={zoomStates[metric.data.name] || null} thresholdState={thresholdStates[metric.data.name] || null} {plotSync} diff --git a/web/frontend/src/generic/plots/MetricPlot.svelte b/web/frontend/src/generic/plots/MetricPlot.svelte index 063b43fb..aa38d858 100644 --- a/web/frontend/src/generic/plots/MetricPlot.svelte +++ b/web/frontend/src/generic/plots/MetricPlot.svelte @@ -43,8 +43,6 @@ subCluster, isShared = false, forNode = false, - numhwthreads = 0, - numaccs = 0, zoomState = null, thresholdState = null, extendedLegendData = null, @@ -83,9 +81,7 @@ const thresholds = $derived(findJobAggregationThresholds( subClusterTopology, metricConfig, - scope, - numhwthreads, - numaccs + scope )); const longestSeries = $derived.by(() => { if (useStatsSeries) { @@ -276,9 +272,7 @@ function findJobAggregationThresholds( subClusterTopology, metricConfig, - scope, - numhwthreads, - numaccs + scope ) { if (!subClusterTopology || !metricConfig || !scope) { @@ -303,21 +297,13 @@ } if (metricConfig?.aggregation == "sum") { - // Scale Thresholds - let fraction; - if (numaccs > 0) fraction = subClusterTopology.accelerators.length / numaccs; - else if (numhwthreads > 0) fraction = subClusterTopology.core.length / numhwthreads; - else fraction = 1; // Fallback - let divisor; - // Exclusive: Fraction = 1; Shared: Fraction > 1 - if (scope == 'node') divisor = fraction; - // Cap divisor at number of available sockets or domains - else if (scope == 'socket') divisor = (fraction < subClusterTopology.socket.length) ? subClusterTopology.socket.length : fraction; - else if (scope == "memoryDomain") divisor = (fraction < subClusterTopology.memoryDomain.length) ? subClusterTopology.socket.length : fraction; - // Use Maximum Division for Smallest Scopes + if (scope == 'node') divisor = 1 // Node Scope: Always return unscaled (Maximum Scope) + // Partial Scopes: Get from Topologies + else if (scope == 'socket') divisor = subClusterTopology.socket.length; + else if (scope == "memoryDomain") divisor = subClusterTopology.memoryDomain.length; else if (scope == "core") divisor = subClusterTopology.core.length; - else if (scope == "hwthread") divisor = subClusterTopology.core.length; // alt. name for core + else if (scope == "hwthread") divisor = subClusterTopology.node.length; else if (scope == "accelerator") divisor = subClusterTopology.accelerators.length; else { console.log('Unknown scope, return default aggregation thresholds for sum', scope) diff --git a/web/frontend/src/job/Metric.svelte b/web/frontend/src/job/Metric.svelte index ca32d9f9..1beb88fb 100644 --- a/web/frontend/src/job/Metric.svelte +++ b/web/frontend/src/job/Metric.svelte @@ -178,8 +178,6 @@ timestep={selectedData.timestep} scope={selectedScope} metric={metricName} - numaccs={job.numAcc} - numhwthreads={job.numHWThreads} series={selectedSeries} {isShared} {zoomState} @@ -194,8 +192,6 @@ timestep={selectedData.timestep} scope={selectedScope} metric={metricName} - numaccs={job.numAcc} - numhwthreads={job.numHWThreads} series={selectedSeries} {isShared} {zoomState} From 88bd83b07e64d949020829976a091929ce7cc8c4 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 6 Mar 2026 10:19:46 +0100 Subject: [PATCH 335/341] add nullsafe fallbacks --- web/frontend/src/generic/plots/MetricPlot.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/frontend/src/generic/plots/MetricPlot.svelte b/web/frontend/src/generic/plots/MetricPlot.svelte index aa38d858..3969161d 100644 --- a/web/frontend/src/generic/plots/MetricPlot.svelte +++ b/web/frontend/src/generic/plots/MetricPlot.svelte @@ -300,11 +300,11 @@ let divisor; if (scope == 'node') divisor = 1 // Node Scope: Always return unscaled (Maximum Scope) // Partial Scopes: Get from Topologies - else if (scope == 'socket') divisor = subClusterTopology.socket.length; - else if (scope == "memoryDomain") divisor = subClusterTopology.memoryDomain.length; - else if (scope == "core") divisor = subClusterTopology.core.length; - else if (scope == "hwthread") divisor = subClusterTopology.node.length; - else if (scope == "accelerator") divisor = subClusterTopology.accelerators.length; + else if (scope == 'socket') divisor = subClusterTopology?.socket?.length || 1; + else if (scope == "memoryDomain") divisor = subClusterTopology?.memoryDomain?.length || 1; + else if (scope == "core") divisor = subClusterTopology?.core?.length || 1; + else if (scope == "hwthread") divisor = subClusterTopology?.node?.length || 1; + else if (scope == "accelerator") divisor = subClusterTopology?.accelerators?.length || 1; else { console.log('Unknown scope, return default aggregation thresholds for sum', scope) divisor = 1; From 70fea39d0342c81e21e497bebb1519c2c98f8313 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 6 Mar 2026 10:56:23 +0100 Subject: [PATCH 336/341] Add note on dynamic memory management for restarts --- ReleaseNotes.md | 2 ++ cmd/cc-backend/main.go | 1 + 2 files changed, 3 insertions(+) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 3d352f20..5447167e 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -271,6 +271,8 @@ For release specific notes visit the [ClusterCockpit Documentation](https://clus ## Known issues +- The new dynamic memory management is not bullet proof yet across restarts. We + will fix that in a subsequent patch release - Currently energy footprint metrics of type energy are ignored for calculating total energy. - With energy footprint metrics of type power the unit is ignored and it is diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 5b51b963..57c8d65b 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -395,6 +395,7 @@ func runServer(ctx context.Context) error { // Set GC percent if not configured if os.Getenv(envGOGC) == "" { + // trigger GC when heap grows 15% above the previous live set debug.SetGCPercent(15) } runtime.SystemdNotify(true, "running") From d2bc046fc64155a1d660e74149fc6f46789490da Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 9 Mar 2026 11:28:30 +0100 Subject: [PATCH 337/341] fix ranged filter GT and LT conditions, reduce energy filter preset --- internal/repository/jobQuery.go | 35 +++++++++---------- .../src/generic/filters/Energy.svelte | 2 +- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index c3fbe042..aacdebce 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -276,28 +276,26 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select return query } -// buildIntCondition creates a BETWEEN clause for integer range filters. -// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required +// buildIntCondition creates clauses for integer range filters, using BETWEEN only if required. func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder { - if cond.From != 0 && cond.To != 0 { + if cond.From != 1 && cond.To != 0 { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) - } else if cond.From != 0 { + } else if cond.From != 1 && cond.To == 0 { return query.Where(field+" >= ?", cond.From) - } else if cond.To != 0 { + } else if cond.From == 1 && cond.To != 0 { return query.Where(field+" <= ?", cond.To) } else { return query } } -// buildFloatCondition creates a BETWEEN clause for float range filters. -// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required +// buildFloatCondition creates a clauses for float range filters, using BETWEEN only if required. func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { - if cond.From != 0.0 && cond.To != 0.0 { + if cond.From != 1.0 && cond.To != 0.0 { return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) - } else if cond.From != 0.0 { + } else if cond.From != 1.0 && cond.To == 0.0 { return query.Where(field+" >= ?", cond.From) - } else if cond.To != 0.0 { + } else if cond.From == 1.0 && cond.To != 0.0 { return query.Where(field+" <= ?", cond.To) } else { return query @@ -336,16 +334,15 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui } } -// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column. -// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required -func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { +// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required. +func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(footprint)") - if condRange.From != 0.0 && condRange.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To) - } else if condRange.From != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") >= ?", condRange.From) - } else if condRange.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To) + if cond.From != 1.0 && cond.To != 0.0 { + return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To) + } else if cond.From != 1.0 && cond.To == 0.0 { + return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From) + } else if cond.From == 1.0 && cond.To != 0.0 { + return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To) } else { return query } diff --git a/web/frontend/src/generic/filters/Energy.svelte b/web/frontend/src/generic/filters/Energy.svelte index 648fdb4d..dc532c86 100644 --- a/web/frontend/src/generic/filters/Energy.svelte +++ b/web/frontend/src/generic/filters/Energy.svelte @@ -29,7 +29,7 @@ /* Const */ const minEnergyPreset = 1; - const maxEnergyPreset = 1000; + const maxEnergyPreset = 100; /* Derived */ // Pending From 282197ebefc551a3b4d844a1249a0d930abe0680 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 10 Mar 2026 06:01:31 +0100 Subject: [PATCH 338/341] fix: Round floats in tagger message Entire-Checkpoint: b68850c6fcff --- internal/tagger/classifyJob.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/tagger/classifyJob.go b/internal/tagger/classifyJob.go index 6a53fae8..1bad61f1 100644 --- a/internal/tagger/classifyJob.go +++ b/internal/tagger/classifyJob.go @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "maps" + "math" "os" "path/filepath" "strings" @@ -111,6 +112,29 @@ type JobClassTagger struct { getMetricConfig func(cluster, subCluster string) map[string]*schema.Metric } +// roundEnv returns a copy of env with all float64 values rounded to 2 decimal places. +// Nested map[string]any and map[string]float64 values are recursed into. +func roundEnv(env map[string]any) map[string]any { + rounded := make(map[string]any, len(env)) + for k, v := range env { + switch val := v.(type) { + case float64: + rounded[k] = math.Round(val*100) / 100 + case map[string]any: + rounded[k] = roundEnv(val) + case map[string]float64: + rm := make(map[string]float64, len(val)) + for mk, mv := range val { + rm[mk] = math.Round(mv*100) / 100 + } + rounded[k] = rm + default: + rounded[k] = v + } + } + return rounded +} + func (t *JobClassTagger) prepareRule(b []byte, fns string) { var rule RuleFormat if err := json.NewDecoder(bytes.NewReader(b)).Decode(&rule); err != nil { @@ -408,7 +432,7 @@ func (t *JobClassTagger) Match(job *schema.Job) { // process hint template var msg bytes.Buffer - if err := ri.hint.Execute(&msg, env); err != nil { + if err := ri.hint.Execute(&msg, roundEnv(env)); err != nil { cclog.Errorf("Template error: %s", err.Error()) continue } From cc38b17472e07eb8bff777c66407c3d8422b48a9 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 10 Mar 2026 17:02:09 +0100 Subject: [PATCH 339/341] fix wrong field checked vor json validity --- internal/repository/jobQuery.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index aacdebce..437801aa 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -63,7 +63,7 @@ func (r *JobRepository) QueryJobs( } } else { // Order by footprint JSON field values - query = query.Where("JSON_VALID(meta_data)") + query = query.Where("JSON_VALID(footprint)") switch order.Order { case model.SortDirectionEnumAsc: query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field)) @@ -335,14 +335,14 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui } // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required. -func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { +func buildFloatJSONCondition(jsonField string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(footprint)") if cond.From != 1.0 && cond.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To) + return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") BETWEEN ? AND ?", cond.From, cond.To) } else if cond.From != 1.0 && cond.To == 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From) + return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") >= ?", cond.From) } else if cond.From == 1.0 && cond.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To) + return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") <= ?", cond.To) } else { return query } From f3e796f3f59e6542ef74318451493dafb9b42f7b Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 10 Mar 2026 17:05:50 +0100 Subject: [PATCH 340/341] add nullsafes to node view --- web/frontend/src/Node.root.svelte | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/frontend/src/Node.root.svelte b/web/frontend/src/Node.root.svelte index 06056466..db424a2f 100644 --- a/web/frontend/src/Node.root.svelte +++ b/web/frontend/src/Node.root.svelte @@ -167,7 +167,7 @@ Selected Node - + @@ -259,7 +259,7 @@

    No dataset(s) returned for {item.name}

    -

    Metric has been disabled for subcluster {$nodeMetricsData.data.nodeMetrics[0].subCluster}.

    +

    Metric has been disabled for subcluster {$nodeMetricsData?.data?.nodeMetrics[0]?.subCluster}.

     {:else if item?.metric} @@ -267,7 +267,7 @@ metric={item.name} timestep={item.metric.timestep} cluster={clusterInfos.find((c) => c.name == cluster)} - subCluster={$nodeMetricsData.data.nodeMetrics[0].subCluster} + subCluster={$nodeMetricsData?.data?.nodeMetrics[0]?.subCluster} series={item.metric.series} enableFlip forNode @@ -286,17 +286,17 @@ {/snippet} ({ ...m, availability: checkMetricAvailability( globalMetrics, m.name, cluster, - $nodeMetricsData.data.nodeMetrics[0].subCluster, + $nodeMetricsData?.data?.nodeMetrics[0]?.subCluster, ), })) - .sort((a, b) => a.name.localeCompare(b.name))} + .sort((a, b) => a.name.localeCompare(b.name)) || []} itemsPerRow={ccconfig.plotConfiguration_plotsPerRow} {gridContent} /> From 5c726641629df4c51077d24d58065d04ded8e9f4 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 10 Mar 2026 18:15:24 +0100 Subject: [PATCH 341/341] bump frontend patch versions --- web/frontend/package-lock.json | 16 ++++++++-------- web/frontend/package.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 08b87ea3..8656abc4 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -20,14 +20,14 @@ "wonka": "^6.3.5" }, "devDependencies": { - "@rollup/plugin-commonjs": "^29.0.1", + "@rollup/plugin-commonjs": "^29.0.2", "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-terser": "^1.0.0", "@timohausmann/quadtree-js": "^1.2.6", "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.53.7" + "svelte": "^5.53.9" } }, "node_modules/@0no-co/graphql.web": { @@ -126,9 +126,9 @@ } }, "node_modules/@rollup/plugin-commonjs": { - "version": "29.0.1", - "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.1.tgz", - "integrity": "sha512-VUEHINN2rQEWPfNUR3mzidRObM1XZKXMQsaG6qBlDqd6M1qyw91nDZvcSozgyjt3x/QKrgKBc5MdxfdxAy6tdg==", + "version": "29.0.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.2.tgz", + "integrity": "sha512-S/ggWH1LU7jTyi9DxZOKyxpVd4hF/OZ0JrEbeLjXk/DFXwRny0tjD2c992zOUYQobLrVkRVMDdmHP16HKP7GRg==", "dev": true, "license": "MIT", "dependencies": { @@ -1193,9 +1193,9 @@ } }, "node_modules/svelte": { - "version": "5.53.7", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.7.tgz", - "integrity": "sha512-uxck1KI7JWtlfP3H6HOWi/94soAl23jsGJkBzN2BAWcQng0+lTrRNhxActFqORgnO9BHVd1hKJhG+ljRuIUWfQ==", + "version": "5.53.9", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.9.tgz", + "integrity": "sha512-MwDfWsN8qZzeP0jlQsWF4k/4B3csb3IbzCRggF+L/QqY7T8bbKvnChEo1cPZztF51HJQhilDbevWYl2LvXbquA==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", diff --git a/web/frontend/package.json b/web/frontend/package.json index fe535fcb..0c206c66 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -7,14 +7,14 @@ "dev": "rollup -c -w" }, "devDependencies": { - "@rollup/plugin-commonjs": "^29.0.1", + "@rollup/plugin-commonjs": "^29.0.2", "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-terser": "^1.0.0", "@timohausmann/quadtree-js": "^1.2.6", "rollup": "^4.59.0", "rollup-plugin-css-only": "^4.5.5", "rollup-plugin-svelte": "^7.2.3", - "svelte": "^5.53.7" + "svelte": "^5.53.9" }, "dependencies": { "@rollup/plugin-replace": "^6.0.3",