From 4083de2a514b91ffd24791e12e63a04622109190 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 9 Dec 2025 10:26:55 +0100 Subject: [PATCH 01/30] Add public dashboard and route, add DoubleMetricPlot and GQL queries - add roofline legend display switch - small fixes --- api/schema.graphqls | 19 + internal/graph/generated/generated.go | 643 +++++++++++++++++ internal/graph/model/models_gen.go | 12 + internal/graph/schema.resolvers.go | 81 +++ internal/routerConfig/routes.go | 31 +- web/frontend/rollup.config.mjs | 1 + web/frontend/src/DashPublic.root.svelte | 671 ++++++++++++++++++ web/frontend/src/Header.svelte | 2 +- web/frontend/src/Status.root.svelte | 68 +- web/frontend/src/dashpublic.entrypoint.js | 13 + .../src/generic/plots/DoubleMetricPlot.svelte | 640 +++++++++++++++++ .../src/generic/plots/Roofline.svelte | 3 +- web/frontend/src/generic/plots/Stacked.svelte | 2 +- web/frontend/src/header/NavbarLinks.svelte | 28 + web/frontend/src/status.entrypoint.js | 1 + web/frontend/src/status/DashDetails.svelte | 82 +++ web/frontend/src/status/DashInternal.svelte | 605 ++++++++++++++++ .../{ => dashdetails}/StatisticsDash.svelte | 10 +- .../{ => dashdetails}/StatusDash.svelte | 14 +- .../status/{ => dashdetails}/UsageDash.svelte | 8 +- web/templates/base.tmpl | 65 +- web/templates/monitoring/dashboard.tmpl | 14 + web/templates/monitoring/status.tmpl | 1 + 23 files changed, 2918 insertions(+), 96 deletions(-) create mode 100644 web/frontend/src/DashPublic.root.svelte create mode 100644 web/frontend/src/dashpublic.entrypoint.js create mode 100644 web/frontend/src/generic/plots/DoubleMetricPlot.svelte create mode 100644 web/frontend/src/status/DashDetails.svelte create mode 100644 web/frontend/src/status/DashInternal.svelte rename web/frontend/src/status/{ => dashdetails}/StatisticsDash.svelte (92%) rename web/frontend/src/status/{ => dashdetails}/StatusDash.svelte (97%) rename web/frontend/src/status/{ => dashdetails}/UsageDash.svelte (98%) create mode 100644 web/templates/monitoring/dashboard.tmpl diff --git a/api/schema.graphqls b/api/schema.graphqls index 8f5e1c7c..1c81e6b6 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -164,6 +164,13 @@ type JobMetricWithName { metric: JobMetric! } +type ClusterMetricWithName { + name: String! + unit: Unit + timestep: Int! + data: [NullableFloat!]! +} + type JobMetric { unit: Unit timestep: Int! @@ -267,6 +274,11 @@ type NodeMetrics { metrics: [JobMetricWithName!]! } +type ClusterMetrics { + nodeCount: Int! + metrics: [ClusterMetricWithName!]! +} + type NodesResultList { items: [NodeMetrics!]! offset: Int @@ -385,6 +397,13 @@ type Query { page: PageRequest resolution: Int ): NodesResultList! + + clusterMetrics( + cluster: String! + metrics: [String!] + from: Time! + to: Time! + ): ClusterMetrics! } type Mutation { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index a3b1a1d3..b1489420 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -66,6 +66,18 @@ type ComplexityRoot struct { SubClusters func(childComplexity int) int } + ClusterMetricWithName struct { + Data func(childComplexity int) int + Name func(childComplexity int) int + Timestep func(childComplexity int) int + Unit func(childComplexity int) int + } + + ClusterMetrics struct { + Metrics func(childComplexity int) int + NodeCount func(childComplexity int) int + } + ClusterSupport struct { Cluster func(childComplexity int) int SubClusters func(childComplexity int) int @@ -319,6 +331,7 @@ type ComplexityRoot struct { Query struct { AllocatedNodes func(childComplexity int, cluster string) int + ClusterMetrics func(childComplexity int, cluster string, metrics []string, from time.Time, to time.Time) int Clusters func(childComplexity int) int GlobalMetrics func(childComplexity int) int Job func(childComplexity int, id string) int @@ -485,6 +498,7 @@ type QueryResolver interface { RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) + ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) } type SubClusterResolver interface { NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) @@ -551,6 +565,48 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return e.complexity.Cluster.SubClusters(childComplexity), true + case "ClusterMetricWithName.data": + if e.complexity.ClusterMetricWithName.Data == nil { + break + } + + return e.complexity.ClusterMetricWithName.Data(childComplexity), true + + case "ClusterMetricWithName.name": + if e.complexity.ClusterMetricWithName.Name == nil { + break + } + + return e.complexity.ClusterMetricWithName.Name(childComplexity), true + + case "ClusterMetricWithName.timestep": + if e.complexity.ClusterMetricWithName.Timestep == nil { + break + } + + return e.complexity.ClusterMetricWithName.Timestep(childComplexity), true + + case "ClusterMetricWithName.unit": + if e.complexity.ClusterMetricWithName.Unit == nil { + break + } + + return e.complexity.ClusterMetricWithName.Unit(childComplexity), true + + case "ClusterMetrics.metrics": + if e.complexity.ClusterMetrics.Metrics == nil { + break + } + + return e.complexity.ClusterMetrics.Metrics(childComplexity), true + + case "ClusterMetrics.nodeCount": + if e.complexity.ClusterMetrics.NodeCount == nil { + break + } + + return e.complexity.ClusterMetrics.NodeCount(childComplexity), true + case "ClusterSupport.cluster": if e.complexity.ClusterSupport.Cluster == nil { break @@ -1699,6 +1755,18 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return e.complexity.Query.AllocatedNodes(childComplexity, args["cluster"].(string)), true + case "Query.clusterMetrics": + if e.complexity.Query.ClusterMetrics == nil { + break + } + + args, err := ec.field_Query_clusterMetrics_args(ctx, rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.ClusterMetrics(childComplexity, args["cluster"].(string), args["metrics"].([]string), args["from"].(time.Time), args["to"].(time.Time)), true + case "Query.clusters": if e.complexity.Query.Clusters == nil { break @@ -2577,6 +2645,13 @@ type JobMetricWithName { metric: JobMetric! } +type ClusterMetricWithName { + name: String! + unit: Unit + timestep: Int! + data: [NullableFloat!]! +} + type JobMetric { unit: Unit timestep: Int! @@ -2680,6 +2755,11 @@ type NodeMetrics { metrics: [JobMetricWithName!]! } +type ClusterMetrics { + nodeCount: Int! + metrics: [ClusterMetricWithName!]! +} + type NodesResultList { items: [NodeMetrics!]! offset: Int @@ -2798,6 +2878,13 @@ type Query { page: PageRequest resolution: Int ): NodesResultList! + + clusterMetrics( + cluster: String! + metrics: [String!] + from: Time! + to: Time! + ): ClusterMetrics! } type Mutation { @@ -3074,6 +3161,32 @@ func (ec *executionContext) field_Query_allocatedNodes_args(ctx context.Context, return args, nil } +func (ec *executionContext) field_Query_clusterMetrics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { + var err error + args := map[string]any{} + arg0, err := graphql.ProcessArgField(ctx, rawArgs, "cluster", ec.unmarshalNString2string) + if err != nil { + return nil, err + } + args["cluster"] = arg0 + arg1, err := graphql.ProcessArgField(ctx, rawArgs, "metrics", ec.unmarshalOString2ᚕstringᚄ) + if err != nil { + return nil, err + } + args["metrics"] = arg1 + arg2, err := graphql.ProcessArgField(ctx, rawArgs, "from", ec.unmarshalNTime2timeᚐTime) + if err != nil { + return nil, err + } + args["from"] = arg2 + arg3, err := graphql.ProcessArgField(ctx, rawArgs, "to", ec.unmarshalNTime2timeᚐTime) + if err != nil { + return nil, err + } + args["to"] = arg3 + return args, nil +} + func (ec *executionContext) field_Query_jobMetrics_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} @@ -3784,6 +3897,283 @@ func (ec *executionContext) fieldContext_Cluster_subClusters(_ context.Context, return fc, nil } +func (ec *executionContext) _ClusterMetricWithName_name(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_name(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Name, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_name(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_unit(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_unit(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Unit, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*schema.Unit) + fc.Result = res + return ec.marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_unit(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "base": + return ec.fieldContext_Unit_base(ctx, field) + case "prefix": + return ec.fieldContext_Unit_prefix(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type Unit", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_timestep(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_timestep(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Timestep, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_timestep(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetricWithName_data(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetricWithName) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetricWithName_data(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Data, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetricWithName_data(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetricWithName", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type NullableFloat does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetrics_nodeCount(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetrics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetrics_nodeCount(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.NodeCount, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetrics_nodeCount(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetrics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ClusterMetrics_metrics(ctx context.Context, field graphql.CollectedField, obj *model.ClusterMetrics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ClusterMetrics_metrics(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.Metrics, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.ClusterMetricWithName) + fc.Result = res + return ec.marshalNClusterMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithNameᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ClusterMetrics_metrics(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ClusterMetrics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "name": + return ec.fieldContext_ClusterMetricWithName_name(ctx, field) + case "unit": + return ec.fieldContext_ClusterMetricWithName_unit(ctx, field) + case "timestep": + return ec.fieldContext_ClusterMetricWithName_timestep(ctx, field) + case "data": + return ec.fieldContext_ClusterMetricWithName_data(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ClusterMetricWithName", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _ClusterSupport_cluster(ctx context.Context, field graphql.CollectedField, obj *schema.ClusterSupport) (ret graphql.Marshaler) { fc, err := ec.fieldContext_ClusterSupport_cluster(ctx, field) if err != nil { @@ -12353,6 +12743,67 @@ func (ec *executionContext) fieldContext_Query_nodeMetricsList(ctx context.Conte return fc, nil } +func (ec *executionContext) _Query_clusterMetrics(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_clusterMetrics(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().ClusterMetrics(rctx, fc.Args["cluster"].(string), fc.Args["metrics"].([]string), fc.Args["from"].(time.Time), fc.Args["to"].(time.Time)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*model.ClusterMetrics) + fc.Result = res + return ec.marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_clusterMetrics(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "nodeCount": + return ec.fieldContext_ClusterMetrics_nodeCount(ctx, field) + case "metrics": + return ec.fieldContext_ClusterMetrics_metrics(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ClusterMetrics", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_clusterMetrics_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return fc, err + } + return fc, nil +} + func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query___type(ctx, field) if err != nil { @@ -17527,6 +17978,101 @@ func (ec *executionContext) _Cluster(ctx context.Context, sel ast.SelectionSet, return out } +var clusterMetricWithNameImplementors = []string{"ClusterMetricWithName"} + +func (ec *executionContext) _ClusterMetricWithName(ctx context.Context, sel ast.SelectionSet, obj *model.ClusterMetricWithName) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, clusterMetricWithNameImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ClusterMetricWithName") + case "name": + out.Values[i] = ec._ClusterMetricWithName_name(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "unit": + out.Values[i] = ec._ClusterMetricWithName_unit(ctx, field, obj) + case "timestep": + out.Values[i] = ec._ClusterMetricWithName_timestep(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "data": + out.Values[i] = ec._ClusterMetricWithName_data(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + +var clusterMetricsImplementors = []string{"ClusterMetrics"} + +func (ec *executionContext) _ClusterMetrics(ctx context.Context, sel ast.SelectionSet, obj *model.ClusterMetrics) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, clusterMetricsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ClusterMetrics") + case "nodeCount": + out.Values[i] = ec._ClusterMetrics_nodeCount(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "metrics": + out.Values[i] = ec._ClusterMetrics_metrics(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var clusterSupportImplementors = []string{"ClusterSupport"} func (ec *executionContext) _ClusterSupport(ctx context.Context, sel ast.SelectionSet, obj *schema.ClusterSupport) graphql.Marshaler { @@ -20101,6 +20647,28 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) + case "clusterMetrics": + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_clusterMetrics(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + } + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "__type": out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { @@ -21205,6 +21773,74 @@ func (ec *executionContext) marshalNCluster2ᚖgithubᚗcomᚋClusterCockpitᚋc return ec._Cluster(ctx, sel, v) } +func (ec *executionContext) marshalNClusterMetricWithName2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithNameᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.ClusterMetricWithName) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNClusterMetricWithName2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetricWithName(ctx context.Context, sel ast.SelectionSet, v *model.ClusterMetricWithName) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._ClusterMetricWithName(ctx, sel, v) +} + +func (ec *executionContext) marshalNClusterMetrics2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx context.Context, sel ast.SelectionSet, v model.ClusterMetrics) graphql.Marshaler { + return ec._ClusterMetrics(ctx, sel, &v) +} + +func (ec *executionContext) marshalNClusterMetrics2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐClusterMetrics(ctx context.Context, sel ast.SelectionSet, v *model.ClusterMetrics) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._ClusterMetrics(ctx, sel, v) +} + func (ec *executionContext) marshalNClusterSupport2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐClusterSupport(ctx context.Context, sel ast.SelectionSet, v schema.ClusterSupport) graphql.Marshaler { return ec._ClusterSupport(ctx, sel, &v) } @@ -24142,6 +24778,13 @@ func (ec *executionContext) marshalOUnit2githubᚗcomᚋClusterCockpitᚋccᚑli return ec._Unit(ctx, sel, &v) } +func (ec *executionContext) marshalOUnit2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋschemaᚐUnit(ctx context.Context, sel ast.SelectionSet, v *schema.Unit) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return ec._Unit(ctx, sel, v) +} + func (ec *executionContext) marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐUser(ctx context.Context, sel ast.SelectionSet, v *model.User) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 4cb414eb..63b2da5d 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -13,6 +13,18 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) +type ClusterMetricWithName struct { + Name string `json:"name"` + Unit *schema.Unit `json:"unit,omitempty"` + Timestep int `json:"timestep"` + Data []schema.Float `json:"data"` +} + +type ClusterMetrics struct { + NodeCount int `json:"nodeCount"` + Metrics []*ClusterMetricWithName `json:"metrics"` +} + type Count struct { Name string `json:"name"` Count int `json:"count"` diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index e4901c46..624ddc64 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "math" "regexp" "slices" "strconv" @@ -973,6 +974,86 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub return nodeMetricsListResult, nil } +// ClusterMetrics is the resolver for the clusterMetrics field. +func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) { + + user := repository.GetUserFromContext(ctx) + if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { + return nil, errors.New("you need to be administrator or support staff for this query") + } + + if metrics == nil { + for _, mc := range archive.GetCluster(cluster).MetricConfig { + metrics = append(metrics, mc.Name) + } + } + + // 'nodes' == nil -> Defaults to all nodes of cluster for existing query workflow + scopes := []schema.MetricScope{"node"} + data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx) + if err != nil { + cclog.Warn("error while loading node data") + return nil, err + } + + clusterMetricData := make([]*model.ClusterMetricWithName, 0) + clusterMetrics := model.ClusterMetrics{NodeCount: 0, Metrics: clusterMetricData} + + collectorTimestep := make(map[string]int) + collectorUnit := make(map[string]schema.Unit) + collectorData := make(map[string][]schema.Float) + + for _, metrics := range data { + clusterMetrics.NodeCount += 1 + for metric, scopedMetrics := range metrics { + _, ok := collectorData[metric] + if !ok { + collectorData[metric] = make([]schema.Float, 0) + for _, scopedMetric := range scopedMetrics { + // Collect Info + collectorTimestep[metric] = scopedMetric.Timestep + collectorUnit[metric] = scopedMetric.Unit + // Collect Initial Data + for _, ser := range scopedMetric.Series { + for _, val := range ser.Data { + collectorData[metric] = append(collectorData[metric], val) + } + } + } + } else { + // Sum up values by index + for _, scopedMetric := range scopedMetrics { + // For This Purpose (Cluster_Wide-Sum of Node Metrics) OK + for _, ser := range scopedMetric.Series { + for i, val := range ser.Data { + collectorData[metric][i] += val + } + } + } + } + } + } + + for metricName, data := range collectorData { + cu := collectorUnit[metricName] + roundedData := make([]schema.Float, 0) + for _, val := range data { + roundedData = append(roundedData, schema.Float((math.Round(float64(val)*100.0) / 100.0))) + } + + cm := model.ClusterMetricWithName{ + Name: metricName, + Unit: &cu, + Timestep: collectorTimestep[metricName], + Data: roundedData, + } + + clusterMetrics.Metrics = append(clusterMetrics.Metrics, &cm) + } + + return &clusterMetrics, nil +} + // NumberOfNodes is the resolver for the numberOfNodes field. func (r *subClusterResolver) NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) { nodeList, err := archive.ParseNodeList(obj.Nodes) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 9c19de52..71edeefb 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -47,7 +47,9 @@ var routes []Route = []Route{ {"/monitoring/systems/list/{cluster}/{subcluster}", "monitoring/systems.tmpl", "Cluster Node List - ClusterCockpit", false, setupClusterListRoute}, {"/monitoring/node/{cluster}/{hostname}", "monitoring/node.tmpl", "Node - ClusterCockpit", false, setupNodeRoute}, {"/monitoring/analysis/{cluster}", "monitoring/analysis.tmpl", "Analysis - ClusterCockpit", true, setupAnalysisRoute}, - {"/monitoring/status/{cluster}", "monitoring/status.tmpl", "Status of - ClusterCockpit", false, setupClusterStatusRoute}, + {"/monitoring/status/{cluster}", "monitoring/status.tmpl", " Dashboard - ClusterCockpit", false, setupClusterStatusRoute}, + {"/monitoring/status/detail/{cluster}", "monitoring/status.tmpl", "Status of - ClusterCockpit", false, setupClusterDetailRoute}, + {"/monitoring/dashboard/{cluster}", "monitoring/dashboard.tmpl", " Dashboard - ClusterCockpit", false, setupDashboardRoute}, } func setupHomeRoute(i InfoType, r *http.Request) InfoType { @@ -117,6 +119,33 @@ func setupClusterStatusRoute(i InfoType, r *http.Request) InfoType { vars := mux.Vars(r) i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] + i["displayType"] = "DASHBOARD" + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupClusterDetailRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["id"] = vars["cluster"] + i["cluster"] = vars["cluster"] + i["displayType"] = "DETAILS" + from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") + if from != "" || to != "" { + i["from"] = from + i["to"] = to + } + return i +} + +func setupDashboardRoute(i InfoType, r *http.Request) InfoType { + vars := mux.Vars(r) + i["id"] = vars["cluster"] + i["cluster"] = vars["cluster"] + i["displayType"] = "PUBLIC" from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") if from != "" || to != "" { i["from"] = from diff --git a/web/frontend/rollup.config.mjs b/web/frontend/rollup.config.mjs index c92d8155..6b7cf884 100644 --- a/web/frontend/rollup.config.mjs +++ b/web/frontend/rollup.config.mjs @@ -74,5 +74,6 @@ export default [ entrypoint('node', 'src/node.entrypoint.js'), entrypoint('analysis', 'src/analysis.entrypoint.js'), entrypoint('status', 'src/status.entrypoint.js'), + entrypoint('dashpublic', 'src/dashpublic.entrypoint.js'), entrypoint('config', 'src/config.entrypoint.js') ]; diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte new file mode 100644 index 00000000..e344aa42 --- /dev/null +++ b/web/frontend/src/DashPublic.root.svelte @@ -0,0 +1,671 @@ + + + + + + +

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

+ + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + + + + + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + + {#if $statusQuery.error} + + Error Requesting StatusQuery: {$statusQuery.error.message} + + {/if} + {#if $statesTimed.error} + + Error Requesting StatesTimed: {$statesTimed.error.message} + + {/if} + {#if $topJobsQuery.error} + + Error Requesting TopJobsQuery: {$topJobsQuery.error.message} + + {/if} + {#if $nodeStatusQuery.error} + + Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + + {/if} + + + {:else} + + + + + Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" + {[...clusterInfo?.processorTypes].toString()} + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + {#if clusterInfo?.totalAccs !== 0} + + + + + + {/if} +
{clusterInfo?.runningJobs} Running Jobs{clusterInfo?.activeUsers} Active Users
+ Flop Rate (Any) + + Memory BW Rate +
+ {clusterInfo?.flopRate} + {clusterInfo?.flopRateUnit} + + {clusterInfo?.memBwRate} + {clusterInfo?.memBwRateUnit} +
Allocated Nodes
+ +
{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes} + Nodes
Allocated Cores
+ +
{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)} + Cores
Allocated Accelerators
+ +
{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs} + Accelerators
+ + + + + + +
+ {#key refinedStateData} +

+ Current Node States +

+ sd.count, + )} + entities={refinedStateData.map( + (sd) => sd.state, + )} + /> + {/key} +
+ + + {#key refinedStateData} + + + + + + + {#each refinedStateData as sd, i} + + + + + + {/each} +
Current StateNodes
{sd.state}{sd.count}
+ {/key} + + + + + + + + Infos + + + Contents + + + + +
+ {#key $statusQuery?.data?.nodeMetrics} + + {/key} +
+ + +
+ +
+ + + +
+ {#key $statesTimed?.data?.nodeStates} + + {/key} +
+ + + {/if} + + diff --git a/web/frontend/src/Header.svelte b/web/frontend/src/Header.svelte index 98a796a2..f7ceac2e 100644 --- a/web/frontend/src/Header.svelte +++ b/web/frontend/src/Header.svelte @@ -120,7 +120,7 @@ href: "/monitoring/status/", icon: "clipboard-data", perCluster: true, - listOptions: false, + listOptions: true, menu: "Info", }, ]; diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 3d9002a1..8175681e 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -6,77 +6,43 @@ --> - - - + - -{#if $initq.fetching} - +{#if displayType !== "DASHBOARD" && displayType !== "DETAILS"} + - - - -{:else if $initq.error} - - - {$initq.error.message} + Unknown displayList type! {:else} - - - - - - - - - - - - - - - - - - - - - + {#if displayStatusDetail} + + + {:else} + + + {/if} {/if} diff --git a/web/frontend/src/dashpublic.entrypoint.js b/web/frontend/src/dashpublic.entrypoint.js new file mode 100644 index 00000000..47287c7a --- /dev/null +++ b/web/frontend/src/dashpublic.entrypoint.js @@ -0,0 +1,13 @@ +import { mount } from 'svelte'; +// import {} from './header.entrypoint.js' +import DashPublic from './DashPublic.root.svelte' + +mount(DashPublic, { + target: document.getElementById('svelte-app'), + props: { + presetCluster: infos.cluster, + }, + context: new Map([ + ['cc-config', clusterCockpitConfig] + ]) +}) diff --git a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte new file mode 100644 index 00000000..94acf45a --- /dev/null +++ b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte @@ -0,0 +1,640 @@ + + + + + +{#if metricData[0]?.data && metricData[0]?.data?.length > 0} +
+{:else} + Cannot render plot: No series data returned for {cluster} +{/if} diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index 79ece220..6425275a 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -36,6 +36,7 @@ subCluster = null, allowSizeChange = false, useColors = true, + useLegend = true, width = 600, height = 380, } = $props(); @@ -534,7 +535,7 @@ width: width, height: height, legend: { - show: true, + show: useLegend, }, cursor: { dataIdx: (u, seriesIdx) => { diff --git a/web/frontend/src/generic/plots/Stacked.svelte b/web/frontend/src/generic/plots/Stacked.svelte index 4c532db1..2616f9ae 100644 --- a/web/frontend/src/generic/plots/Stacked.svelte +++ b/web/frontend/src/generic/plots/Stacked.svelte @@ -156,7 +156,7 @@ { scale: "y", grid: { show: true }, - labelFont: "sans-serif", + // labelFont: "sans-serif", label: ylabel + (yunit ? ` (${yunit})` : ''), // values: (u, vals) => vals.map((v) => formatNumber(v)), }, diff --git a/web/frontend/src/header/NavbarLinks.svelte b/web/frontend/src/header/NavbarLinks.svelte index 41454d12..04fec0ea 100644 --- a/web/frontend/src/header/NavbarLinks.svelte +++ b/web/frontend/src/header/NavbarLinks.svelte @@ -64,6 +64,34 @@ {/each} + {:else if item.title === 'Status'} + + + + {item.title} + + + {#each clusters as cluster} + + + {cluster.name} + + + + Status Dashboard + + + Status Details + + + + {/each} + + {:else} diff --git a/web/frontend/src/status.entrypoint.js b/web/frontend/src/status.entrypoint.js index c3407c13..e21b3612 100644 --- a/web/frontend/src/status.entrypoint.js +++ b/web/frontend/src/status.entrypoint.js @@ -6,6 +6,7 @@ mount(Status, { target: document.getElementById('svelte-app'), props: { presetCluster: infos.cluster, + displayType: displayType, }, context: new Map([ ['cc-config', clusterCockpitConfig] diff --git a/web/frontend/src/status/DashDetails.svelte b/web/frontend/src/status/DashDetails.svelte new file mode 100644 index 00000000..410d8df4 --- /dev/null +++ b/web/frontend/src/status/DashDetails.svelte @@ -0,0 +1,82 @@ + + + + + + + + +

Current Status of Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}"

+ + + + +{#if $initq.fetching} + + + + + +{:else if $initq.error} + + + {$initq.error.message} + + +{:else} + + + + + + + + + + + + + + + + + + + + + +{/if} diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte new file mode 100644 index 00000000..73fa639d --- /dev/null +++ b/web/frontend/src/status/DashInternal.svelte @@ -0,0 +1,605 @@ + + + + + + +

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

+ + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + + + + + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + + {#if $statusQuery.error} + + Error Requesting StatusQuery: {$statusQuery.error.message} + + {/if} + {#if $statesTimed.error} + + Error Requesting StatesTimed: {$statesTimed.error.message} + + {/if} + {#if $topJobsQuery.error} + + Error Requesting TopJobsQuery: {$topJobsQuery.error.message} + + {/if} + {#if $nodeStatusQuery.error} + + Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + + {/if} + + + {:else} + + + + + Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" + {[...clusterInfo?.processorTypes].toString()} + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + {#if clusterInfo?.totalAccs !== 0} + + + + + + {/if} +
{clusterInfo?.runningJobs} Running Jobs{clusterInfo?.activeUsers} Active Users
+ Flop Rate (Any) + + Memory BW Rate +
+ {clusterInfo?.flopRate} + {clusterInfo?.flopRateUnit} + + {clusterInfo?.memBwRate} + {clusterInfo?.memBwRateUnit} +
Allocated Nodes
+ +
{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes} + Nodes
Allocated Cores
+ +
{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)} + Cores
Allocated Accelerators
+ +
{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs} + Accelerators
+ + + + + + +
+

+ Top Projects: Jobs +

+ tp['totalJobs'], + )} + entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} + /> +
+ + + + + + + + + {#each $topJobsQuery.data.jobsStatistics as tp, i} + + + + + + {/each} +
ProjectJobs
+ {scrambleNames ? scramble(tp.id) : tp.id} + + {tp['totalJobs']}
+ + + + +
+ {#key $statusQuery?.data?.jobsMetricStats} + + {/key} +
+ + + {#if clusterInfo?.totalAccs == 0} + + {:else} + + {/if} + + +
+ {#key $statesTimed?.data?.nodeStates} + + {/key} +
+ + +
+ {#key $statesTimed?.data?.healthStates} + + {/key} +
+ + + {/if} + + diff --git a/web/frontend/src/status/StatisticsDash.svelte b/web/frontend/src/status/dashdetails/StatisticsDash.svelte similarity index 92% rename from web/frontend/src/status/StatisticsDash.svelte rename to web/frontend/src/status/dashdetails/StatisticsDash.svelte index efd7a4cb..0a1cd46f 100644 --- a/web/frontend/src/status/StatisticsDash.svelte +++ b/web/frontend/src/status/dashdetails/StatisticsDash.svelte @@ -22,11 +22,11 @@ } from "@urql/svelte"; import { convert2uplot, - } from "../generic/utils.js"; - import PlotGrid from "../generic/PlotGrid.svelte"; - import Histogram from "../generic/plots/Histogram.svelte"; - import HistogramSelection from "../generic/select/HistogramSelection.svelte"; - import Refresher from "../generic/helper/Refresher.svelte"; + } from "../../generic/utils.js"; + import PlotGrid from "../../generic/PlotGrid.svelte"; + import Histogram from "../../generic/plots/Histogram.svelte"; + import HistogramSelection from "../../generic/select/HistogramSelection.svelte"; + import Refresher from "../../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { diff --git a/web/frontend/src/status/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte similarity index 97% rename from web/frontend/src/status/StatusDash.svelte rename to web/frontend/src/status/dashdetails/StatusDash.svelte index 03a8cc4a..ee604836 100644 --- a/web/frontend/src/status/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -22,12 +22,12 @@ gql, getContextClient, } from "@urql/svelte"; - import { formatDurationTime } from "../generic/units.js"; - import Refresher from "../generic/helper/Refresher.svelte"; - import TimeSelection from "../generic/select/TimeSelection.svelte"; - import Roofline from "../generic/plots/Roofline.svelte"; - import Pie, { colors } from "../generic/plots/Pie.svelte"; - import Stacked from "../generic/plots/Stacked.svelte"; + import { formatDurationTime } from "../../generic/units.js"; + import Refresher from "../../generic/helper/Refresher.svelte"; + import TimeSelection from "../../generic/select/TimeSelection.svelte"; + import Roofline from "../../generic/plots/Roofline.svelte"; + import Pie, { colors } from "../../generic/plots/Pie.svelte"; + import Stacked from "../../generic/plots/Stacked.svelte"; /* Svelte 5 Props */ let { @@ -83,7 +83,7 @@ } `, variables: { - filter: { cluster: { eq: cluster }, timeStart: stackedFrom}, + filter: { cluster: { eq: cluster }, timeStart: 1760096999}, typeNode: "node", typeHealth: "health" }, diff --git a/web/frontend/src/status/UsageDash.svelte b/web/frontend/src/status/dashdetails/UsageDash.svelte similarity index 98% rename from web/frontend/src/status/UsageDash.svelte rename to web/frontend/src/status/dashdetails/UsageDash.svelte index 74dd7a99..2071465a 100644 --- a/web/frontend/src/status/UsageDash.svelte +++ b/web/frontend/src/status/dashdetails/UsageDash.svelte @@ -27,10 +27,10 @@ scramble, scrambleNames, convert2uplot, - } from "../generic/utils.js"; - import Pie, { colors } from "../generic/plots/Pie.svelte"; - import Histogram from "../generic/plots/Histogram.svelte"; - import Refresher from "../generic/helper/Refresher.svelte"; + } from "../../generic/utils.js"; + import Pie, { colors } from "../../generic/plots/Pie.svelte"; + import Histogram from "../../generic/plots/Histogram.svelte"; + import Refresher from "../../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { diff --git a/web/templates/base.tmpl b/web/templates/base.tmpl index 7fd35a0a..28eab339 100644 --- a/web/templates/base.tmpl +++ b/web/templates/base.tmpl @@ -23,34 +23,49 @@ - {{block "navigation" .}} -
- {{end}} + {{if eq .Infos.displayType "PUBLIC"}} +
+
+ {{block "content-public" .}} + Whoops, you should not see this... [MAIN] + {{end}} +
+
-
-
- {{block "content" .}} - Whoops, you should not see this... - {{end}} -
-
+ {{block "javascript-public" .}} + Whoops, you should not see this... [JS] + {{end}} - {{block "footer" .}} -
- -
    -
  • Version {{ .Build.Version }}
    • -
  • Hash {{ .Build.Hash }}
    • -
  • Built {{ .Build.Buildtime }}
    • -
-
- {{end}} + {{else}} + {{block "navigation" .}} +
+ {{end}} - {{block "javascript" .}} - +
+
+ {{block "content" .}} + Whoops, you should not see this... [MAIN] + {{end}} +
+
+ + {{block "footer" .}} +
+ +
    +
  • Version {{ .Build.Version }}
    • +
  • Hash {{ .Build.Hash }}
    • +
  • Built {{ .Build.Buildtime }}
    • +
+
+ {{end}} + + {{block "javascript" .}} + + {{end}} {{end}} diff --git a/web/templates/monitoring/dashboard.tmpl b/web/templates/monitoring/dashboard.tmpl new file mode 100644 index 00000000..06666cde --- /dev/null +++ b/web/templates/monitoring/dashboard.tmpl @@ -0,0 +1,14 @@ +{{define "content-public"}} +
+{{end}} + +{{define "stylesheets"}} + +{{end}} +{{define "javascript-public"}} + + +{{end}} diff --git a/web/templates/monitoring/status.tmpl b/web/templates/monitoring/status.tmpl index 15aff693..b6a84140 100644 --- a/web/templates/monitoring/status.tmpl +++ b/web/templates/monitoring/status.tmpl @@ -8,6 +8,7 @@ {{define "javascript"}} From 6e385db378b314f0ba31e0b74d05a4baa30b7da1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 11 Dec 2025 18:51:19 +0100 Subject: [PATCH 02/30] color roofline plot, add option to match pie and table color for ndoestate --- web/frontend/src/DashPublic.root.svelte | 201 +++++++++--------- .../src/generic/plots/DoubleMetricPlot.svelte | 4 +- web/frontend/src/generic/plots/Pie.svelte | 13 +- .../src/generic/plots/Roofline.svelte | 189 ++++++++++++++-- .../src/generic/plots/RooflineLegacy.svelte | 8 +- web/frontend/src/generic/plots/Stacked.svelte | 18 +- 6 files changed, 304 insertions(+), 129 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index e344aa42..2e060126 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -82,7 +82,7 @@ } `, variables: { - filter: { cluster: { eq: presetCluster }, timeStart: 1760096999}, // DEBUG VALUE, use StackedFrom + filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, // DEBUG VALUE 1760096999, use StackedFrom typeNode: "node", typeHealth: "health" }, @@ -97,6 +97,7 @@ query ( $cluster: String! $metrics: [String!] + # $nmetrics: [String!] $from: Time! $to: Time! $clusterFrom: Time! @@ -183,7 +184,7 @@ totalCores totalAccs } - # TEST + # ClusterMetrics for doubleMetricPlot clusterMetrics( cluster: $cluster metrics: $metrics @@ -205,7 +206,8 @@ `, variables: { cluster: presetCluster, - metrics: ["flops_any", "mem_bw"], // Fixed names for roofline and status bars + metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline + // nmetrics: ["cpu_load", "acc_utilization"], // Metrics for Node Graph from: from.toISOString(), clusterFrom: clusterFrom.toISOString(), to: to.toISOString(), @@ -349,18 +351,18 @@ }); /* Functions */ - function legendColors(targetIdx, useAltColors) { - // Reuses first color if targetIdx overflows - let c; - if (useCbColors) { - c = [...colors['colorblind']]; - } else if (useAltColors) { - c = [...colors['alternative']]; - } else { - c = [...colors['default']]; - } - return c[(c.length + targetIdx) % c.length]; - } + // function legendColors(targetIdx, useAltColors) { + // // Reuses first color if targetIdx overflows + // let c; + // if (useCbColors) { + // c = [...colors['colorblind']]; + // } else if (useAltColors) { + // c = [...colors['alternative']]; + // } else { + // c = [...colors['default']]; + // } + // return c[(c.length + targetIdx) % c.length]; + // } function transformNodesStatsToData(subclusterData) { let data = null @@ -425,10 +427,10 @@ - + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} @@ -461,13 +463,24 @@ {:else} - - - - + + + + +

Cluster {presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}

+ + +

CPU(s)

{[...clusterInfo?.processorTypes].join(', ')}

+ + + + + + + @@ -545,77 +558,7 @@ - - - -
- {#key refinedStateData} -

- Current Node States -

- sd.count, - )} - entities={refinedStateData.map( - (sd) => sd.state, - )} - /> - {/key} -
- - - {#key refinedStateData} -
- - - - - - {#each refinedStateData as sd, i} - - - - - - {/each} -
Current StateNodes
{sd.state}{sd.count}
- {/key} - - - - - - - - Infos - - - Contents - - - - -
- {#key $statusQuery?.data?.nodeMetrics} - - {/key} -
- +
+ + +
+ {#key $statusQuery?.data?.nodeMetrics} + + {/key} +
+ + + + +
+ {#key refinedStateData} + + sd.count, + )} + entities={refinedStateData.map( + (sd) => sd.state, + )} + fixColors={refinedStateData.map( + (sd) => colors['nodeStates'][sd.state], + )} + /> + {/key} +
+ + + {#key refinedStateData} + + + + + + + {#each refinedStateData as sd, i} + + + + + + {/each} +
StateCount
{sd.state.charAt(0).toUpperCase() + sd.state.slice(1)}{sd.count}
+ {/key} + + + +
{#key $statesTimed?.data?.nodeStates} @@ -659,7 +670,7 @@ xlabel="Time" ylabel="Nodes" yunit = "#Count" - title = "Node States" + title = "Cluster Status" stateType = "Node" /> {/key} diff --git a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte index 94acf45a..9579f36b 100644 --- a/web/frontend/src/generic/plots/DoubleMetricPlot.svelte +++ b/web/frontend/src/generic/plots/DoubleMetricPlot.svelte @@ -33,6 +33,7 @@ // metric, width = 0, height = 300, + fixLinewidth = null, timestep, numNodes, metricData, @@ -52,7 +53,7 @@ // const subClusterTopology = getContext("getHardwareTopology")(cluster, subCluster); // const metricConfig = getContext("getMetricConfig")(cluster, subCluster, metric); const lineColors = clusterCockpitConfig.plotConfiguration_colorScheme; - const lineWidth = clusterCockpitConfig.plotConfiguration_lineWidth / window.devicePixelRatio; + const lineWidth = fixLinewidth ? fixLinewidth : clusterCockpitConfig.plotConfiguration_lineWidth / window.devicePixelRatio; // const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false; const renderSleepTime = 200; // const normalLineColor = "#000000"; @@ -444,6 +445,7 @@ const opts = { width, height, + title: 'Cluster Utilization', plugins: [legendAsTooltipPlugin()], series: plotSeries, axes: [ diff --git a/web/frontend/src/generic/plots/Pie.svelte b/web/frontend/src/generic/plots/Pie.svelte index 45be1f59..5e31b7dd 100644 --- a/web/frontend/src/generic/plots/Pie.svelte +++ b/web/frontend/src/generic/plots/Pie.svelte @@ -59,7 +59,15 @@ 'rgb(135,133,0)', 'rgb(0,167,108)', 'rgb(189,189,189)', - ] + ], + nodeStates: { + allocated: "rgba(0, 128, 0, 0.75)", + down: "rgba(255, 0, 0, 0.75)", + idle: "rgba(0, 0, 255, 0.75)", + reserved: "rgba(255, 0, 255, 0.75)", + mixed: "rgba(255, 215, 0, 0.75)", + unknown: "rgba(0, 0, 0, 0.75)" + } } @@ -77,6 +85,7 @@ entities, displayLegend = false, useAltColors = false, + fixColors = null } = $props(); /* Const Init */ @@ -98,6 +107,8 @@ c = [...colors['colorblind']]; } else if (useAltColors) { c = [...colors['alternative']]; + } else if (fixColors?.length > 0) { + c = [...fixColors]; } else { c = [...colors['default']]; } diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index 6425275a..c28ab1fa 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -34,15 +34,18 @@ nodesData = null, cluster = null, subCluster = null, + fixTitle = null, + yMinimum = null, allowSizeChange = false, useColors = true, useLegend = true, + colorBackground = false, width = 600, height = 380, } = $props(); /* Const Init */ - const lineWidth = clusterCockpitConfig.plotConfiguration_lineWidth; + const lineWidth = 2 // clusterCockpitConfig.plotConfiguration_lineWidth; const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false; /* Var Init */ @@ -294,7 +297,7 @@ } else { // No Colors: Use Black u.ctx.strokeStyle = "rgb(0, 0, 0)"; - u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)"; + u.ctx.fillStyle = colorBackground ? "rgb(0, 0, 0)" : "rgba(0, 0, 0, 0.5)"; } // Get Values @@ -527,6 +530,7 @@ let plotTitle = "CPU Roofline Diagram"; if (jobsData) plotTitle = "Job Average Roofline Diagram"; if (nodesData) plotTitle = "Node Average Roofline Diagram"; + if (fixTitle) plotTitle = fixTitle if (roofData) { const opts = { @@ -617,7 +621,7 @@ }, y: { range: [ - 0.01, + yMinimum ? yMinimum : 0.01, subCluster?.flopRateSimd?.value ? nearestThousand(subCluster.flopRateSimd.value) : 10000, @@ -669,6 +673,7 @@ u.ctx.lineWidth = lineWidth; u.ctx.beginPath(); + // Get Values const ycut = 0.01 * subCluster.memoryBandwidth.value; const scalarKnee = (subCluster.flopRateScalar.value - ycut) / @@ -676,19 +681,27 @@ const simdKnee = (subCluster.flopRateSimd.value - ycut) / subCluster.memoryBandwidth.value; - const scalarKneeX = u.valToPos(scalarKnee, "x", true), // Value, axis, toCanvasPixels - simdKneeX = u.valToPos(simdKnee, "x", true), - flopRateScalarY = u.valToPos( - subCluster.flopRateScalar.value, - "y", - true, - ), - flopRateSimdY = u.valToPos( - subCluster.flopRateSimd.value, - "y", - true, - ); + // Get Const Coords + const originX = u.valToPos(0.01, "x", true); + const originY = u.valToPos(yMinimum ? yMinimum : 0.01, "y", true); + + const outerX = u.valToPos(1000, "x", true); // rightmost x in plot coords + const outerY = u.valToPos( + subCluster?.flopRateSimd?.value + ? nearestThousand(subCluster.flopRateSimd.value) + : 10000, + "y", + true + ); + + const scalarKneeX = u.valToPos(scalarKnee, "x", true) // Value, axis, toCanvasPixels + const simdKneeX = u.valToPos(simdKnee, "x", true) + + const flopRateScalarY = u.valToPos(subCluster.flopRateScalar.value, "y", true) + const flopRateSimdY = u.valToPos(subCluster.flopRateSimd.value, "y", true); + + /* Render Lines */ if ( scalarKneeX < width * window.devicePixelRatio - @@ -728,10 +741,10 @@ y1, x2, y2, - u.valToPos(0.01, "x", true), - u.valToPos(1.0, "y", true), // X-Axis Start Coords - u.valToPos(1000, "x", true), - u.valToPos(1.0, "y", true), // X-Axis End Coords + originX, // x3; X-Axis Start Coord-X + originY, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + originY, // y4; X-Axis End Coord-Y ); if (xAxisIntersect.x > x1) { @@ -746,6 +759,144 @@ u.ctx.stroke(); // Reset grid lineWidth u.ctx.lineWidth = 0.15; + + /* Render Area */ + if (colorBackground) { + + u.ctx.beginPath(); + // Additional Coords for Colored Regions + const yhalf = u.valToPos(ycut/2, "y", true) + const simdShift = u.valToPos(simdKnee*1.75, "x", true) + + let upperBorderIntersect = lineIntersect( + x1, + y1, + x2, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateSimdY*1.667, // y4; X-Axis End Coord-Y + ); + + let lowerBorderIntersect = lineIntersect( + x1, + y1, + x2, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y + ); + + let helperUpperBorderIntersect = lineIntersect( + x1, + yhalf, + simdShift, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateSimdY*1.667, // y4; X-Axis End Coord-Y + ); + + let helperLowerBorderIntersect = lineIntersect( + x1, + yhalf, + simdShift, + y2, + originX, // x3; X-Axis Start Coord-X + flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y + ); + + let helperLowerBorderIntersectTop = lineIntersect( + x1, + yhalf, + simdShift, + y2, + scalarKneeX, // x3; X-Axis Start Coord-X + flopRateScalarY, // y3; X-Axis Start Coord-Y + outerX, // x4; X-Axis End Coord-X + flopRateScalarY, // y4; X-Axis End Coord-Y + ); + + // Diagonal Helper + u.ctx.moveTo(x1, yhalf); + u.ctx.lineTo(simdShift, y2); + // Upper Simd Helper + u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667); + u.ctx.lineTo(outerX, flopRateSimdY*1.667); + // Lower Scalar Helper + u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667); + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); + + u.ctx.stroke(); + + /* Color Regions */ + // MemoryBound + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.lineTo(x1, y1); // YCut + u.ctx.lineTo(x2, y2); // Upper Knee + u.ctx.lineTo(simdShift, y2); // Upper Helper Knee + u.ctx.lineTo(x1, yhalf); // Half yCut + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 200, 0, 0.4)"; // Yellow + u.ctx.fill(); + u.ctx.restore(); + + // Compute Lower + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Helper Knee + u.ctx.lineTo(scalarKneeX, flopRateScalarY); // Lower Knee + u.ctx.lineTo(outerX, flopRateScalarY); // Outer Border + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); // Outer Lower Helper Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue + u.ctx.fill(); + u.ctx.restore(); + + // Compute Upper + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee + u.ctx.lineTo(simdKneeX, flopRateSimdY); // Upper Knee + u.ctx.lineTo(outerX, flopRateSimdY); // Outer Border + u.ctx.lineTo(outerX, flopRateSimdY*1.667); // Outer Upper Helper Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue + u.ctx.fill(); + u.ctx.restore(); + + // Nomansland Lower + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(originX, originY); // Origin + u.ctx.lineTo(originX, yhalf); // YCut Half + u.ctx.lineTo(helperLowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Inner Helper Knee + u.ctx.lineTo(outerX, flopRateScalarY*1.1667); // Lower Inner Border + u.ctx.lineTo(outerX, originY); // Lower Right Corner + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light + u.ctx.fill(); + u.ctx.restore(); + + // Nomansland Upper + u.ctx.save(); + u.ctx.beginPath(); + u.ctx.moveTo(helperLowerBorderIntersectTop.x, flopRateScalarY); // Lower Knee Top + u.ctx.lineTo(helperUpperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee + u.ctx.lineTo(outerX, flopRateSimdY*1.667); // Upper Inner Border + u.ctx.lineTo(outerX, flopRateScalarY); // Lower Knee Border + u.ctx.closePath(); + u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light + u.ctx.fill(); + u.ctx.restore(); + } } /* Render Scales */ diff --git a/web/frontend/src/generic/plots/RooflineLegacy.svelte b/web/frontend/src/generic/plots/RooflineLegacy.svelte index 624253bf..6ee96ef1 100644 --- a/web/frontend/src/generic/plots/RooflineLegacy.svelte +++ b/web/frontend/src/generic/plots/RooflineLegacy.svelte @@ -315,10 +315,10 @@ y1, x2, y2, - u.valToPos(0.01, "x", true), - u.valToPos(1.0, "y", true), // X-Axis Start Coords - u.valToPos(1000, "x", true), - u.valToPos(1.0, "y", true), // X-Axis End Coords + u.valToPos(0.01, "x", true), // x3; X-Axis Start Coord-X + u.valToPos(1.0, "y", true), // y3; X-Axis Start Coord-Y + u.valToPos(1000, "x", true), // x4; X-Axis End Coord-X + u.valToPos(1.0, "y", true), // y4; X-Axis End Coord-Y ); if (xAxisIntersect.x > x1) { diff --git a/web/frontend/src/generic/plots/Stacked.svelte b/web/frontend/src/generic/plots/Stacked.svelte index 2616f9ae..c9ec1d72 100644 --- a/web/frontend/src/generic/plots/Stacked.svelte +++ b/web/frontend/src/generic/plots/Stacked.svelte @@ -39,63 +39,63 @@ label: "Full", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)", + fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)", stroke: cbmode ? "rgb(0, 110, 0)" : "green", }, partial: { label: "Partial", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)", + fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)", stroke: cbmode ? "rgb(235, 172, 35)" : "gold", }, failed: { label: "Failed", scale: "y", width: lineWidth, - fill: cbmode ? "rgb(181, 29, 20, 0.4)" : "rgba(255, 0, 0, 0.4)", + fill: cbmode ? "rgb(181, 29, 20, 0.6)" : "rgba(255, 0, 0, 0.6)", stroke: cbmode ? "rgb(181, 29, 20)" : "red", }, idle: { label: "Idle", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 140, 249, 0.4)" : "rgba(0, 0, 255, 0.4)", + fill: cbmode ? "rgba(0, 140, 249, 0.6)" : "rgba(0, 0, 255, 0.6)", stroke: cbmode ? "rgb(0, 140, 249)" : "blue", }, allocated: { label: "Allocated", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)", + fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)", stroke: cbmode ? "rgb(0, 110, 0)" : "green", }, reserved: { label: "Reserved", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(209, 99, 230, 0.4)" : "rgba(255, 0, 255, 0.4)", + fill: cbmode ? "rgba(209, 99, 230, 0.6)" : "rgba(255, 0, 255, 0.6)", stroke: cbmode ? "rgb(209, 99, 230)" : "magenta", }, mixed: { label: "Mixed", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)", + fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)", stroke: cbmode ? "rgb(235, 172, 35)" : "gold", }, down: { label: "Down", scale: "y", width: lineWidth, - fill: cbmode ? "rgba(181, 29 ,20, 0.4)" : "rgba(255, 0, 0, 0.4)", + fill: cbmode ? "rgba(181, 29 ,20, 0.6)" : "rgba(255, 0, 0, 0.6)", stroke: cbmode ? "rgb(181, 29, 20)" : "red", }, unknown: { label: "Unknown", scale: "y", width: lineWidth, - fill: "rgba(0, 0, 0, 0.4)", + fill: "rgba(0, 0, 0, 0.6)", stroke: "black", } }; From 0d62181272e3d406b2543aab2d8fd765d2245594 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 12 Dec 2025 11:19:37 +0100 Subject: [PATCH 03/30] move roofline elements below series data render --- web/frontend/src/generic/plots/Roofline.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index c28ab1fa..3b53a662 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -651,7 +651,7 @@ hooks: { // setSeries: [ (u, seriesIdx) => console.log('setSeries', seriesIdx) ], // setLegend: [ u => console.log('setLegend', u.legend.idxs) ], - drawClear: [ + drawClear: [ // drawClear hook which fires before anything exists, so will render under the grid (u) => { qt = qt || new Quadtree(0, 0, u.bbox.width, u.bbox.height); qt.clear(); @@ -663,7 +663,7 @@ }); }, ], - draw: [ + drawAxes: [ // drawAxes hook, which fires after axes and grid have been rendered (u) => { // draw roofs when subCluster set if (subCluster != null) { From 79e1c236fe576b98ae8495ea897aa9a6af63f889 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 12 Dec 2025 17:51:54 +0100 Subject: [PATCH 04/30] cleanup, adapt internalDash, remove debug query value --- web/frontend/src/DashPublic.root.svelte | 120 ++---------------- .../src/generic/plots/Roofline.svelte | 7 - web/frontend/src/status/DashInternal.svelte | 79 ++++++------ .../src/status/dashdetails/StatusDash.svelte | 2 +- 4 files changed, 54 insertions(+), 154 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 2e060126..055e4899 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -16,19 +16,14 @@ } from "@urql/svelte"; import { init, - scramble, - scrambleNames, - convert2uplot } from "./generic/utils.js"; import { - formatDurationTime, formatNumber, } from "./generic/units.js"; import { Row, Col, Card, - CardTitle, CardHeader, CardBody, Spinner, @@ -39,9 +34,10 @@ import Roofline from "./generic/plots/Roofline.svelte"; import Pie, { colors } from "./generic/plots/Pie.svelte"; import Stacked from "./generic/plots/Stacked.svelte"; - // import Histogram from "./generic/plots/Histogram.svelte"; import DoubleMetric from "./generic/plots/DoubleMetricPlot.svelte"; + // Todo: Refresher-Tick + /* Svelte 5 Props */ let { presetCluster, @@ -53,7 +49,6 @@ const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false /* States */ - let pagingState = $state({page: 1, itemsPerPage: 10}) // Top 10 let from = $state(new Date(Date.now() - (5 * 60 * 1000))); let clusterFrom = $state(new Date(Date.now() - (8 * 60 * 60 * 1000))); let to = $state(new Date(Date.now())); @@ -68,13 +63,8 @@ const statesTimed = $derived(queryStore({ client: client, query: gql` - query ($filter: [NodeFilter!], $typeNode: String!, $typeHealth: String!) { - nodeStates: nodeStatesTimed(filter: $filter, type: $typeNode) { - state - counts - times - } - healthStates: nodeStatesTimed(filter: $filter, type: $typeHealth) { + query ($filter: [NodeFilter!], $type: String!) { + nodeStatesTimed(filter: $filter, type: $type) { state counts times @@ -82,9 +72,8 @@ } `, variables: { - filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, // DEBUG VALUE 1760096999, use StackedFrom - typeNode: "node", - typeHealth: "health" + filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom}, + type: "node", }, requestPolicy: "network-only" })); @@ -97,7 +86,6 @@ query ( $cluster: String! $metrics: [String!] - # $nmetrics: [String!] $from: Time! $to: Time! $clusterFrom: Time! @@ -207,7 +195,6 @@ variables: { cluster: presetCluster, metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline - // nmetrics: ["cpu_load", "acc_utilization"], // Metrics for Node Graph from: from.toISOString(), clusterFrom: clusterFrom.toISOString(), to: to.toISOString(), @@ -219,31 +206,6 @@ requestPolicy: "network-only" })); - const topJobsQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $paging: PageRequest! - ) { - jobsStatistics( - filter: $filter - page: $paging - sortBy: TOTALJOBS - groupBy: PROJECT - ) { - id - totalJobs - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster} }], - paging: pagingState // Top 10 - }, - requestPolicy: "network-only" - })); - // Note: nodeMetrics are requested on configured $timestep resolution const nodeStatusQuery = $derived(queryStore({ client: client, @@ -351,19 +313,6 @@ }); /* Functions */ - // function legendColors(targetIdx, useAltColors) { - // // Reuses first color if targetIdx overflows - // let c; - // if (useCbColors) { - // c = [...colors['colorblind']]; - // } else if (useAltColors) { - // c = [...colors['alternative']]; - // } else { - // c = [...colors['default']]; - // } - // return c[(c.length + targetIdx) % c.length]; - // } - function transformNodesStatsToData(subclusterData) { let data = null const x = [], y = [] @@ -415,30 +364,18 @@ return result } - /* Inspect */ - $inspect(clusterInfo).with((type, clusterInfo) => { - console.log(type, 'clusterInfo', clusterInfo) - }); - - $inspect($statusQuery?.data?.clusterMetrics).with((type, clusterMetrics) => { - console.log(type, 'clusterMetrics', clusterMetrics) - }); - - - {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + {#if $statusQuery.fetching || $statesTimed.fetching || $nodeStatusQuery.fetching} - {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} + {:else if $statusQuery.error || $statesTimed.error || $nodeStatusQuery.error} {#if $statusQuery.error} @@ -450,11 +387,6 @@ Error Requesting StatesTimed: {$statesTimed.error.message} {/if} - {#if $topJobsQuery.error} - - Error Requesting TopJobsQuery: {$topJobsQuery.error.message} - - {/if} {#if $nodeStatusQuery.error} Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} @@ -477,10 +409,6 @@ - @@ -559,7 +487,7 @@ - +
- @@ -620,9 +525,6 @@
{#key refinedStateData} -
- {#key $statesTimed?.data?.nodeStates} + {#key $statesTimed?.data?.nodeStatesTimed} { - console.log(type, 'clusterInfo', clusterInfo) - }); - - - -

{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)} Dashboard

- - + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching}
@@ -409,7 +422,7 @@ Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}" - {[...clusterInfo?.processorTypes].toString()} + {[...clusterInfo?.processorTypes].join(', ')}
@@ -488,6 +501,7 @@ + @@ -529,6 +543,7 @@ +
{#key $statusQuery?.data?.jobsMetricStats} @@ -544,31 +559,20 @@ {/key}
- - {#if clusterInfo?.totalAccs == 0} - +
+ - {:else} - - {/if} +
+
{#key $statesTimed?.data?.nodeStates} @@ -584,6 +588,7 @@ {/key}
+
{#key $statesTimed?.data?.healthStates} diff --git a/web/frontend/src/status/dashdetails/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte index ee604836..f1f5a1a1 100644 --- a/web/frontend/src/status/dashdetails/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -83,7 +83,7 @@ } `, variables: { - filter: { cluster: { eq: cluster }, timeStart: 1760096999}, + filter: { cluster: { eq: cluster }, timeStart: stackedFrom}, typeNode: "node", typeHealth: "health" }, From c5aff1a2ca39fc8363be8c9d56070278700ea45e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 15 Dec 2025 13:55:02 +0100 Subject: [PATCH 05/30] add autorefresh, remove leftover query --- web/frontend/src/DashPublic.root.svelte | 61 ++++++------------ .../src/generic/helper/Refresher.svelte | 43 +++++++------ web/frontend/src/status/DashInternal.svelte | 64 +++++++------------ .../src/status/dashdetails/StatusDash.svelte | 2 +- 4 files changed, 69 insertions(+), 101 deletions(-) diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index 055e4899..d3eb57bc 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -35,8 +35,7 @@ import Pie, { colors } from "./generic/plots/Pie.svelte"; import Stacked from "./generic/plots/Stacked.svelte"; import DoubleMetric from "./generic/plots/DoubleMetricPlot.svelte"; - - // Todo: Refresher-Tick + import Refresher from "./generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { @@ -206,35 +205,6 @@ requestPolicy: "network-only" })); - // Note: nodeMetrics are requested on configured $timestep resolution - const nodeStatusQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $selectedHistograms: [String!] - $numDurationBins: String - ) { - jobsStatistics(filter: $filter, metrics: $selectedHistograms, numDurationBins: $numDurationBins) { - histNumCores { - count - value - } - histNumAccs { - count - value - } - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }], - selectedHistograms: [], // No Metrics requested for node hardware stats - Empty Array can be used for refresh - numDurationBins: "1h", // Hardcode or selector? - }, - requestPolicy: "network-only" - })); - const clusterInfo = $derived.by(() => { if ($initq?.data?.clusters) { let rawInfos = {}; @@ -368,28 +338,39 @@ - {#if $statusQuery.fetching || $statesTimed.fetching || $nodeStatusQuery.fetching} + +
+ { + from = new Date(Date.now() - 5 * 60 * 1000); + to = new Date(Date.now()); + clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000)) + + if (interval) stackedFrom += Math.floor(interval / 1000); + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh + }} + /> + + + {#if $statusQuery.fetching || $statesTimed.fetching} - {:else if $statusQuery.error || $statesTimed.error || $nodeStatusQuery.error} + {:else if $statusQuery.error || $statesTimed.error} {#if $statusQuery.error} - Error Requesting StatusQuery: {$statusQuery.error.message} + Error Requesting Status Data: {$statusQuery.error.message} {/if} {#if $statesTimed.error} - Error Requesting StatesTimed: {$statesTimed.error.message} - - {/if} - {#if $nodeStatusQuery.error} - - Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + Error Requesting Node Scheduler States: {$statesTimed.error.message} {/if} diff --git a/web/frontend/src/generic/helper/Refresher.svelte b/web/frontend/src/generic/helper/Refresher.svelte index 7f568bfb..ca05bf6d 100644 --- a/web/frontend/src/generic/helper/Refresher.svelte +++ b/web/frontend/src/generic/helper/Refresher.svelte @@ -14,6 +14,7 @@ let { initially = null, presetClass = "", + hideSelector = false, onRefresh } = $props(); @@ -36,25 +37,27 @@ }); - - - - - - - - - - + + + + + + + + +{/if} diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte index 495f9983..c6abf068 100644 --- a/web/frontend/src/status/DashInternal.svelte +++ b/web/frontend/src/status/DashInternal.svelte @@ -39,6 +39,7 @@ import Pie, { colors } from "../generic/plots/Pie.svelte"; import Stacked from "../generic/plots/Stacked.svelte"; import DoubleMetric from "../generic/plots/DoubleMetricPlot.svelte"; + import Refresher from "../generic/helper/Refresher.svelte"; /* Svelte 5 Props */ let { @@ -224,35 +225,6 @@ requestPolicy: "network-only" })); - // Note: nodeMetrics are requested on configured $timestep resolution - const nodeStatusQuery = $derived(queryStore({ - client: client, - query: gql` - query ( - $filter: [JobFilter!]! - $selectedHistograms: [String!] - $numDurationBins: String - ) { - jobsStatistics(filter: $filter, metrics: $selectedHistograms, numDurationBins: $numDurationBins) { - histNumCores { - count - value - } - histNumAccs { - count - value - } - } - } - `, - variables: { - filter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }], - selectedHistograms: [], // No Metrics requested for node hardware stats - Empty Array can be used for refresh - numDurationBins: "1h", // Hardcode or selector? - }, - requestPolicy: "network-only" - })); - const clusterInfo = $derived.by(() => { if ($initq?.data?.clusters) { let rawInfos = {}; @@ -385,33 +357,45 @@ - {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching || $nodeStatusQuery.fetching} + + + { + from = new Date(Date.now() - 5 * 60 * 1000); + to = new Date(Date.now()); + clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000)) + pagingState = { page:1, itemsPerPage: 10 }; + + if (interval) stackedFrom += Math.floor(interval / 1000); + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh + }} + /> + + + {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching} - {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error || $nodeStatusQuery.error} - + {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error} + {#if $statusQuery.error} - Error Requesting StatusQuery: {$statusQuery.error.message} + Error Requesting Status Data: {$statusQuery.error.message} {/if} {#if $statesTimed.error} - Error Requesting StatesTimed: {$statesTimed.error.message} + Error Requesting Node Scheduler States: {$statesTimed.error.message} {/if} {#if $topJobsQuery.error} - Error Requesting TopJobsQuery: {$topJobsQuery.error.message} - - {/if} - {#if $nodeStatusQuery.error} - - Error Requesting NodeStatusQuery: {$nodeStatusQuery.error.message} + Error Requesting Jobs By Project: {$topJobsQuery.error.message} {/if} diff --git a/web/frontend/src/status/dashdetails/StatusDash.svelte b/web/frontend/src/status/dashdetails/StatusDash.svelte index f1f5a1a1..f858969f 100644 --- a/web/frontend/src/status/dashdetails/StatusDash.svelte +++ b/web/frontend/src/status/dashdetails/StatusDash.svelte @@ -402,7 +402,7 @@ to = new Date(Date.now()); if (interval) stackedFrom += Math.floor(interval / 1000); - else stackedFrom += 1 // Workaround: TineSelection not linked, just trigger new data on manual refresh + else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh }} /> From d56b0e93db21d2cff011f2f5afcf43c1cbd3e840 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 15 Dec 2025 15:10:10 +0100 Subject: [PATCH 06/30] cleanup routes, cleanup root components --- internal/routerConfig/routes.go | 17 +------------ web/frontend/src/DashPublic.root.svelte | 8 +++--- web/frontend/src/Status.root.svelte | 31 +++++++++-------------- web/frontend/src/dashpublic.entrypoint.js | 2 +- web/templates/base.tmpl | 4 +-- web/templates/monitoring/dashboard.tmpl | 2 +- 6 files changed, 21 insertions(+), 43 deletions(-) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 71edeefb..c2126cd0 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -120,11 +120,6 @@ func setupClusterStatusRoute(i InfoType, r *http.Request) InfoType { i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] i["displayType"] = "DASHBOARD" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } return i } @@ -133,11 +128,6 @@ func setupClusterDetailRoute(i InfoType, r *http.Request) InfoType { i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] i["displayType"] = "DETAILS" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } return i } @@ -145,12 +135,7 @@ func setupDashboardRoute(i InfoType, r *http.Request) InfoType { vars := mux.Vars(r) i["id"] = vars["cluster"] i["cluster"] = vars["cluster"] - i["displayType"] = "PUBLIC" - from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to") - if from != "" || to != "" { - i["from"] = from - i["to"] = to - } + i["displayType"] = "PUBLIC" // Used in Main Template return i } diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index d3eb57bc..dac3f9aa 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -6,9 +6,9 @@ --> - - -{#if displayType !== "DASHBOARD" && displayType !== "DETAILS"} +{#if displayType === 'DETAILS'} + +{:else if displayType === 'DASHBOARD'} + +{:else} - Unknown displayList type! + + + Unknown DisplayType for Status View! + + -{:else} - {#if displayStatusDetail} - - - {:else} - - - {/if} {/if} diff --git a/web/frontend/src/dashpublic.entrypoint.js b/web/frontend/src/dashpublic.entrypoint.js index 47287c7a..b9e92ffb 100644 --- a/web/frontend/src/dashpublic.entrypoint.js +++ b/web/frontend/src/dashpublic.entrypoint.js @@ -5,7 +5,7 @@ import DashPublic from './DashPublic.root.svelte' mount(DashPublic, { target: document.getElementById('svelte-app'), props: { - presetCluster: infos.cluster, + presetCluster: presetCluster, }, context: new Map([ ['cc-config', clusterCockpitConfig] diff --git a/web/templates/base.tmpl b/web/templates/base.tmpl index 28eab339..a1bd4134 100644 --- a/web/templates/base.tmpl +++ b/web/templates/base.tmpl @@ -27,13 +27,13 @@
{{block "content-public" .}} - Whoops, you should not see this... [MAIN] + Whoops, you should not see this... [PUBLIC] {{end}}
{{block "javascript-public" .}} - Whoops, you should not see this... [JS] + Whoops, you should not see this... [JS PUBLIC] {{end}} {{else}} diff --git a/web/templates/monitoring/dashboard.tmpl b/web/templates/monitoring/dashboard.tmpl index 06666cde..1ef94558 100644 --- a/web/templates/monitoring/dashboard.tmpl +++ b/web/templates/monitoring/dashboard.tmpl @@ -7,7 +7,7 @@ {{end}} {{define "javascript-public"}} From 46351389b6c39186d9a71cd254ff55e1d2700c5a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 15 Dec 2025 21:25:00 +0100 Subject: [PATCH 07/30] Add ai agent guidelines --- AGENTS.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..847bc094 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# ClusterCockpit Backend - Agent Guidelines + +## Build/Test Commands + +- Build: `make` or `go build ./cmd/cc-backend` +- Run all tests: `make test` (runs: `go clean -testcache && go build ./... && go vet ./... && go test ./...`) +- Run single test: `go test -run TestName ./path/to/package` +- Run single test file: `go test ./path/to/package -run TestName` +- Frontend build: `cd web/frontend && npm install && npm run build` +- Generate GraphQL: `make graphql` (uses gqlgen) +- Generate Swagger: `make swagger` (uses swaggo/swag) + +## Code Style + +- **Formatting**: Use `gofumpt` for all Go files (strict requirement) +- **Copyright header**: All files must include copyright header (see existing files) +- **Package docs**: Document packages with comprehensive package-level comments explaining purpose, usage, configuration +- **Imports**: Standard library first, then external packages, then internal packages (grouped with blank lines) +- **Naming**: Use camelCase for private, PascalCase for exported; descriptive names (e.g., `JobRepository`, `handleError`) +- **Error handling**: Return errors, don't panic; use custom error types where appropriate; log with cclog package +- **Logging**: Use `cclog` package (e.g., `cclog.Errorf()`, `cclog.Warnf()`, `cclog.Debugf()`) +- **Testing**: Use standard `testing` package; use `testify/assert` for assertions; name tests `TestFunctionName` +- **Comments**: Document all exported functions/types with godoc-style comments +- **Structs**: Document fields with inline comments, especially for complex configurations +- **HTTP handlers**: Return proper status codes; use `handleError()` helper for consistent error responses +- **JSON**: Use struct tags for JSON marshaling; `DisallowUnknownFields()` for strict decoding From 33c38f94646e94abb364823bd2280892af8c164f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 15 Dec 2025 21:25:30 +0100 Subject: [PATCH 08/30] Fix start time in tasks --- internal/taskmanager/compressionService.go | 2 +- internal/taskmanager/retentionService.go | 4 ++-- internal/taskmanager/stopJobsExceedTime.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/taskmanager/compressionService.go b/internal/taskmanager/compressionService.go index c2df852d..1da2f68d 100644 --- a/internal/taskmanager/compressionService.go +++ b/internal/taskmanager/compressionService.go @@ -17,7 +17,7 @@ import ( func RegisterCompressionService(compressOlderThan int) { cclog.Info("Register compression service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o5, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(5, 0, 0))), gocron.NewTask( func() { var jobs []*schema.Job diff --git a/internal/taskmanager/retentionService.go b/internal/taskmanager/retentionService.go index e452ffd0..92cd36ab 100644 --- a/internal/taskmanager/retentionService.go +++ b/internal/taskmanager/retentionService.go @@ -16,7 +16,7 @@ import ( func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { cclog.Info("Register retention delete service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(14, 30, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) @@ -43,7 +43,7 @@ func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { func RegisterRetentionMoveService(age int, includeDB bool, location string, omitTagged bool) { cclog.Info("Register retention move service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(4, 0, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) diff --git a/internal/taskmanager/stopJobsExceedTime.go b/internal/taskmanager/stopJobsExceedTime.go index e59b3aee..b763f561 100644 --- a/internal/taskmanager/stopJobsExceedTime.go +++ b/internal/taskmanager/stopJobsExceedTime.go @@ -16,7 +16,7 @@ import ( func RegisterStopJobsExceedTime() { cclog.Info("Register undead jobs service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o3, 0, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))), gocron.NewTask( func() { err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) From 6f49998ad38332e1d671cae96f4fbab61edeb0a9 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:49:17 +0100 Subject: [PATCH 09/30] Switch to new go tool pattern for external tool deps --- Makefile | 4 ++-- go.mod | 5 +++++ tools.go | 9 --------- 3 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 tools.go diff --git a/Makefile b/Makefile index 0e19095a..0378b700 100644 --- a/Makefile +++ b/Makefile @@ -50,12 +50,12 @@ frontend: swagger: $(info ===> GENERATE swagger) - @go run github.com/swaggo/swag/cmd/swag init --parseDependency -d ./internal/api -g rest.go -o ./api + @go tool github.com/swaggo/swag/cmd/swag init --parseDependency -d ./internal/api -g rest.go -o ./api @mv ./api/docs.go ./internal/api/docs.go graphql: $(info ===> GENERATE graphql) - @go run github.com/99designs/gqlgen + @go tool github.com/99designs/gqlgen clean: $(info ===> CLEAN) diff --git a/go.mod b/go.mod index 3b3583bd..75e62f1e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,11 @@ go 1.24.0 toolchain go1.24.1 +tool ( + github.com/99designs/gqlgen + github.com/swaggo/swag/cmd/swag +) + require ( github.com/99designs/gqlgen v0.17.84 github.com/ClusterCockpit/cc-lib v1.0.0 diff --git a/tools.go b/tools.go deleted file mode 100644 index 950056c4..00000000 --- a/tools.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build tools -// +build tools - -package tools - -import ( - _ "github.com/99designs/gqlgen" - _ "github.com/swaggo/swag/cmd/swag" -) From 0306723307a2ddb25ae27284e4143320de393858 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:55:31 +0100 Subject: [PATCH 10/30] Introduce transparent compression for importJob function in all archive backends --- pkg/archive/fsBackend.go | 53 +++++++++++++++++++++++++++++------- pkg/archive/s3Backend.go | 26 +++++++++++++++--- pkg/archive/sqliteBackend.go | 25 +++++++++++++++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 1e9d7db3..22c1772a 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -603,19 +603,52 @@ func (fsa *FsArchive) ImportJob( return err } - f, err = os.Create(path.Join(dir, "data.json")) - if err != nil { - cclog.Error("Error while creating filepath for data.json") + var dataBuf bytes.Buffer + if err := EncodeJobData(&dataBuf, jobData); err != nil { + cclog.Error("Error while encoding job metricdata") return err } - if err := EncodeJobData(f, jobData); err != nil { - cclog.Error("Error while encoding job metricdata to data.json file") - return err + + if dataBuf.Len() > 2000 { + f, err = os.Create(path.Join(dir, "data.json.gz")) + if err != nil { + cclog.Error("Error while creating filepath for data.json.gz") + return err + } + gzipWriter := gzip.NewWriter(f) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Error("Error while writing compressed job data") + gzipWriter.Close() + f.Close() + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Warn("Error while closing gzip writer") + f.Close() + return err + } + if err := f.Close(); err != nil { + cclog.Warn("Error while closing data.json.gz file") + return err + } + } else { + f, err = os.Create(path.Join(dir, "data.json")) + if err != nil { + cclog.Error("Error while creating filepath for data.json") + return err + } + if _, err := f.Write(dataBuf.Bytes()); err != nil { + cclog.Error("Error while writing job metricdata to data.json file") + f.Close() + return err + } + if err := f.Close(); err != nil { + cclog.Warn("Error while closing data.json file") + return err + } } - if err := f.Close(); err != nil { - cclog.Warn("Error while closing data.json file") - } - return err + + return nil } func (fsa *FsArchive) StoreClusterCfg(name string, config *schema.Cluster) error { diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index 5b3d9f02..eacdde76 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -467,7 +467,6 @@ func (s3a *S3Archive) StoreJobMeta(job *schema.Job) error { func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) error { ctx := context.Background() - // Upload meta.json metaKey := getS3Key(jobMeta, "meta.json") var metaBuf bytes.Buffer if err := EncodeJobMeta(&metaBuf, jobMeta); err != nil { @@ -485,18 +484,37 @@ func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) er return err } - // Upload data.json - dataKey := getS3Key(jobMeta, "data.json") var dataBuf bytes.Buffer if err := EncodeJobData(&dataBuf, jobData); err != nil { cclog.Error("S3Archive ImportJob() > encoding data error") return err } + var dataKey string + var dataBytes []byte + + if dataBuf.Len() > 2000 { + dataKey = getS3Key(jobMeta, "data.json.gz") + var compressedBuf bytes.Buffer + gzipWriter := gzip.NewWriter(&compressedBuf) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Errorf("S3Archive ImportJob() > gzip write error: %v", err) + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Errorf("S3Archive ImportJob() > gzip close error: %v", err) + return err + } + dataBytes = compressedBuf.Bytes() + } else { + dataKey = getS3Key(jobMeta, "data.json") + dataBytes = dataBuf.Bytes() + } + _, err = s3a.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s3a.bucket), Key: aws.String(dataKey), - Body: bytes.NewReader(dataBuf.Bytes()), + Body: bytes.NewReader(dataBytes), }) if err != nil { cclog.Errorf("S3Archive ImportJob() > PutObject data error: %v", err) diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 49aeb79d..589beea4 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -361,16 +361,37 @@ func (sa *SqliteArchive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) return err } + var dataBytes []byte + var compressed bool + + if dataBuf.Len() > 2000 { + var compressedBuf bytes.Buffer + gzipWriter := gzip.NewWriter(&compressedBuf) + if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil { + cclog.Errorf("SqliteArchive ImportJob() > gzip write error: %v", err) + return err + } + if err := gzipWriter.Close(); err != nil { + cclog.Errorf("SqliteArchive ImportJob() > gzip close error: %v", err) + return err + } + dataBytes = compressedBuf.Bytes() + compressed = true + } else { + dataBytes = dataBuf.Bytes() + compressed = false + } + now := time.Now().Unix() _, err := sa.db.Exec(` INSERT INTO jobs (job_id, cluster, start_time, meta_json, data_json, data_compressed, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, 0, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_id, cluster, start_time) DO UPDATE SET meta_json = excluded.meta_json, data_json = excluded.data_json, data_compressed = excluded.data_compressed, updated_at = excluded.updated_at - `, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBuf.Bytes(), now, now) + `, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBytes, compressed, now, now) if err != nil { cclog.Errorf("SqliteArchive ImportJob() > insert error: %v", err) return err From e6286768a74fc323c60346e2a45ec271b9d44d20 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 08:56:48 +0100 Subject: [PATCH 11/30] Refactor variabel naming and update doc comments --- internal/api/job.go | 14 +++--- internal/graph/schema.resolvers.go | 20 ++++---- internal/graph/util.go | 10 ++-- internal/repository/job.go | 17 ++++--- internal/repository/jobCreate.go | 13 ++--- internal/repository/jobFind.go | 67 +++++++++++++------------- internal/repository/job_test.go | 2 +- internal/repository/node.go | 4 +- internal/repository/repository_test.go | 2 +- internal/repository/tags.go | 19 ++++---- internal/tagger/detectApp_test.go | 2 +- internal/tagger/tagger.go | 4 +- internal/tagger/tagger_test.go | 2 +- 13 files changed, 88 insertions(+), 88 deletions(-) diff --git a/internal/api/job.go b/internal/api/job.go index 7701374a..919772f4 100644 --- a/internal/api/job.go +++ b/internal/api/job.go @@ -253,7 +253,7 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request) return } - job, err = api.JobRepository.FindById(r.Context(), id) // Get Job from Repo by ID + job, err = api.JobRepository.FindByID(r.Context(), id) // Get Job from Repo by ID } else { handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -346,7 +346,7 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) { return } - job, err = api.JobRepository.FindById(r.Context(), id) + job, err = api.JobRepository.FindByID(r.Context(), id) } else { handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -445,7 +445,7 @@ func (api *RestAPI) editMeta(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -493,7 +493,7 @@ func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -557,7 +557,7 @@ func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) { return } - job, err := api.JobRepository.FindById(r.Context(), id) + job, err := api.JobRepository.FindByID(r.Context(), id) if err != nil { handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw) return @@ -796,7 +796,7 @@ func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) { return } - err = api.JobRepository.DeleteJobById(id) + err = api.JobRepository.DeleteJobByID(id) } else { handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return @@ -852,7 +852,7 @@ func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) return } - err = api.JobRepository.DeleteJobById(*job.ID) + err = api.JobRepository.DeleteJobByID(*job.ID) if err != nil { handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 75556938..418d0ee3 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -376,7 +376,7 @@ func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, erro cclog.Warn("Error while parsing job id") return nil, err } - return repo.GetNodeById(numericId, false) + return repo.GetNodeByID(numericId, false) } // Nodes is the resolver for the nodes field. @@ -442,7 +442,7 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) return nil, err } - job, err := r.Repo.FindById(ctx, numericId) + job, err := r.Repo.FindByID(ctx, numericId) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -1003,10 +1003,12 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } // SubCluster returns generated.SubClusterResolver implementation. func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} } -type clusterResolver struct{ *Resolver } -type jobResolver struct{ *Resolver } -type metricValueResolver struct{ *Resolver } -type mutationResolver struct{ *Resolver } -type nodeResolver struct{ *Resolver } -type queryResolver struct{ *Resolver } -type subClusterResolver struct{ *Resolver } +type ( + clusterResolver struct{ *Resolver } + jobResolver struct{ *Resolver } + metricValueResolver struct{ *Resolver } + mutationResolver struct{ *Resolver } + nodeResolver struct{ *Resolver } + queryResolver struct{ *Resolver } + subClusterResolver struct{ *Resolver } +) diff --git a/internal/graph/util.go b/internal/graph/util.go index 38c4914f..220c3a84 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -2,12 +2,14 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package graph import ( "context" "fmt" "math" + "slices" "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" @@ -185,11 +187,5 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF func requireField(ctx context.Context, name string) bool { fields := graphql.CollectAllFields(ctx) - for _, f := range fields { - if f == name { - return true - } - } - - return false + return slices.Contains(fields, name) } diff --git a/internal/repository/job.go b/internal/repository/job.go index 2f003e3b..f23a14cf 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -376,7 +376,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged bool) (int, return cnt, err } -func (r *JobRepository) DeleteJobById(id int64) error { +func (r *JobRepository) DeleteJobByID(id int64) error { // Invalidate cache entries before deletion r.cache.Del(fmt.Sprintf("metadata:%d", id)) r.cache.Del(fmt.Sprintf("energyFootprint:%d", id)) @@ -577,10 +577,10 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { return nil } -func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) { +func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) { query := sq.Select("job.id").From("job"). Join("jobtag ON jobtag.job_id = job.id"). - Where(sq.Eq{"jobtag.tag_id": tagId}).Distinct() + Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct() rows, err := query.RunWith(r.stmtCache).Query() if err != nil { cclog.Error("Error while running query") @@ -589,15 +589,15 @@ func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) { jobIds := make([]int64, 0, 100) for rows.Next() { - var jobId int64 + var jobID int64 - if err := rows.Scan(&jobId); err != nil { + if err := rows.Scan(&jobID); err != nil { rows.Close() cclog.Warn("Error while scanning rows") return nil, err } - jobIds = append(jobIds, jobId) + jobIds = append(jobIds, jobID) } return jobIds, nil @@ -731,10 +731,11 @@ func (r *JobRepository) UpdateEnergy( metricEnergy := 0.0 if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil { // Note: For DB data, calculate and save as kWh - if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh) + switch sc.MetricConfig[i].Energy { + case "energy": // this metric has energy as unit (Joules or Wh) cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp) // FIXME: Needs sum as stats type - } else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt) + case "power": // this metric has power as unit (Watt) // Energy: Power (in Watts) * Time (in Seconds) // Unit: (W * (s / 3600)) / 1000 = kWh // Round 2 Digits: round(Energy * 100) / 100 diff --git a/internal/repository/jobCreate.go b/internal/repository/jobCreate.go index 2fcc69e9..efd262b8 100644 --- a/internal/repository/jobCreate.go +++ b/internal/repository/jobCreate.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -109,27 +110,27 @@ func (r *JobRepository) Start(job *schema.Job) (id int64, err error) { // Stop updates the job with the database id jobId using the provided arguments. func (r *JobRepository) Stop( - jobId int64, + jobID int64, duration int32, state schema.JobState, monitoringStatus int32, ) (err error) { // Invalidate cache entries as job state is changing - r.cache.Del(fmt.Sprintf("metadata:%d", jobId)) - r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobId)) + r.cache.Del(fmt.Sprintf("metadata:%d", jobID)) + r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID)) stmt := sq.Update("job"). Set("job_state", state). Set("duration", duration). Set("monitoring_status", monitoringStatus). - Where("job.id = ?", jobId) + Where("job.id = ?", jobID) _, err = stmt.RunWith(r.stmtCache).Exec() return err } func (r *JobRepository) StopCached( - jobId int64, + jobID int64, duration int32, state schema.JobState, monitoringStatus int32, @@ -140,7 +141,7 @@ func (r *JobRepository) StopCached( Set("job_state", state). Set("duration", duration). Set("monitoring_status", monitoringStatus). - Where("job_cache.id = ?", jobId) + Where("job_cache.id = ?", jobID) _, err = stmt.RunWith(r.stmtCache).Exec() return err diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 11f66c40..c4051e7f 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -22,13 +23,13 @@ import ( // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows func (r *JobRepository) Find( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) (*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). - Where("job.job_id = ?", *jobId) + Where("job.job_id = ?", *jobID) if cluster != nil { q = q.Where("job.cluster = ?", *cluster) @@ -44,12 +45,12 @@ func (r *JobRepository) Find( } func (r *JobRepository) FindCached( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) (*schema.Job, error) { q := sq.Select(jobCacheColumns...).From("job_cache"). - Where("job_cache.job_id = ?", *jobId) + Where("job_cache.job_id = ?", *jobID) if cluster != nil { q = q.Where("job_cache.cluster = ?", *cluster) @@ -63,19 +64,19 @@ func (r *JobRepository) FindCached( return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// Find executes a SQL query to find a specific batch job. -// The job is queried using the batch job id, the cluster name, -// and the start time of the job in UNIX epoch time seconds. -// It returns a pointer to a schema.Job data structure and an error variable. -// To check if no job was found test err == sql.ErrNoRows +// FindAll executes a SQL query to find all batch jobs matching the given criteria. +// Jobs are queried using the batch job id, and optionally filtered by cluster name +// and start time (UNIX epoch time seconds). +// It returns a slice of pointers to schema.Job data structures and an error variable. +// An empty slice is returned if no matching jobs are found. func (r *JobRepository) FindAll( - jobId *int64, + jobID *int64, cluster *string, startTime *int64, ) ([]*schema.Job, error) { start := time.Now() q := sq.Select(jobColumns...).From("job"). - Where("job.job_id = ?", *jobId) + Where("job.job_id = ?", *jobID) if cluster != nil { q = q.Where("job.cluster = ?", *cluster) @@ -139,13 +140,13 @@ func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) { return jl, nil } -// FindById executes a SQL query to find a specific batch job. +// FindByID executes a SQL query to find a specific batch job. // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) q, qerr := SecurityCheck(ctx, q) if qerr != nil { @@ -155,14 +156,14 @@ func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByIdWithUser executes a SQL query to find a specific batch job. +// FindByIDWithUser executes a SQL query to find a specific batch job. // The job is queried using the database id. The user is passed directly, // instead as part of the context. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) q, qerr := SecurityCheckWithUser(user, q) if qerr != nil { @@ -172,24 +173,24 @@ func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schem return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByIdDirect executes a SQL query to find a specific batch job. +// FindByIDDirect executes a SQL query to find a specific batch job. // The job is queried using the database id. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByIdDirect(jobId int64) (*schema.Job, error) { +func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) { q := sq.Select(jobColumns...). - From("job").Where("job.id = ?", jobId) + From("job").Where("job.id = ?", jobID) return scanJob(q.RunWith(r.stmtCache).QueryRow()) } -// FindByJobId executes a SQL query to find a specific batch job. +// FindByJobID executes a SQL query to find a specific batch job. // The job is queried using the slurm id and the clustername. // It returns a pointer to a schema.Job data structure and an error variable. // To check if no job was found test err == sql.ErrNoRows -func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime int64, cluster string) (*schema.Job, error) { +func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) { q := sq.Select(jobColumns...). From("job"). - Where("job.job_id = ?", jobId). + Where("job.job_id = ?", jobID). Where("job.cluster = ?", cluster). Where("job.start_time = ?", startTime) @@ -205,10 +206,10 @@ func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime // The job is queried using the slurm id,a username and the cluster. // It returns a bool. // If job was found, user is owner: test err != sql.ErrNoRows -func (r *JobRepository) IsJobOwner(jobId int64, startTime int64, user string, cluster string) bool { +func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool { q := sq.Select("id"). From("job"). - Where("job.job_id = ?", jobId). + Where("job.job_id = ?", jobID). Where("job.hpc_user = ?", user). Where("job.cluster = ?", cluster). Where("job.start_time = ?", startTime) @@ -269,19 +270,19 @@ func (r *JobRepository) FindConcurrentJobs( queryString := fmt.Sprintf("cluster=%s", job.Cluster) for rows.Next() { - var id, jobId, startTime sql.NullInt64 + var id, jobID, startTime sql.NullInt64 - if err = rows.Scan(&id, &jobId, &startTime); err != nil { + if err = rows.Scan(&id, &jobID, &startTime); err != nil { cclog.Warn("Error while scanning rows") return nil, err } if id.Valid { - queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64)) + queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64)) items = append(items, &model.JobLink{ ID: fmt.Sprint(id.Int64), - JobID: int(jobId.Int64), + JobID: int(jobID.Int64), }) } } @@ -294,19 +295,19 @@ func (r *JobRepository) FindConcurrentJobs( defer rows.Close() for rows.Next() { - var id, jobId, startTime sql.NullInt64 + var id, jobID, startTime sql.NullInt64 - if err := rows.Scan(&id, &jobId, &startTime); err != nil { + if err := rows.Scan(&id, &jobID, &startTime); err != nil { cclog.Warn("Error while scanning rows") return nil, err } if id.Valid { - queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64)) + queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64)) items = append(items, &model.JobLink{ ID: fmt.Sprint(id.Int64), - JobID: int(jobId.Int64), + JobID: int(jobID.Int64), }) } } diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index 9415bf98..c89225b3 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -33,7 +33,7 @@ func TestFind(t *testing.T) { func TestFindById(t *testing.T) { r := setup(t) - job, err := r.FindById(getContext(t), 338) + job, err := r.FindByID(getContext(t), 338) if err != nil { t.Fatal(err) } diff --git a/internal/repository/node.go b/internal/repository/node.go index 1e302704..9a1f3530 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -106,7 +106,7 @@ func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) return node, nil } -func (r *NodeRepository) GetNodeById(id int64, withMeta bool) (*schema.Node, error) { +func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) { node := &schema.Node{} var timestamp int if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state", @@ -240,7 +240,6 @@ func (r *NodeRepository) QueryNodes( page *model.PageRequest, order *model.OrderByInput, // Currently unused! ) ([]*schema.Node, error) { - query, qerr := AccessCheck(ctx, sq.Select("hostname", "cluster", "subcluster", "node_state", "health_state", "MAX(time_stamp) as time"). From("node"). @@ -309,7 +308,6 @@ func (r *NodeRepository) CountNodes( ctx context.Context, filters []*model.NodeFilter, ) (int, error) { - query, qerr := AccessCheck(ctx, sq.Select("time_stamp", "count(*) as countRes"). From("node"). diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 5603c31c..1346e4da 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -55,7 +55,7 @@ func BenchmarkDB_FindJobById(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := db.FindById(getContext(b), jobId) + _, err := db.FindByID(getContext(b), jobId) noErr(b, err) } }) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 52bd9076..5ca13382 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package repository import ( @@ -18,7 +19,7 @@ import ( // AddTag adds the tag with id `tagId` to the job with the database id `jobId`. // Requires user authentication for security checks. func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warnf("Error finding job %d for user %s: %v", job, user.Username, err) return nil, err @@ -32,7 +33,7 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche // AddTagDirect adds a tag without user security checks. // Use only for internal/admin operations. func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdDirect(job) + j, err := r.FindByIDDirect(job) if err != nil { cclog.Warnf("Error finding job %d: %v", job, err) return nil, err @@ -43,10 +44,10 @@ func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error }) } -// Removes a tag from a job by tag id. -// Used by GraphQL API +// RemoveTag removes the tag with the database id `tag` from the job with the database id `job`. +// Requires user authentication for security checks. Used by GraphQL API. func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) { - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -75,8 +76,8 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema. return tags, archive.UpdateTags(j, archiveTags) } -// Removes a tag from a job by tag info -// Used by REST API +// RemoveJobTagByRequest removes a tag from the job with the database id `job` by tag type, name, and scope. +// Requires user authentication for security checks. Used by REST API. func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) { // Get Tag ID to delete tagID, exists := r.TagId(tagType, tagName, tagScope) @@ -86,7 +87,7 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT } // Get Job - j, err := r.FindByIdWithUser(user, job) + j, err := r.FindByIDWithUser(user, job) if err != nil { cclog.Warn("Error while finding job by id") return nil, err @@ -124,7 +125,7 @@ func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) { continue } - job, err := r.FindByIdDirect(j) + job, err := r.FindByIDDirect(j) if err != nil { cclog.Warnf("Error while getting job %d", j) continue diff --git a/internal/tagger/detectApp_test.go b/internal/tagger/detectApp_test.go index 295ee97c..f9fc91d0 100644 --- a/internal/tagger/detectApp_test.go +++ b/internal/tagger/detectApp_test.go @@ -43,7 +43,7 @@ func TestRegister(t *testing.T) { func TestMatch(t *testing.T) { r := setup(t) - job, err := r.FindByIdDirect(317) + job, err := r.FindByIDDirect(317) noErr(t, err) var tagger AppTagger diff --git a/internal/tagger/tagger.go b/internal/tagger/tagger.go index 028d9efe..2ba18a14 100644 --- a/internal/tagger/tagger.go +++ b/internal/tagger/tagger.go @@ -40,7 +40,7 @@ type JobTagger struct { // startTaggers are applied when a job starts (e.g., application detection) startTaggers []Tagger // stopTaggers are applied when a job completes (e.g., job classification) - stopTaggers []Tagger + stopTaggers []Tagger } func newTagger() { @@ -98,7 +98,7 @@ func RunTaggers() error { } for _, id := range jl { - job, err := r.FindByIdDirect(id) + job, err := r.FindByIDDirect(id) if err != nil { cclog.Errorf("Error while getting job %s", err) return err diff --git a/internal/tagger/tagger_test.go b/internal/tagger/tagger_test.go index c81fac4a..fb4bc54e 100644 --- a/internal/tagger/tagger_test.go +++ b/internal/tagger/tagger_test.go @@ -18,7 +18,7 @@ func TestInit(t *testing.T) { func TestJobStartCallback(t *testing.T) { Init() r := setup(t) - job, err := r.FindByIdDirect(525) + job, err := r.FindByIDDirect(525) noErr(t, err) jobs := make([]*schema.Job, 0, 1) From 7fce6fa401acb9cf443b561ca2e44bb436287d19 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:11:09 +0100 Subject: [PATCH 12/30] Parallelize the Iter function in all archive backends --- pkg/archive/fsBackend.go | 69 +++++++++++++++-------- pkg/archive/s3Backend.go | 56 ++++++++++++------- pkg/archive/sqliteBackend.go | 103 +++++++++++++++++++++-------------- 3 files changed, 144 insertions(+), 84 deletions(-) diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 22c1772a..b8d2a94b 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "text/tabwriter" "time" @@ -490,7 +491,46 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { ch := make(chan JobContainer) + go func() { + defer close(ch) + + numWorkers := 4 + jobPaths := make(chan string, numWorkers*2) + var wg sync.WaitGroup + + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for jobPath := range jobPaths { + job, err := loadJobMeta(filepath.Join(jobPath, "meta.json")) + if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { + cclog.Errorf("in %s: %s", jobPath, err.Error()) + continue + } + + if loadMetricData { + isCompressed := true + filename := filepath.Join(jobPath, "data.json.gz") + + if !util.CheckFileExists(filename) { + filename = filepath.Join(jobPath, "data.json") + isCompressed = false + } + + data, err := loadJobData(filename, isCompressed) + if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { + cclog.Errorf("in %s: %s", jobPath, err.Error()) + } + ch <- JobContainer{Meta: job, Data: &data} + } else { + ch <- JobContainer{Meta: job, Data: nil} + } + } + }() + } + clustersDir, err := os.ReadDir(fsa.path) if err != nil { cclog.Fatalf("Reading clusters failed @ cluster dirs: %s", err.Error()) @@ -507,7 +547,6 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { for _, lvl1Dir := range lvl1Dirs { if !lvl1Dir.IsDir() { - // Could be the cluster.json file continue } @@ -525,35 +564,17 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer { for _, startTimeDir := range startTimeDirs { if startTimeDir.IsDir() { - job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json")) - if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { - cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) - } - - if loadMetricData { - isCompressed := true - filename := filepath.Join(dirpath, startTimeDir.Name(), "data.json.gz") - - if !util.CheckFileExists(filename) { - filename = filepath.Join(dirpath, startTimeDir.Name(), "data.json") - isCompressed = false - } - - data, err := loadJobData(filename, isCompressed) - if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) { - cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) - } - ch <- JobContainer{Meta: job, Data: &data} - } else { - ch <- JobContainer{Meta: job, Data: nil} - } + jobPaths <- filepath.Join(dirpath, startTimeDir.Name()) } } } } } - close(ch) + + close(jobPaths) + wg.Wait() }() + return ch } diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go index eacdde76..c874a320 100644 --- a/pkg/archive/s3Backend.go +++ b/pkg/archive/s3Backend.go @@ -17,6 +17,7 @@ import ( "os" "strconv" "strings" + "sync" "text/tabwriter" "time" @@ -813,29 +814,18 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { ctx := context.Background() defer close(ch) - for _, cluster := range s3a.clusters { - prefix := cluster + "/" + numWorkers := 4 + metaKeys := make(chan string, numWorkers*2) + var wg sync.WaitGroup - paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s3a.bucket), - Prefix: aws.String(prefix), - }) - - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error()) - } - - for _, obj := range page.Contents { - if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") { - continue - } - - // Load job metadata + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for metaKey := range metaKeys { result, err := s3a.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s3a.bucket), - Key: obj.Key, + Key: aws.String(metaKey), }) if err != nil { cclog.Errorf("S3Archive Iter() > GetObject meta error: %v", err) @@ -867,8 +857,34 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer { ch <- JobContainer{Meta: job, Data: nil} } } + }() + } + + for _, cluster := range s3a.clusters { + prefix := cluster + "/" + + paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s3a.bucket), + Prefix: aws.String(prefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error()) + } + + for _, obj := range page.Contents { + if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") { + continue + } + metaKeys <- *obj.Key + } } } + + close(metaKeys) + wg.Wait() }() return ch diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 589beea4..6fa188ba 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -16,6 +16,7 @@ import ( "os" "slices" "strconv" + "sync" "text/tabwriter" "time" @@ -547,62 +548,84 @@ func (sa *SqliteArchive) CompressLast(starttime int64) int64 { return last } +type sqliteJobRow struct { + metaBlob []byte + dataBlob []byte + compressed bool +} + func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { ch := make(chan JobContainer) go func() { defer close(ch) - rows, err := sa.db.Query("SELECT job_id, cluster, start_time, meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") + rows, err := sa.db.Query("SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") if err != nil { cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) } defer rows.Close() - for rows.Next() { - var jobID int64 - var cluster string - var startTime int64 - var metaBlob []byte - var dataBlob []byte - var compressed bool + numWorkers := 4 + jobRows := make(chan sqliteJobRow, numWorkers*2) + var wg sync.WaitGroup - if err := rows.Scan(&jobID, &cluster, &startTime, &metaBlob, &dataBlob, &compressed); err != nil { + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for row := range jobRows { + job, err := DecodeJobMeta(bytes.NewReader(row.metaBlob)) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err) + continue + } + + if loadMetricData && row.dataBlob != nil { + var reader io.Reader = bytes.NewReader(row.dataBlob) + if row.compressed { + gzipReader, err := gzip.NewReader(reader) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + continue + } + decompressed, err := io.ReadAll(gzipReader) + gzipReader.Close() + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decompress error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + continue + } + reader = bytes.NewReader(decompressed) + } + + key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime) + jobData, err := DecodeJobData(reader, key) + if err != nil { + cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err) + ch <- JobContainer{Meta: job, Data: nil} + } else { + ch <- JobContainer{Meta: job, Data: &jobData} + } + } else { + ch <- JobContainer{Meta: job, Data: nil} + } + } + }() + } + + for rows.Next() { + var row sqliteJobRow + if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) continue } - - job, err := DecodeJobMeta(bytes.NewReader(metaBlob)) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err) - continue - } - - if loadMetricData && dataBlob != nil { - var reader io.Reader = bytes.NewReader(dataBlob) - if compressed { - gzipReader, err := gzip.NewReader(reader) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err) - ch <- JobContainer{Meta: job, Data: nil} - continue - } - defer gzipReader.Close() - reader = gzipReader - } - - key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime) - jobData, err := DecodeJobData(reader, key) - if err != nil { - cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err) - ch <- JobContainer{Meta: job, Data: nil} - } else { - ch <- JobContainer{Meta: job, Data: &jobData} - } - } else { - ch <- JobContainer{Meta: job, Data: nil} - } + jobRows <- row } + + close(jobRows) + wg.Wait() }() return ch From 72b2560ecfcd6eb1bafe46a01840547f0c508275 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:11:26 +0100 Subject: [PATCH 13/30] Add progress bar for import function --- tools/archive-manager/main.go | 158 +++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 31 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 30aa9088..ae0ae9e6 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "strings" "sync" "sync/atomic" "time" @@ -33,78 +34,172 @@ func parseDate(in string) int64 { return 0 } +// countJobs counts the total number of jobs in the source archive. +func countJobs(srcBackend archive.ArchiveBackend) int { + count := 0 + for range srcBackend.Iter(false) { + count++ + } + return count +} + +// formatDuration formats a duration as a human-readable string. +func formatDuration(d time.Duration) string { + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } else if d < time.Hour { + return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) + } + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) +} + +// progressMeter displays import progress to the terminal. +type progressMeter struct { + total int + processed int32 + imported int32 + skipped int32 + failed int32 + startTime time.Time + done chan struct{} +} + +func newProgressMeter(total int) *progressMeter { + return &progressMeter{ + total: total, + startTime: time.Now(), + done: make(chan struct{}), + } +} + +func (p *progressMeter) start() { + go func() { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + p.render() + case <-p.done: + p.render() + fmt.Println() + return + } + } + }() +} + +func (p *progressMeter) render() { + processed := atomic.LoadInt32(&p.processed) + imported := atomic.LoadInt32(&p.imported) + skipped := atomic.LoadInt32(&p.skipped) + failed := atomic.LoadInt32(&p.failed) + + elapsed := time.Since(p.startTime) + percent := float64(processed) / float64(p.total) * 100 + if p.total == 0 { + percent = 0 + } + + var eta string + var throughput float64 + if processed > 0 { + throughput = float64(processed) / elapsed.Seconds() + remaining := float64(p.total-int(processed)) / throughput + eta = formatDuration(time.Duration(remaining) * time.Second) + } else { + eta = "calculating..." + } + + barWidth := 30 + filled := int(float64(barWidth) * float64(processed) / float64(p.total)) + if p.total == 0 { + filled = 0 + } + + var bar strings.Builder + for i := range barWidth { + if i < filled { + bar.WriteString("█") + } else { + bar.WriteString("░") + } + } + + fmt.Printf("\r[%s] %5.1f%% | %d/%d | %.1f jobs/s | ETA: %s | ✓%d ○%d ✗%d ", + bar.String(), percent, processed, p.total, throughput, eta, imported, skipped, failed) +} + +func (p *progressMeter) stop() { + close(p.done) +} + // importArchive imports all jobs from a source archive backend to a destination archive backend. // It uses parallel processing with a worker pool to improve performance. // Returns the number of successfully imported jobs, failed jobs, and any error encountered. func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, error) { cclog.Info("Starting parallel archive import...") - // Use atomic counters for thread-safe updates - var imported int32 - var failed int32 - var skipped int32 + cclog.Info("Counting jobs in source archive (this may take a long time) ...") + totalJobs := countJobs(srcBackend) + cclog.Infof("Found %d jobs to process", totalJobs) + + progress := newProgressMeter(totalJobs) - // Number of parallel workers numWorkers := 4 cclog.Infof("Using %d parallel workers", numWorkers) - // Create channels for job distribution jobs := make(chan archive.JobContainer, numWorkers*2) - // WaitGroup to track worker completion var wg sync.WaitGroup - // Start worker goroutines + progress.start() + for i := range numWorkers { wg.Add(1) go func(workerID int) { defer wg.Done() for job := range jobs { - // Validate job metadata if job.Meta == nil { cclog.Warn("Skipping job with nil metadata") - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Validate job data if job.Data == nil { cclog.Warnf("Job %d from cluster %s has no metric data, skipping", job.Meta.JobID, job.Meta.Cluster) - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Check if job already exists in destination if dstBackend.Exists(job.Meta) { cclog.Debugf("Job %d (cluster: %s, start: %d) already exists in destination, skipping", job.Meta.JobID, job.Meta.Cluster, job.Meta.StartTime) - atomic.AddInt32(&skipped, 1) + atomic.AddInt32(&progress.skipped, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Import job to destination if err := dstBackend.ImportJob(job.Meta, job.Data); err != nil { cclog.Errorf("Failed to import job %d from cluster %s: %s", job.Meta.JobID, job.Meta.Cluster, err.Error()) - atomic.AddInt32(&failed, 1) + atomic.AddInt32(&progress.failed, 1) + atomic.AddInt32(&progress.processed, 1) continue } - // Successfully imported - newCount := atomic.AddInt32(&imported, 1) - if newCount%100 == 0 { - cclog.Infof("Progress: %d jobs imported, %d skipped, %d failed", - newCount, atomic.LoadInt32(&skipped), atomic.LoadInt32(&failed)) - } + atomic.AddInt32(&progress.imported, 1) + atomic.AddInt32(&progress.processed, 1) } }(i) } - // Feed jobs to workers go func() { - // Import cluster configs first clusters := srcBackend.GetClusters() for _, clusterName := range clusters { clusterCfg, err := srcBackend.LoadClusterCfg(clusterName) @@ -126,15 +221,16 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err close(jobs) }() - // Wait for all workers to complete wg.Wait() + progress.stop() - finalImported := int(atomic.LoadInt32(&imported)) - finalFailed := int(atomic.LoadInt32(&failed)) - finalSkipped := int(atomic.LoadInt32(&skipped)) + finalImported := int(atomic.LoadInt32(&progress.imported)) + finalFailed := int(atomic.LoadInt32(&progress.failed)) + finalSkipped := int(atomic.LoadInt32(&progress.skipped)) - cclog.Infof("Import completed: %d jobs imported, %d skipped, %d failed", - finalImported, finalSkipped, finalFailed) + elapsed := time.Since(progress.startTime) + cclog.Infof("Import completed in %s: %d jobs imported, %d skipped, %d failed", + formatDuration(elapsed), finalImported, finalSkipped, finalFailed) if finalFailed > 0 { return finalImported, finalFailed, fmt.Errorf("%d jobs failed to import", finalFailed) @@ -150,7 +246,7 @@ func main() { flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path. Default is ./var/job-archive") flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "info", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagRemoveCluster, "remove-cluster", "", "Remove cluster from archive and database") flag.StringVar(&flagRemoveBefore, "remove-before", "", "Remove all jobs with start time before date (Format: 2006-Jan-04)") From 14f1192ccbb04fe66019366726d05c0925306d3c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:35:33 +0100 Subject: [PATCH 14/30] Introduce central nats client --- cmd/cc-backend/main.go | 8 ++ cmd/cc-backend/server.go | 6 + pkg/nats/client.go | 246 +++++++++++++++++++++++++++++++++++++++ pkg/nats/config.go | 63 ++++++++++ 4 files changed, 323 insertions(+) create mode 100644 pkg/nats/client.go create mode 100644 pkg/nats/config.go diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index d89109e3..c3e33872 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -30,6 +30,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/tagger" "github.com/ClusterCockpit/cc-backend/internal/taskmanager" "github.com/ClusterCockpit/cc-backend/pkg/archive" + "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" ccconf "github.com/ClusterCockpit/cc-lib/ccConfig" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" @@ -267,6 +268,13 @@ func generateJWT(authHandle *auth.Authentication, username string) error { } func initSubsystems() error { + // Initialize nats client + natsConfig := ccconf.GetPackageConfig("nats") + if err := nats.Init(natsConfig); err != nil { + return fmt.Errorf("initializing nats client: %w", err) + } + nats.Connect() + // Initialize job archive archiveCfg := ccconf.GetPackageConfig("archive") if archiveCfg == nil { diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go index 975d38a1..2c5ce8bc 100644 --- a/cmd/cc-backend/server.go +++ b/cmd/cc-backend/server.go @@ -31,6 +31,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph/generated" "github.com/ClusterCockpit/cc-backend/internal/memorystore" "github.com/ClusterCockpit/cc-backend/internal/routerConfig" + "github.com/ClusterCockpit/cc-backend/pkg/nats" "github.com/ClusterCockpit/cc-backend/web" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/runtimeEnv" @@ -363,6 +364,11 @@ func (s *Server) Shutdown(ctx context.Context) { shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + nc := nats.GetClient() + if nc != nil { + nc.Close() + } + // First shut down the server gracefully (waiting for all ongoing requests) if err := s.server.Shutdown(shutdownCtx); err != nil { cclog.Errorf("Server shutdown error: %v", err) diff --git a/pkg/nats/client.go b/pkg/nats/client.go new file mode 100644 index 00000000..e61d060b --- /dev/null +++ b/pkg/nats/client.go @@ -0,0 +1,246 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package nats provides a generic NATS messaging client for publish/subscribe communication. +// +// The package wraps the nats.go library with connection management, automatic reconnection +// handling, and subscription tracking. It supports multiple authentication methods including +// username/password and credential files. +// +// # Configuration +// +// Configure the client via JSON in the application config: +// +// { +// "nats": { +// "address": "nats://localhost:4222", +// "username": "user", +// "password": "secret" +// } +// } +// +// Or using a credentials file: +// +// { +// "nats": { +// "address": "nats://localhost:4222", +// "creds-file-path": "/path/to/creds.json" +// } +// } +// +// # Usage +// +// The package provides a singleton client initialized once and retrieved globally: +// +// nats.Init(rawConfig) +// nats.Connect() +// +// client := nats.GetClient() +// client.Subscribe("events", func(subject string, data []byte) { +// fmt.Printf("Received: %s\n", data) +// }) +// +// client.Publish("events", []byte("hello")) +// +// # Thread Safety +// +// All Client methods are safe for concurrent use. +package nats + +import ( + "context" + "fmt" + "sync" + + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/nats-io/nats.go" +) + +var ( + clientOnce sync.Once + clientInstance *Client +) + +// Client wraps a NATS connection with subscription management. +type Client struct { + conn *nats.Conn + subscriptions []*nats.Subscription + mu sync.Mutex +} + +// MessageHandler is a callback function for processing received messages. +type MessageHandler func(subject string, data []byte) + +// Connect initializes the singleton NATS client using the global Keys config. +func Connect() { + clientOnce.Do(func() { + if Keys.Address == "" { + cclog.Warn("NATS: no address configured, skipping connection") + return + } + + client, err := NewClient(nil) + if err != nil { + cclog.Errorf("NATS connection failed: %v", err) + return + } + + clientInstance = client + }) +} + +// GetClient returns the singleton NATS client instance. +func GetClient() *Client { + if clientInstance == nil { + cclog.Warn("NATS client not initialized") + } + return clientInstance +} + +// NewClient creates a new NATS client. If cfg is nil, uses the global Keys config. +func NewClient(cfg *NatsConfig) (*Client, error) { + if cfg == nil { + cfg = &Keys + } + + if cfg.Address == "" { + return nil, fmt.Errorf("NATS address is required") + } + + var opts []nats.Option + + if cfg.Username != "" && cfg.Password != "" { + opts = append(opts, nats.UserInfo(cfg.Username, cfg.Password)) + } + + if cfg.CredsFilePath != "" { + opts = append(opts, nats.UserCredentials(cfg.CredsFilePath)) + } + + opts = append(opts, nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + if err != nil { + cclog.Warnf("NATS disconnected: %v", err) + } + })) + + opts = append(opts, nats.ReconnectHandler(func(nc *nats.Conn) { + cclog.Infof("NATS reconnected to %s", nc.ConnectedUrl()) + })) + + opts = append(opts, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + cclog.Errorf("NATS error: %v", err) + })) + + nc, err := nats.Connect(cfg.Address, opts...) + if err != nil { + return nil, fmt.Errorf("NATS connect failed: %w", err) + } + + cclog.Infof("NATS connected to %s", cfg.Address) + + return &Client{ + conn: nc, + subscriptions: make([]*nats.Subscription, 0), + }, nil +} + +// Subscribe registers a handler for messages on the given subject. +func (c *Client) Subscribe(subject string, handler MessageHandler) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.Subscribe(subject, func(msg *nats.Msg) { + handler(msg.Subject, msg.Data) + }) + if err != nil { + return fmt.Errorf("NATS subscribe to '%s' failed: %w", subject, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS subscribed to '%s'", subject) + return nil +} + +// SubscribeQueue registers a handler with queue group for load-balanced message processing. +func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) { + handler(msg.Subject, msg.Data) + }) + if err != nil { + return fmt.Errorf("NATS queue subscribe to '%s' (queue: %s) failed: %w", subject, queue, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS queue subscribed to '%s' (queue: %s)", subject, queue) + return nil +} + +// SubscribeChan subscribes to a subject and delivers messages to the provided channel. +func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error { + c.mu.Lock() + defer c.mu.Unlock() + + sub, err := c.conn.ChanSubscribe(subject, ch) + if err != nil { + return fmt.Errorf("NATS chan subscribe to '%s' failed: %w", subject, err) + } + + c.subscriptions = append(c.subscriptions, sub) + cclog.Infof("NATS chan subscribed to '%s'", subject) + return nil +} + +// Publish sends data to the specified subject. +func (c *Client) Publish(subject string, data []byte) error { + if err := c.conn.Publish(subject, data); err != nil { + return fmt.Errorf("NATS publish to '%s' failed: %w", subject, err) + } + return nil +} + +// Request sends a request and waits for a response with the given context timeout. +func (c *Client) Request(subject string, data []byte, timeout context.Context) ([]byte, error) { + msg, err := c.conn.RequestWithContext(timeout, subject, data) + if err != nil { + return nil, fmt.Errorf("NATS request to '%s' failed: %w", subject, err) + } + return msg.Data, nil +} + +// Flush flushes the connection buffer to ensure all published messages are sent. +func (c *Client) Flush() error { + return c.conn.Flush() +} + +// Close unsubscribes all subscriptions and closes the NATS connection. +func (c *Client) Close() { + c.mu.Lock() + defer c.mu.Unlock() + + for _, sub := range c.subscriptions { + if err := sub.Unsubscribe(); err != nil { + cclog.Warnf("NATS unsubscribe failed: %v", err) + } + } + c.subscriptions = nil + + if c.conn != nil { + c.conn.Close() + cclog.Info("NATS connection closed") + } +} + +// IsConnected returns true if the client has an active connection. +func (c *Client) IsConnected() bool { + return c.conn != nil && c.conn.IsConnected() +} + +// Connection returns the underlying NATS connection for advanced usage. +func (c *Client) Connection() *nats.Conn { + return c.conn +} diff --git a/pkg/nats/config.go b/pkg/nats/config.go new file mode 100644 index 00000000..32a0bbda --- /dev/null +++ b/pkg/nats/config.go @@ -0,0 +1,63 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nats + +import ( + "bytes" + "encoding/json" + + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" +) + +// NatsConfig holds the configuration for connecting to a NATS server. +type NatsConfig struct { + Address string `json:"address"` // NATS server address (e.g., "nats://localhost:4222") + Username string `json:"username"` // Username for authentication (optional) + Password string `json:"password"` // Password for authentication (optional) + CredsFilePath string `json:"creds-file-path"` // Path to credentials file (optional) +} + +// Keys holds the global NATS configuration loaded via Init. +var Keys NatsConfig + +const ConfigSchema = `{ + "type": "object", + "description": "Configuration for NATS messaging client.", + "properties": { + "address": { + "description": "Address of the NATS server (e.g., 'nats://localhost:4222').", + "type": "string" + }, + "username": { + "description": "Username for NATS authentication (optional).", + "type": "string" + }, + "password": { + "description": "Password for NATS authentication (optional).", + "type": "string" + }, + "creds-file-path": { + "description": "Path to NATS credentials file for authentication (optional).", + "type": "string" + } + }, + "required": ["address"] +}` + +// Init initializes the global Keys configuration from JSON. +func Init(rawConfig json.RawMessage) error { + var err error + + if rawConfig != nil { + dec := json.NewDecoder(bytes.NewReader(rawConfig)) + dec.DisallowUnknownFields() + if err = dec.Decode(&Keys); err != nil { + cclog.Errorf("Error while initializing nats client: %s", err.Error()) + } + } + + return err +} From 5e2cbd75fae36d949180c0d71105def3a0482ea7 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 16 Dec 2025 09:45:48 +0100 Subject: [PATCH 15/30] Review and refactor --- internal/memorystore/api.go | 16 +++- internal/memorystore/archive.go | 51 ++++++----- internal/memorystore/avroCheckpoint.go | 19 ++-- internal/memorystore/avroHelper.go | 6 +- internal/memorystore/avroStruct.go | 9 +- internal/memorystore/buffer.go | 14 +-- internal/memorystore/checkpoint.go | 48 +++------- internal/memorystore/config.go | 26 +++--- internal/memorystore/level.go | 2 +- internal/memorystore/lineprotocol.go | 119 +++---------------------- internal/memorystore/memorystore.go | 40 +++------ internal/memorystore/stats.go | 2 +- 12 files changed, 108 insertions(+), 244 deletions(-) diff --git a/internal/memorystore/api.go b/internal/memorystore/api.go index 1f7a531f..b96dc1fd 100644 --- a/internal/memorystore/api.go +++ b/internal/memorystore/api.go @@ -6,12 +6,18 @@ package memorystore import ( + "errors" "math" "github.com/ClusterCockpit/cc-lib/schema" "github.com/ClusterCockpit/cc-lib/util" ) +var ( + ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'") + ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty") +) + type APIMetricData struct { Error *string `json:"error,omitempty"` Data schema.FloatArray `json:"data,omitempty"` @@ -109,10 +115,14 @@ func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metr } func FetchData(req APIQueryRequest) (*APIQueryResponse, error) { - req.WithData = true - req.WithData = true - req.WithData = true + if req.From > req.To { + return nil, ErrInvalidTimeRange + } + if req.Cluster == "" && req.ForAllNodes != nil { + return nil, ErrEmptyCluster + } + req.WithData = true ms := GetMemoryStore() response := APIQueryResponse{ diff --git a/internal/memorystore/archive.go b/internal/memorystore/archive.go index 56065aaf..5019ee7a 100644 --- a/internal/memorystore/archive.go +++ b/internal/memorystore/archive.go @@ -32,17 +32,14 @@ func Archiving(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(d) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: t := time.Now().Add(-d) cclog.Infof("[METRICSTORE]> start archiving checkpoints (older than %s)...", t.Format(time.RFC3339)) n, err := ArchiveCheckpoints(Keys.Checkpoints.RootDir, @@ -165,25 +162,33 @@ func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead n := 0 for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - r, err := os.Open(filename) + // Use closure to ensure file is closed immediately after use, + // avoiding file descriptor leak from defer in loop + err := func() error { + filename := filepath.Join(dir, checkpoint) + r, err := os.Open(filename) + if err != nil { + return err + } + defer r.Close() + + w, err := zw.Create(checkpoint) + if err != nil { + return err + } + + if _, err = io.Copy(w, r); err != nil { + return err + } + + if err = os.Remove(filename); err != nil { + return err + } + return nil + }() if err != nil { return n, err } - defer r.Close() - - w, err := zw.Create(checkpoint) - if err != nil { - return n, err - } - - if _, err = io.Copy(w, r); err != nil { - return n, err - } - - if err = os.Remove(filename); err != nil { - return n, err - } n += 1 } diff --git a/internal/memorystore/avroCheckpoint.go b/internal/memorystore/avroCheckpoint.go index 4d361514..42e5f623 100644 --- a/internal/memorystore/avroCheckpoint.go +++ b/internal/memorystore/avroCheckpoint.go @@ -24,9 +24,8 @@ import ( "github.com/linkedin/goavro/v2" ) -var NumAvroWorkers int = 4 +var NumAvroWorkers int = DefaultAvroWorkers var startUp bool = true -var ErrNoNewData error = errors.New("no data in the pool") func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { levels := make([]*AvroLevel, 0) @@ -464,19 +463,15 @@ func generateRecord(data map[string]schema.Float) map[string]any { } func correctKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, ":", "___") - key = strings.ReplaceAll(key, ".", "__") - + key = strings.ReplaceAll(key, "_", "_0x5F_") + key = strings.ReplaceAll(key, ":", "_0x3A_") + key = strings.ReplaceAll(key, ".", "_0x2E_") return key } func ReplaceKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, "___", ":") - key = strings.ReplaceAll(key, "__", ".") - + key = strings.ReplaceAll(key, "_0x2E_", ".") + key = strings.ReplaceAll(key, "_0x3A_", ":") + key = strings.ReplaceAll(key, "_0x5F_", "_") return key } diff --git a/internal/memorystore/avroHelper.go b/internal/memorystore/avroHelper.go index 64e57064..a6f6c9bf 100644 --- a/internal/memorystore/avroHelper.go +++ b/internal/memorystore/avroHelper.go @@ -42,7 +42,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { metricName := "" for _, selectorName := range val.Selector { - metricName += selectorName + Delimiter + metricName += selectorName + SelectorDelimiter } metricName += val.MetricName @@ -54,7 +54,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { var selector []string selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10)) - if !testEq(oldSelector, selector) { + if !stringSlicesEqual(oldSelector, selector) { // Get the Avro level for the metric avroLevel = avroStore.root.findAvroLevelOrCreate(selector) @@ -71,7 +71,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) { }() } -func testEq(a, b []string) bool { +func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { return false } diff --git a/internal/memorystore/avroStruct.go b/internal/memorystore/avroStruct.go index cc8005c7..bde9e02b 100644 --- a/internal/memorystore/avroStruct.go +++ b/internal/memorystore/avroStruct.go @@ -13,12 +13,11 @@ import ( var ( LineProtocolMessages = make(chan *AvroStruct) - Delimiter = "ZZZZZ" + // SelectorDelimiter separates hierarchical selector components in metric names for Avro encoding + SelectorDelimiter = "_SEL_" ) -// CheckpointBufferMinutes should always be in minutes. -// Its controls the amount of data to hold for given amount of time. -var CheckpointBufferMinutes = 3 +var CheckpointBufferMinutes = DefaultCheckpointBufferMin type AvroStruct struct { MetricName string @@ -73,7 +72,7 @@ func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel { } } - // The level does not exist, take write lock for unqiue access: + // The level does not exist, take write lock for unique access: l.lock.Lock() // While this thread waited for the write lock, another thread // could have created the child node. diff --git a/internal/memorystore/buffer.go b/internal/memorystore/buffer.go index cd2fd8fd..55be2ada 100644 --- a/internal/memorystore/buffer.go +++ b/internal/memorystore/buffer.go @@ -12,15 +12,12 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) -// Default buffer capacity. -// `buffer.data` will only ever grow up to it's capacity and a new link +// BufferCap is the default buffer capacity. +// buffer.data will only ever grow up to its capacity and a new link // in the buffer chain will be created if needed so that no copying // of data or reallocation needs to happen on writes. -const ( - BufferCap int = 512 -) +const BufferCap int = DefaultBufferCapacity -// So that we can reuse allocations var bufferPool sync.Pool = sync.Pool{ New: func() any { return &buffer{ @@ -75,7 +72,6 @@ func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) { newbuf := newBuffer(ts, b.frequency) newbuf.prev = b b.next = newbuf - b.close() b = newbuf idx = 0 } @@ -103,8 +99,6 @@ func (b *buffer) firstWrite() int64 { return b.start + (b.frequency / 2) } -func (b *buffer) close() {} - // Return all known values from `from` to `to`. Gaps of information are represented as NaN. // Simple linear interpolation is done between the two neighboring cells if possible. // If values at the start or end are missing, instead of NaN values, the second and thrid @@ -139,8 +133,6 @@ func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int6 data[i] += schema.NaN } else if t < b.start { data[i] += schema.NaN - // } else if b.data[idx].IsNaN() { - // data[i] += interpolate(idx, b.data) } else { data[i] += b.data[idx] } diff --git a/internal/memorystore/checkpoint.go b/internal/memorystore/checkpoint.go index e19cbf76..c676977c 100644 --- a/internal/memorystore/checkpoint.go +++ b/internal/memorystore/checkpoint.go @@ -28,15 +28,10 @@ import ( "github.com/linkedin/goavro/v2" ) -// File operation constants const ( - // CheckpointFilePerms defines default permissions for checkpoint files CheckpointFilePerms = 0o644 - // CheckpointDirPerms defines default permissions for checkpoint directories - CheckpointDirPerms = 0o755 - // GCTriggerInterval determines how often GC is forced during checkpoint loading - // GC is triggered every GCTriggerInterval*NumWorkers loaded hosts - GCTriggerInterval = 100 + CheckpointDirPerms = 0o755 + GCTriggerInterval = DefaultGCTriggerInterval ) // Whenever changed, update MarshalJSON as well! @@ -71,17 +66,14 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(d) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", lastCheckpoint.Format(time.RFC3339)) now := time.Now() n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, @@ -98,33 +90,23 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { } else { go func() { defer wg.Done() - d, _ := time.ParseDuration("1m") select { case <-ctx.Done(): return case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute): - // This is the first tick untill we collect the data for given minutes. GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) - } - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + ticker := time.NewTicker(DefaultAvroCheckpointInterval) + defer ticker.Stop() for { select { case <-ctx.Done(): return - case <-ticks: - // Regular ticks of 1 minute to write data. + case <-ticker.C: GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) } } }() @@ -329,7 +311,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) ( lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics)) nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension) if err != nil { - cclog.Fatalf("[METRICSTORE]> error while loading checkpoints: %s", err.Error()) + cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error()) atomic.AddInt32(&errs, 1) } atomic.AddInt32(&n, int32(nn)) @@ -506,8 +488,8 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error { for key, floatArray := range metricsData { metricName := ReplaceKey(key) - if strings.Contains(metricName, Delimiter) { - subString := strings.Split(metricName, Delimiter) + if strings.Contains(metricName, SelectorDelimiter) { + subString := strings.Split(metricName, SelectorDelimiter) lvl := l @@ -557,12 +539,10 @@ func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schem next: nil, archived: true, } - b.close() minfo, ok := m.Metrics[metricName] if !ok { return nil - // return errors.New("Unkown metric: " + name) } prev := l.metrics[minfo.offset] @@ -616,17 +596,15 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error { b := &buffer{ frequency: metric.Frequency, start: metric.Start, - data: metric.Data[0:n:n], // Space is wasted here :( + data: metric.Data[0:n:n], prev: nil, next: nil, archived: true, } - b.close() minfo, ok := m.Metrics[name] if !ok { continue - // return errors.New("Unkown metric: " + name) } prev := l.metrics[minfo.offset] diff --git a/internal/memorystore/config.go b/internal/memorystore/config.go index 8196ed69..fbd62341 100644 --- a/internal/memorystore/config.go +++ b/internal/memorystore/config.go @@ -7,6 +7,16 @@ package memorystore import ( "fmt" + "time" +) + +const ( + DefaultMaxWorkers = 10 + DefaultBufferCapacity = 512 + DefaultGCTriggerInterval = 100 + DefaultAvroWorkers = 4 + DefaultCheckpointBufferMin = 3 + DefaultAvroCheckpointInterval = time.Minute ) var InternalCCMSFlag bool = false @@ -14,7 +24,7 @@ var InternalCCMSFlag bool = false type MetricStoreConfig struct { // Number of concurrent workers for checkpoint and archive operations. // If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10) - NumWorkers int `json:"num-workers"` + NumWorkers int `json:"num-workers"` Checkpoints struct { FileFormat string `json:"file-format"` Interval string `json:"interval"` @@ -31,20 +41,6 @@ type MetricStoreConfig struct { RootDir string `json:"directory"` DeleteInstead bool `json:"delete-instead"` } `json:"archive"` - Nats []*NatsConfig `json:"nats"` -} - -type NatsConfig struct { - // Address of the nats server - Address string `json:"address"` - - // Username/Password, optional - Username string `json:"username"` - Password string `json:"password"` - - // Creds file path - Credsfilepath string `json:"creds-file-path"` - Subscriptions []struct { // Channel name SubscribeTo string `json:"subscribe-to"` diff --git a/internal/memorystore/level.go b/internal/memorystore/level.go index aaa12103..f3b3d3f5 100644 --- a/internal/memorystore/level.go +++ b/internal/memorystore/level.go @@ -46,7 +46,7 @@ func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level { } } - // The level does not exist, take write lock for unqiue access: + // The level does not exist, take write lock for unique access: l.lock.Lock() // While this thread waited for the write lock, another thread // could have created the child node. diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index 2bbd7eeb..87d3b9e9 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -11,113 +11,31 @@ import ( "sync" "time" + "github.com/ClusterCockpit/cc-backend/pkg/nats" cclog "github.com/ClusterCockpit/cc-lib/ccLogger" "github.com/ClusterCockpit/cc-lib/schema" "github.com/influxdata/line-protocol/v2/lineprotocol" - "github.com/nats-io/nats.go" ) -// Each connection is handled in it's own goroutine. This is a blocking function. -// func ReceiveRaw(ctx context.Context, -// listener net.Listener, -// handleLine func(*lineprotocol.Decoder, string) error, -// ) error { -// var wg sync.WaitGroup - -// wg.Add(1) -// go func() { -// defer wg.Done() -// <-ctx.Done() -// if err := listener.Close(); err != nil { -// log.Printf("listener.Close(): %s", err.Error()) -// } -// }() - -// for { -// conn, err := listener.Accept() -// if err != nil { -// if errors.Is(err, net.ErrClosed) { -// break -// } - -// log.Printf("listener.Accept(): %s", err.Error()) -// } - -// wg.Add(2) -// go func() { -// defer wg.Done() -// defer conn.Close() - -// dec := lineprotocol.NewDecoder(conn) -// connctx, cancel := context.WithCancel(context.Background()) -// defer cancel() -// go func() { -// defer wg.Done() -// select { -// case <-connctx.Done(): -// conn.Close() -// case <-ctx.Done(): -// conn.Close() -// } -// }() - -// if err := handleLine(dec, "default"); err != nil { -// if errors.Is(err, net.ErrClosed) { -// return -// } - -// log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error()) -// errmsg := make([]byte, 128) -// errmsg = append(errmsg, `error: `...) -// errmsg = append(errmsg, err.Error()...) -// errmsg = append(errmsg, '\n') -// conn.Write(errmsg) -// } -// }() -// } - -// wg.Wait() -// return nil -// } - -// ReceiveNats connects to a nats server and subscribes to "updates". This is a -// blocking function. handleLine will be called for each line recieved via -// nats. Send `true` through the done channel for gracefull termination. -func ReceiveNats(conf *(NatsConfig), - ms *MemoryStore, +func ReceiveNats(ms *MemoryStore, workers int, ctx context.Context, ) error { - var opts []nats.Option - if conf.Username != "" && conf.Password != "" { - opts = append(opts, nats.UserInfo(conf.Username, conf.Password)) - } - - if conf.Credsfilepath != "" { - opts = append(opts, nats.UserCredentials(conf.Credsfilepath)) - } - - nc, err := nats.Connect(conf.Address, opts...) - if err != nil { - return err - } - defer nc.Close() + nc := nats.GetClient() var wg sync.WaitGroup - var subs []*nats.Subscription - msgs := make(chan *nats.Msg, workers*2) + msgs := make(chan []byte, workers*2) - for _, sc := range conf.Subscriptions { + for _, sc := range Keys.Subscriptions { clusterTag := sc.ClusterTag - var sub *nats.Subscription if workers > 1 { wg.Add(workers) for range workers { go func() { for m := range msgs { - dec := lineprotocol.NewDecoderWithBytes(m.Data) + dec := lineprotocol.NewDecoderWithBytes(m) if err := DecodeLine(dec, ms, clusterTag); err != nil { cclog.Errorf("error: %s", err.Error()) } @@ -127,37 +45,24 @@ func ReceiveNats(conf *(NatsConfig), }() } - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - msgs <- m + nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) { + msgs <- data }) } else { - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - dec := lineprotocol.NewDecoderWithBytes(m.Data) + nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) { + dec := lineprotocol.NewDecoderWithBytes(data) if err := DecodeLine(dec, ms, clusterTag); err != nil { cclog.Errorf("error: %s", err.Error()) } }) } - - if err != nil { - return err - } - cclog.Infof("NATS subscription to '%s' on '%s' established", sc.SubscribeTo, conf.Address) - subs = append(subs, sub) + cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo) } <-ctx.Done() - for _, sub := range subs { - err = sub.Unsubscribe() - if err != nil { - cclog.Errorf("NATS unsubscribe failed: %s", err.Error()) - } - } close(msgs) wg.Wait() - nc.Close() - cclog.Print("NATS connection closed") return nil } @@ -266,8 +171,6 @@ func DecodeLine(dec *lineprotocol.Decoder, case "stype-id": subTypeBuf = append(subTypeBuf, val...) default: - // Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need) - // return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val)) } } diff --git a/internal/memorystore/memorystore.go b/internal/memorystore/memorystore.go index 3e372f34..259a86ed 100644 --- a/internal/memorystore/memorystore.go +++ b/internal/memorystore/memorystore.go @@ -44,8 +44,6 @@ var ( shutdownFunc context.CancelFunc ) - - type Metric struct { Name string Value schema.Float @@ -71,8 +69,7 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { // Set NumWorkers from config or use default if Keys.NumWorkers <= 0 { - maxWorkers := 10 - Keys.NumWorkers = min(runtime.NumCPU()/2+1, maxWorkers) + Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers) } cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers) @@ -144,20 +141,9 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) { // Store the shutdown function for later use by Shutdown() shutdownFunc = shutdown - if Keys.Nats != nil { - for _, natsConf := range Keys.Nats { - // TODO: When multiple nats configs share a URL, do a single connect. - wg.Add(1) - nc := natsConf - go func() { - // err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx) - err := ReceiveNats(nc, ms, 1, ctx) - if err != nil { - cclog.Fatal(err) - } - wg.Done() - }() - } + err = ReceiveNats(ms, 1, ctx) + if err != nil { + cclog.Fatal(err) } } @@ -244,18 +230,18 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) { return } - ticks := func() <-chan time.Time { - d := d / 2 - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() + tickInterval := d / 2 + if tickInterval <= 0 { + return + } + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-ticks: + case <-ticker.C: t := time.Now().Add(-d) cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) freed, err := ms.Free(nil, t.Unix()) @@ -332,7 +318,7 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso minfo, ok := m.Metrics[metric] if !ok { - return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: " + metric) + return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric) } n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1) diff --git a/internal/memorystore/stats.go b/internal/memorystore/stats.go index 91b1f2cc..b2cb539a 100644 --- a/internal/memorystore/stats.go +++ b/internal/memorystore/stats.go @@ -77,7 +77,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6 minfo, ok := m.Metrics[metric] if !ok { - return nil, 0, 0, errors.New("unkown metric: " + metric) + return nil, 0, 0, errors.New("unknown metric: " + metric) } n, samples := 0, 0 From 102109388b8ae68c216c28702190abd8c23e5304 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 16 Dec 2025 13:54:17 +0100 Subject: [PATCH 16/30] link to public dashboard in admin options, add return button do public dashboard --- web/frontend/src/Config.root.svelte | 6 ++-- web/frontend/src/DashPublic.root.svelte | 6 ++++ web/frontend/src/config.entrypoint.js | 1 + web/frontend/src/config/AdminSettings.svelte | 6 ++-- web/frontend/src/config/admin/Options.svelte | 32 ++++++++++++++++++-- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/web/frontend/src/Config.root.svelte b/web/frontend/src/Config.root.svelte index e8a2045b..171b2a08 100644 --- a/web/frontend/src/Config.root.svelte +++ b/web/frontend/src/Config.root.svelte @@ -6,7 +6,8 @@ - `isSupport Bool!`: Is currently logged in user support authority - `isApi Bool!`: Is currently logged in user api authority - `username String!`: Empty string if auth. is disabled, otherwise the username as string - - `ncontent String!`: The currently displayed message on the homescreen + - `ncontent String!`: The currently displayed message on the homescreen + - `clusters [String]`: The available clusternames --> @@ -30,7 +32,7 @@ Admin Options - + {/if} diff --git a/web/frontend/src/DashPublic.root.svelte b/web/frontend/src/DashPublic.root.svelte index dac3f9aa..676e4969 100644 --- a/web/frontend/src/DashPublic.root.svelte +++ b/web/frontend/src/DashPublic.root.svelte @@ -30,6 +30,7 @@ Table, Progress, Icon, + Button } from "@sveltestrap/sveltestrap"; import Roofline from "./generic/plots/Roofline.svelte"; import Pie, { colors } from "./generic/plots/Pie.svelte"; @@ -353,6 +354,11 @@ }} /> + + + {#if $statusQuery.fetching || $statesTimed.fetching} diff --git a/web/frontend/src/config.entrypoint.js b/web/frontend/src/config.entrypoint.js index f9d8e45a..5d11eedd 100644 --- a/web/frontend/src/config.entrypoint.js +++ b/web/frontend/src/config.entrypoint.js @@ -10,6 +10,7 @@ mount(Config, { isApi: isApi, username: username, ncontent: ncontent, + clusters: hClusters.map((c) => c.name) // Defined in Header Template }, context: new Map([ ['cc-config', clusterCockpitConfig], diff --git a/web/frontend/src/config/AdminSettings.svelte b/web/frontend/src/config/AdminSettings.svelte index 1a426b8c..79072e30 100644 --- a/web/frontend/src/config/AdminSettings.svelte +++ b/web/frontend/src/config/AdminSettings.svelte @@ -3,6 +3,7 @@ Properties: - `ncontent String`: The homepage notice content + - `clusters [String]`: The available clusternames --> -{#if data && collectData[0].length > 0} +{#if data && collectData.length > 0}
diff --git a/web/frontend/src/status/DashInternal.svelte b/web/frontend/src/status/DashInternal.svelte index c6abf068..47a841a3 100644 --- a/web/frontend/src/status/DashInternal.svelte +++ b/web/frontend/src/status/DashInternal.svelte @@ -487,45 +487,51 @@ - - -
-

- Top Projects: Jobs -

- tp['totalJobs'], - )} - entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} - /> -
- - -
- - - - - - {#each $topJobsQuery.data.jobsStatistics as tp, i} - - - - + {#if topJobsQuery?.data?.jobsStatistics?.length > 0} + + +
+

+ Top Projects: Jobs +

+ tp['totalJobs'], + )} + entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)} + /> +
+ + +
ProjectJobs
- {scrambleNames ? scramble(tp.id) : tp.id} - - {tp['totalJobs']}
+ + + + - {/each} -
ProjectJobs
- - + {#each $topJobsQuery.data.jobsStatistics as tp, i} + + + + {scrambleNames ? scramble(tp.id) : tp.id} + + + {tp['totalJobs']} + + {/each} + + + + {:else} + Cannot render job status: No state data returned for Pie Chart + {/if} From 43e5fd113108d2262b4ba44be86135dd6a08180f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 05:44:49 +0100 Subject: [PATCH 18/30] Add NATS API backend --- internal/api/nats.go | 232 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 internal/api/nats.go diff --git a/internal/api/nats.go b/internal/api/nats.go new file mode 100644 index 00000000..e02e424f --- /dev/null +++ b/internal/api/nats.go @@ -0,0 +1,232 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-backend. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package api + +import ( + "bytes" + "database/sql" + "encoding/json" + "sync" + "time" + + "github.com/ClusterCockpit/cc-backend/internal/archiver" + "github.com/ClusterCockpit/cc-backend/internal/importer" + "github.com/ClusterCockpit/cc-backend/internal/repository" + "github.com/ClusterCockpit/cc-backend/pkg/nats" + cclog "github.com/ClusterCockpit/cc-lib/ccLogger" + "github.com/ClusterCockpit/cc-lib/schema" +) + +// NATS subject constants for Job and Node APIs. +const ( + SubjectJobStart = "cc.job.start" + SubjectJobStop = "cc.job.stop" + SubjectNodeState = "cc.node.state" +) + +// NatsAPI provides NATS subscription-based handlers for Job and Node operations. +// It mirrors the functionality of the REST API but uses NATS messaging. +type NatsAPI struct { + JobRepository *repository.JobRepository + // RepositoryMutex protects job creation operations from race conditions + // when checking for duplicate jobs during startJob calls. + RepositoryMutex sync.Mutex +} + +// NewNatsAPI creates a new NatsAPI instance with default dependencies. +func NewNatsAPI() *NatsAPI { + return &NatsAPI{ + JobRepository: repository.GetJobRepository(), + } +} + +// StartSubscriptions registers all NATS subscriptions for Job and Node APIs. +// Returns an error if the NATS client is not available or subscription fails. +func (api *NatsAPI) StartSubscriptions() error { + client := nats.GetClient() + if client == nil { + cclog.Warn("NATS client not available, skipping API subscriptions") + return nil + } + + if err := client.Subscribe(SubjectJobStart, api.handleStartJob); err != nil { + return err + } + + if err := client.Subscribe(SubjectJobStop, api.handleStopJob); err != nil { + return err + } + + if err := client.Subscribe(SubjectNodeState, api.handleNodeState); err != nil { + return err + } + + cclog.Info("NATS API subscriptions started") + return nil +} + +// handleStartJob processes job start messages received via NATS. +// Expected JSON payload follows the schema.Job structure. +func (api *NatsAPI) handleStartJob(subject string, data []byte) { + req := schema.Job{ + Shared: "none", + MonitoringStatus: schema.MonitoringStatusRunningOrArchiving, + } + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + cclog.Debugf("NATS %s: %s", subject, req.GoString()) + req.State = schema.JobStateRunning + + if err := importer.SanityChecks(&req); err != nil { + cclog.Errorf("NATS %s: sanity check failed: %v", subject, err) + return + } + + var unlockOnce sync.Once + api.RepositoryMutex.Lock() + defer unlockOnce.Do(api.RepositoryMutex.Unlock) + + jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil) + if err != nil && err != sql.ErrNoRows { + cclog.Errorf("NATS %s: checking for duplicate failed: %v", subject, err) + return + } + if err == nil { + for _, job := range jobs { + if (req.StartTime - job.StartTime) < secondsPerDay { + cclog.Errorf("NATS %s: job with jobId %d, cluster %s already exists (dbid: %d)", + subject, req.JobID, req.Cluster, job.ID) + return + } + } + } + + id, err := api.JobRepository.Start(&req) + if err != nil { + cclog.Errorf("NATS %s: insert into database failed: %v", subject, err) + return + } + unlockOnce.Do(api.RepositoryMutex.Unlock) + + for _, tag := range req.Tags { + if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil { + cclog.Errorf("NATS %s: adding tag to new job %d failed: %v", subject, id, err) + return + } + } + + cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", + id, req.Cluster, req.JobID, req.User, req.StartTime) +} + +// handleStopJob processes job stop messages received via NATS. +// Expected JSON payload follows the StopJobAPIRequest structure. +func (api *NatsAPI) handleStopJob(subject string, data []byte) { + var req StopJobAPIRequest + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + if req.JobID == nil { + cclog.Errorf("NATS %s: the field 'jobId' is required", subject) + return + } + + job, err := api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime) + if err != nil { + cachedJob, cachedErr := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime) + if cachedErr != nil { + cclog.Errorf("NATS %s: finding job failed: %v (cached lookup also failed: %v)", + subject, err, cachedErr) + return + } + job = cachedJob + } + + if job.State != schema.JobStateRunning { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: job has already been stopped (state is: %s)", + subject, job.JobID, job.ID, job.Cluster, job.State) + return + } + + if job.StartTime > req.StopTime { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d", + subject, job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime) + return + } + + if req.State != "" && !req.State.Valid() { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: invalid job state: %#v", + subject, job.JobID, job.ID, job.Cluster, req.State) + return + } else if req.State == "" { + req.State = schema.JobStateCompleted + } + + job.Duration = int32(req.StopTime - job.StartTime) + job.State = req.State + api.JobRepository.Mutex.Lock() + defer api.JobRepository.Mutex.Unlock() + + if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { + cclog.Errorf("NATS %s: jobId %d (id %d) on %s: marking job as '%s' failed: %v", + subject, job.JobID, job.ID, job.Cluster, job.State, err) + return + } + } + + cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s", + job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State) + + if job.MonitoringStatus == schema.MonitoringStatusDisabled { + return + } + + archiver.TriggerArchiving(job) +} + +// handleNodeState processes node state update messages received via NATS. +// Expected JSON payload follows the UpdateNodeStatesRequest structure. +func (api *NatsAPI) handleNodeState(subject string, data []byte) { + var req UpdateNodeStatesRequest + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + cclog.Errorf("NATS %s: parsing request failed: %v", subject, err) + return + } + + repo := repository.GetNodeRepository() + + for _, node := range req.Nodes { + state := determineState(node.States) + nodeState := schema.NodeStateDB{ + TimeStamp: time.Now().Unix(), + NodeState: state, + CpusAllocated: node.CpusAllocated, + MemoryAllocated: node.MemoryAllocated, + GpusAllocated: node.GpusAllocated, + HealthState: schema.MonitoringStateFull, + JobsRunning: node.JobsRunning, + } + + repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState) + } + + cclog.Debugf("NATS %s: updated %d node states for cluster %s", subject, len(req.Nodes), req.Cluster) +} From d30c6ef3bf61183d3abd849cd91ee0d76c9cd1b6 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 06:08:09 +0100 Subject: [PATCH 19/30] Make NATS API subjects configurable --- internal/api/nats.go | 33 ++++++++++++++++----------------- internal/config/config.go | 8 ++++++++ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/internal/api/nats.go b/internal/api/nats.go index e02e424f..1bfe9051 100644 --- a/internal/api/nats.go +++ b/internal/api/nats.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/internal/archiver" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/nats" @@ -20,13 +21,6 @@ import ( "github.com/ClusterCockpit/cc-lib/schema" ) -// NATS subject constants for Job and Node APIs. -const ( - SubjectJobStart = "cc.job.start" - SubjectJobStop = "cc.job.stop" - SubjectNodeState = "cc.node.state" -) - // NatsAPI provides NATS subscription-based handlers for Job and Node operations. // It mirrors the functionality of the REST API but uses NATS messaging. type NatsAPI struct { @@ -52,19 +46,24 @@ func (api *NatsAPI) StartSubscriptions() error { return nil } - if err := client.Subscribe(SubjectJobStart, api.handleStartJob); err != nil { - return err - } + if config.Keys.APISubjects != nil { - if err := client.Subscribe(SubjectJobStop, api.handleStopJob); err != nil { - return err - } + s := config.Keys.APISubjects - if err := client.Subscribe(SubjectNodeState, api.handleNodeState); err != nil { - return err - } + if err := client.Subscribe(s.SubjectJobStart, api.handleStartJob); err != nil { + return err + } - cclog.Info("NATS API subscriptions started") + if err := client.Subscribe(s.SubjectJobStop, api.handleStopJob); err != nil { + return err + } + + if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil { + return err + } + + cclog.Info("NATS API subscriptions started") + } return nil } diff --git a/internal/config/config.go b/internal/config/config.go index 69a44440..25ca27eb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,8 @@ type ProgramConfig struct { // Addresses from which secured admin API endpoints can be reached, can be wildcard "*" APIAllowedIPs []string `json:"apiAllowedIPs"` + APISubjects *NATSConfig `json:"apiSubjects"` + // Drop root permissions once .env was read and the port was taken. User string `json:"user"` Group string `json:"group"` @@ -87,6 +89,12 @@ type ResampleConfig struct { Trigger int `json:"trigger"` } +type NATSConfig struct { + SubjectJobStart string `json:"subjectJobStart"` + SubjectJobStop string `json:"subjectJobStop"` + SubjectNodeState string `json:"subjectNodeState"` +} + type IntRange struct { From int `json:"from"` To int `json:"to"` From 88dc5036b3d8b64deed2c2d7161708ccef4599c0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 06:32:53 +0100 Subject: [PATCH 20/30] Make import function interuptible and replace countJobs with external call to fd --- tools/archive-manager/main.go | 110 ++++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index ae0ae9e6..940c92d4 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -5,13 +5,18 @@ package main import ( + "context" "encoding/json" "flag" "fmt" "os" + "os/exec" + "os/signal" + "strconv" "strings" "sync" "sync/atomic" + "syscall" "time" "github.com/ClusterCockpit/cc-backend/internal/config" @@ -34,13 +39,56 @@ func parseDate(in string) int64 { return 0 } -// countJobs counts the total number of jobs in the source archive. -func countJobs(srcBackend archive.ArchiveBackend) int { - count := 0 - for range srcBackend.Iter(false) { - count++ +// countJobs counts the total number of jobs in the source archive using external fd command. +// It requires the fd binary to be available in PATH. +// The srcConfig parameter should be the JSON configuration string containing the archive path. +func countJobs(srcConfig string) (int, error) { + fdPath, err := exec.LookPath("fd") + if err != nil { + return 0, fmt.Errorf("fd binary not found in PATH: %w", err) } - return count + + var config struct { + Kind string `json:"kind"` + Path string `json:"path"` + } + if err := json.Unmarshal([]byte(srcConfig), &config); err != nil { + return 0, fmt.Errorf("failed to parse source config: %w", err) + } + + if config.Path == "" { + return 0, fmt.Errorf("no path found in source config") + } + + fdCmd := exec.Command(fdPath, "meta.json", config.Path) + wcCmd := exec.Command("wc", "-l") + + pipe, err := fdCmd.StdoutPipe() + if err != nil { + return 0, fmt.Errorf("failed to create pipe: %w", err) + } + wcCmd.Stdin = pipe + + if err := fdCmd.Start(); err != nil { + return 0, fmt.Errorf("failed to start fd command: %w", err) + } + + output, err := wcCmd.Output() + if err != nil { + return 0, fmt.Errorf("failed to run wc command: %w", err) + } + + if err := fdCmd.Wait(); err != nil { + return 0, fmt.Errorf("fd command failed: %w", err) + } + + countStr := strings.TrimSpace(string(output)) + count, err := strconv.Atoi(countStr) + if err != nil { + return 0, fmt.Errorf("failed to parse count from wc output '%s': %w", countStr, err) + } + + return count, nil } // formatDuration formats a duration as a human-readable string. @@ -137,12 +185,34 @@ func (p *progressMeter) stop() { // importArchive imports all jobs from a source archive backend to a destination archive backend. // It uses parallel processing with a worker pool to improve performance. +// The import can be interrupted by CTRL-C (SIGINT) and will terminate gracefully. // Returns the number of successfully imported jobs, failed jobs, and any error encountered. -func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, error) { +func importArchive(srcBackend, dstBackend archive.ArchiveBackend, srcConfig string) (int, int, error) { cclog.Info("Starting parallel archive import...") + cclog.Info("Press CTRL-C to interrupt (will finish current jobs before exiting)") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + var interrupted atomic.Bool + + go func() { + <-sigChan + cclog.Warn("Interrupt received, stopping import (finishing current jobs)...") + interrupted.Store(true) + cancel() + // Stop listening for further signals to allow force quit with second CTRL-C + signal.Stop(sigChan) + }() cclog.Info("Counting jobs in source archive (this may take a long time) ...") - totalJobs := countJobs(srcBackend) + totalJobs, err := countJobs(srcConfig) + if err != nil { + return 0, 0, fmt.Errorf("failed to count jobs: %w", err) + } cclog.Infof("Found %d jobs to process", totalJobs) progress := newProgressMeter(totalJobs) @@ -200,8 +270,14 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err } go func() { + defer close(jobs) + clusters := srcBackend.GetClusters() for _, clusterName := range clusters { + if ctx.Err() != nil { + return + } + clusterCfg, err := srcBackend.LoadClusterCfg(clusterName) if err != nil { cclog.Errorf("Failed to load cluster config for %s: %v", clusterName, err) @@ -216,9 +292,14 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err } for job := range srcBackend.Iter(true) { - jobs <- job + select { + case <-ctx.Done(): + // Drain remaining items from iterator to avoid resource leak + // but don't process them + return + case jobs <- job: + } } - close(jobs) }() wg.Wait() @@ -229,6 +310,13 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err finalSkipped := int(atomic.LoadInt32(&progress.skipped)) elapsed := time.Since(progress.startTime) + + if interrupted.Load() { + cclog.Warnf("Import interrupted after %s: %d jobs imported, %d skipped, %d failed", + formatDuration(elapsed), finalImported, finalSkipped, finalFailed) + return finalImported, finalFailed, fmt.Errorf("import interrupted by user") + } + cclog.Infof("Import completed in %s: %d jobs imported, %d skipped, %d failed", formatDuration(elapsed), finalImported, finalSkipped, finalFailed) @@ -284,7 +372,7 @@ func main() { cclog.Info("Destination backend initialized successfully") // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, flagSrcConfig) if err != nil { cclog.Errorf("Import completed with errors: %s", err.Error()) if failed > 0 { From 4ecc050c4cb17353c66833ab3c7e16e39d46e2e6 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 07:03:01 +0100 Subject: [PATCH 21/30] Fix deadlock if NATS is not configured --- cmd/cc-backend/main.go | 2 +- internal/memorystore/lineprotocol.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index c3e33872..f9b198df 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -271,7 +271,7 @@ func initSubsystems() error { // Initialize nats client natsConfig := ccconf.GetPackageConfig("nats") if err := nats.Init(natsConfig); err != nil { - return fmt.Errorf("initializing nats client: %w", err) + cclog.Warnf("initializing (optional) nats client: %s", err.Error()) } nats.Connect() diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index 87d3b9e9..aebdbdca 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -23,6 +23,11 @@ func ReceiveNats(ms *MemoryStore, ) error { nc := nats.GetClient() + if nc == nil { + cclog.Warn("NATS client not initialized") + return nil + } + var wg sync.WaitGroup msgs := make(chan []byte, workers*2) From 0a5e15509627940bf360e816541e63c19d8c5f0e Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 07:03:10 +0100 Subject: [PATCH 22/30] Remove debug setting --- internal/taskmanager/retentionService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/taskmanager/retentionService.go b/internal/taskmanager/retentionService.go index 2bd221b4..acd07307 100644 --- a/internal/taskmanager/retentionService.go +++ b/internal/taskmanager/retentionService.go @@ -16,7 +16,7 @@ import ( func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) { cclog.Info("Register retention delete service") - s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(14, 30, 0))), + s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))), gocron.NewTask( func() { startTime := time.Now().Unix() - int64(age*24*3600) From f4b00e9de173abfded748b683d9cd53d76088c13 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:38:00 +0100 Subject: [PATCH 23/30] Use Info instead of warn loglevel for database file missing msg --- internal/repository/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 58ab3e69..dec93a94 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -118,7 +118,7 @@ func MigrateDB(backend string, db string) error { v, dirty, err := m.Version() if err != nil { if err == migrate.ErrNilVersion { - cclog.Warn("Legacy database without version or missing database file!") + cclog.Info("Legacy database without version or missing database file!") } else { return err } From d1a78c13a4f7b99dea3e44259fa5a46549460f2c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:38:14 +0100 Subject: [PATCH 24/30] Make loglevel info default for demo --- startDemo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/startDemo.sh b/startDemo.sh index 10ba7f0c..b494e814 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -4,7 +4,7 @@ if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend -server -dev + ./cc-backend -server -dev -loglevel info else make ./cc-backend --init @@ -15,5 +15,5 @@ else rm ./job-archive-demo.tar ./cc-backend -dev -init-db -add-user demo:admin,api:demo - ./cc-backend -server -dev + ./cc-backend -server -dev -loglevel info fi From 79a2ca8ae8cb3c401c4ef38b7807d4b6d1871c2b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 08:44:37 +0100 Subject: [PATCH 25/30] Adapt unit test to new API --- tools/archive-manager/import_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/archive-manager/import_test.go b/tools/archive-manager/import_test.go index 02288285..b1032118 100644 --- a/tools/archive-manager/import_test.go +++ b/tools/archive-manager/import_test.go @@ -48,7 +48,7 @@ func TestImportFileToSqlite(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -111,13 +111,13 @@ func TestImportFileToFile(t *testing.T) { } // Create destination archive directory - if err := os.MkdirAll(dstArchive, 0755); err != nil { + if err := os.MkdirAll(dstArchive, 0o755); err != nil { t.Fatalf("Failed to create destination directory: %s", err.Error()) } // Write version file versionFile := filepath.Join(dstArchive, "version.txt") - if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil { + if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil { t.Fatalf("Failed to write version file: %s", err.Error()) } @@ -136,7 +136,7 @@ func TestImportFileToFile(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -183,7 +183,7 @@ func TestImportDataIntegrity(t *testing.T) { } // Perform import - _, _, err = importArchive(srcBackend, dstBackend) + _, _, err = importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %s", err.Error()) } @@ -253,13 +253,13 @@ func TestImportEmptyArchive(t *testing.T) { dstDb := filepath.Join(tmpdir, "dst-archive.db") // Create empty source archive - if err := os.MkdirAll(srcArchive, 0755); err != nil { + if err := os.MkdirAll(srcArchive, 0o755); err != nil { t.Fatalf("Failed to create source directory: %s", err.Error()) } // Write version file versionFile := filepath.Join(srcArchive, "version.txt") - if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil { + if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil { t.Fatalf("Failed to write version file: %s", err.Error()) } @@ -277,7 +277,7 @@ func TestImportEmptyArchive(t *testing.T) { } // Perform import - imported, failed, err := importArchive(srcBackend, dstBackend) + imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import from empty archive should not fail: %s", err.Error()) } @@ -321,13 +321,13 @@ func TestImportDuplicateJobs(t *testing.T) { } // First import - imported1, _, err := importArchive(srcBackend, dstBackend) + imported1, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Fatalf("First import failed: %s", err.Error()) } // Second import (should skip all jobs) - imported2, _, err := importArchive(srcBackend, dstBackend) + imported2, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Second import failed: %s", err.Error()) } @@ -366,7 +366,7 @@ func TestImportToEmptyFileDestination(t *testing.T) { util.CopyDir(testDataPath, srcArchive) // Setup empty destination directory - os.MkdirAll(dstArchive, 0755) + os.MkdirAll(dstArchive, 0o755) // NOTE: NOT writing version.txt here! // Initialize source @@ -384,7 +384,7 @@ func TestImportToEmptyFileDestination(t *testing.T) { } // Perform import - imported, _, err := importArchive(srcBackend, dstBackend) + imported, _, err := importArchive(srcBackend, dstBackend, srcConfig) if err != nil { t.Errorf("Import failed: %v", err) } From b8fdfc30c06766651a39774fbc9685fb64b5c71f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 17 Dec 2025 10:12:49 +0100 Subject: [PATCH 26/30] Fix performance bugs in sqlite archive backend --- internal/importer/initDB.go | 14 ++++---- pkg/archive/sqliteBackend.go | 52 ++++++++++++++++++++++------ pkg/archive/sqliteBackend_test.go | 57 ++++++++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index a4789576..12f49010 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -111,18 +111,22 @@ func InitDB() error { continue } - id, err := r.TransactionAddNamed(t, + id, jobErr := r.TransactionAddNamed(t, repository.NamedJobInsert, jobMeta) - if err != nil { - cclog.Errorf("repository initDB(): %v", err) + if jobErr != nil { + cclog.Errorf("repository initDB(): %v", jobErr) errorOccured++ continue } + // Job successfully inserted, increment counter + i += 1 + for _, tag := range jobMeta.Tags { tagstr := tag.Name + ":" + tag.Type tagID, ok := tags[tagstr] if !ok { + var err error tagID, err = r.TransactionAdd(t, addTagQuery, tag.Name, tag.Type) @@ -138,10 +142,6 @@ func InitDB() error { setTagQuery, id, tagID) } - - if err == nil { - i += 1 - } } if errorOccured > 0 { diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go index 6fa188ba..0b7a22d2 100644 --- a/pkg/archive/sqliteBackend.go +++ b/pkg/archive/sqliteBackend.go @@ -61,6 +61,7 @@ CREATE TABLE IF NOT EXISTS jobs ( CREATE INDEX IF NOT EXISTS idx_jobs_cluster ON jobs(cluster); CREATE INDEX IF NOT EXISTS idx_jobs_start_time ON jobs(start_time); +CREATE INDEX IF NOT EXISTS idx_jobs_order ON jobs(cluster, start_time); CREATE INDEX IF NOT EXISTS idx_jobs_lookup ON jobs(cluster, job_id, start_time); CREATE TABLE IF NOT EXISTS clusters ( @@ -560,11 +561,15 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { go func() { defer close(ch) - rows, err := sa.db.Query("SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time") - if err != nil { - cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) + const chunkSize = 1000 + offset := 0 + + var query string + if loadMetricData { + query = "SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?" + } else { + query = "SELECT meta_json FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?" } - defer rows.Close() numWorkers := 4 jobRows := make(chan sqliteJobRow, numWorkers*2) @@ -615,13 +620,40 @@ func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer { }() } - for rows.Next() { - var row sqliteJobRow - if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { - cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) - continue + for { + rows, err := sa.db.Query(query, chunkSize, offset) + if err != nil { + cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error()) } - jobRows <- row + + rowCount := 0 + for rows.Next() { + var row sqliteJobRow + + if loadMetricData { + if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil { + cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) + continue + } + } else { + if err := rows.Scan(&row.metaBlob); err != nil { + cclog.Errorf("SqliteArchive Iter() > scan error: %v", err) + continue + } + row.dataBlob = nil + row.compressed = false + } + + jobRows <- row + rowCount++ + } + rows.Close() + + if rowCount < chunkSize { + break + } + + offset += chunkSize } close(jobRows) diff --git a/pkg/archive/sqliteBackend_test.go b/pkg/archive/sqliteBackend_test.go index 285055fc..b72b8f6c 100644 --- a/pkg/archive/sqliteBackend_test.go +++ b/pkg/archive/sqliteBackend_test.go @@ -294,7 +294,7 @@ func TestSqliteCompress(t *testing.T) { // Compress should not panic even with missing data sa.Compress([]*schema.Job{job}) - + t.Log("Compression method verified") } @@ -311,3 +311,58 @@ func TestSqliteConfigParsing(t *testing.T) { t.Errorf("expected dbPath '/tmp/test.db', got '%s'", cfg.DBPath) } } + +func TestSqliteIterChunking(t *testing.T) { + tmpfile := t.TempDir() + "/test.db" + defer os.Remove(tmpfile) + + var sa SqliteArchive + _, err := sa.Init(json.RawMessage(`{"dbPath":"` + tmpfile + `"}`)) + if err != nil { + t.Fatalf("init failed: %v", err) + } + defer sa.db.Close() + + const totalJobs = 2500 + for i := 1; i <= totalJobs; i++ { + job := &schema.Job{ + JobID: int64(i), + Cluster: "test", + StartTime: int64(i * 1000), + NumNodes: 1, + Resources: []*schema.Resource{{Hostname: "node001"}}, + } + if err := sa.StoreJobMeta(job); err != nil { + t.Fatalf("store failed: %v", err) + } + } + + t.Run("IterWithoutData", func(t *testing.T) { + count := 0 + for container := range sa.Iter(false) { + if container.Meta == nil { + t.Error("expected non-nil meta") + } + if container.Data != nil { + t.Error("expected nil data when loadMetricData is false") + } + count++ + } + if count != totalJobs { + t.Errorf("expected %d jobs, got %d", totalJobs, count) + } + }) + + t.Run("IterWithData", func(t *testing.T) { + count := 0 + for container := range sa.Iter(true) { + if container.Meta == nil { + t.Error("expected non-nil meta") + } + count++ + } + if count != totalJobs { + t.Errorf("expected %d jobs, got %d", totalJobs, count) + } + }) +} From d2f2d7895426a4a50ccd507f3716542d220c2a0b Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Wed, 17 Dec 2025 15:58:42 +0100 Subject: [PATCH 27/30] Changing JWT output to stdout and change to help text --- .gitignore | 2 +- cmd/cc-backend/cli.go | 2 +- cmd/cc-backend/main.go | 2 +- configs/config.json | 8 ++++++-- startDemo.sh | 33 +++++++++++++++++++++++++-------- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index e03d8071..db9f922b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ /var/checkpoints* migrateTimestamps.pl -test_ccms_write_api.sh +test_ccms_write_api* /web/frontend/public/build /web/frontend/node_modules diff --git a/cmd/cc-backend/cli.go b/cmd/cc-backend/cli.go index af32b643..9ee56cb2 100644 --- a/cmd/cc-backend/cli.go +++ b/cmd/cc-backend/cli.go @@ -33,6 +33,6 @@ func cliInit() { flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: ") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") - flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info , warn (default), err, crit]`") flag.Parse() } diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index f9b198df..104182f2 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -263,7 +263,7 @@ func generateJWT(authHandle *auth.Authentication, username string) error { return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err) } - cclog.Infof("JWT: Successfully generated JWT for user '%s': %s", user.Username, jwt) + fmt.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) return nil } diff --git a/configs/config.json b/configs/config.json index 5bffc969..88a9e930 100644 --- a/configs/config.json +++ b/configs/config.json @@ -9,8 +9,12 @@ "apiAllowedIPs": ["*"], "short-running-jobs-duration": 300, "resampling": { - "trigger": 30, - "resolutions": [600, 300, 120, 60] + "minimumPoints": 600, + "trigger": 180, + "resolutions": [ + 240, + 60 + ] } }, "cron": { diff --git a/startDemo.sh b/startDemo.sh index b494e814..904caa02 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -1,19 +1,36 @@ #!/bin/sh -# rm -rf var +rm -rf var if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend -server -dev -loglevel info + ./cc-backend -server -dev else make - ./cc-backend --init + wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar + tar xf job-archive-dev.tar + rm ./job-archive-dev.tar + + cp ./configs/env-template.txt .env cp ./configs/config-demo.json config.json - wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar - tar xf job-archive-demo.tar - rm ./job-archive-demo.tar + echo 3 > /home/adityauj/cc-backend/var/job-archive/version.txt + + ./cc-backend --loglevel info -migrate-db + ./cc-backend --loglevel info -dev -init-db -add-user demo:admin,api:demo + + # Generate JWT and extract only the token value + JWT=$(./cc-backend -jwt demo | grep -oP "(?<=JWT: Successfully generated JWT for user 'demo': ).*") + + # Replace the existing JWT in test_ccms_write_api.sh with the new one + if [ -n "$JWT" ]; then + sed -i "1s|^JWT=.*|JWT=\"$JWT\"|" test_ccms_write_api.sh + echo "✅ Updated JWT in test_ccms_write_api.sh" + else + echo "❌ Failed to generate JWT for demo user" + exit 1 + fi + + ./cc-backend -server -dev - ./cc-backend -dev -init-db -add-user demo:admin,api:demo - ./cc-backend -server -dev -loglevel info fi From 32e535384708caeae0ac886293360515ff8eb9d0 Mon Sep 17 00:00:00 2001 From: Aditya Ujeniya Date: Wed, 17 Dec 2025 18:14:36 +0100 Subject: [PATCH 28/30] Fix to NATS deadlock and revert demo script --- cmd/cc-backend/main.go | 2 +- configs/config-demo.json | 17 ++++++++++++++++- internal/memorystore/lineprotocol.go | 1 - startDemo.sh | 22 ++++------------------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 104182f2..6239d36c 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -263,7 +263,7 @@ func generateJWT(authHandle *auth.Authentication, username string) error { return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err) } - fmt.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) + cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt) return nil } diff --git a/configs/config-demo.json b/configs/config-demo.json index 70ca2a02..58366fb5 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -29,6 +29,11 @@ "max-age": "2000h" } }, + "nats": { + "address": "nats://0.0.0.0:4222", + "username": "root", + "password": "root" + }, "clusters": [ { "name": "fritz", @@ -86,6 +91,16 @@ "interval": "1h", "directory": "./var/archive" }, - "retention-in-memory": "48h" + "retention-in-memory": "48h", + "subscriptions": [ + { + "subscribe-to": "hpc-nats", + "cluster-tag": "fritz" + }, + { + "subscribe-to": "hpc-nats", + "cluster-tag": "alex" + } + ] } } \ No newline at end of file diff --git a/internal/memorystore/lineprotocol.go b/internal/memorystore/lineprotocol.go index aebdbdca..6404361f 100644 --- a/internal/memorystore/lineprotocol.go +++ b/internal/memorystore/lineprotocol.go @@ -64,7 +64,6 @@ func ReceiveNats(ms *MemoryStore, cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo) } - <-ctx.Done() close(msgs) wg.Wait() diff --git a/startDemo.sh b/startDemo.sh index 904caa02..e709db27 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -1,6 +1,6 @@ #!/bin/sh -rm -rf var +# rm -rf var if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' @@ -14,23 +14,9 @@ else cp ./configs/env-template.txt .env cp ./configs/config-demo.json config.json - echo 3 > /home/adityauj/cc-backend/var/job-archive/version.txt - - ./cc-backend --loglevel info -migrate-db - ./cc-backend --loglevel info -dev -init-db -add-user demo:admin,api:demo - - # Generate JWT and extract only the token value - JWT=$(./cc-backend -jwt demo | grep -oP "(?<=JWT: Successfully generated JWT for user 'demo': ).*") - - # Replace the existing JWT in test_ccms_write_api.sh with the new one - if [ -n "$JWT" ]; then - sed -i "1s|^JWT=.*|JWT=\"$JWT\"|" test_ccms_write_api.sh - echo "✅ Updated JWT in test_ccms_write_api.sh" - else - echo "❌ Failed to generate JWT for demo user" - exit 1 - fi + ./cc-backend -migrate-db + ./cc-backend -dev -init-db -add-user demo:admin,api:demo ./cc-backend -server -dev -fi +fi \ No newline at end of file From 19c8e9beb15f1fc3ff6044eedc12f9e96d36ad78 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 18 Dec 2025 10:44:58 +0100 Subject: [PATCH 29/30] move extensive NodeMetricsList handling to node repo func --- internal/graph/schema.resolvers.go | 129 ++------------------------- internal/repository/node.go | 136 +++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 123 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 02046443..9747479a 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -806,140 +806,24 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub return nil, errors.New("you need to be administrator or support staff for this query") } + nodeRepo := repository.GetNodeRepository() + nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page) + if nerr != nil { + return nil, errors.New("Could not retrieve node list required for resolving NodeMetricsList") + } + if metrics == nil { for _, mc := range archive.GetCluster(cluster).MetricConfig { metrics = append(metrics, mc.Name) } } - // Build Filters - queryFilters := make([]*model.NodeFilter, 0) - if cluster != "" { - queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}}) - } - if subCluster != "" { - queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}}) - } - if nodeFilter != "" && stateFilter != "notindb" { - queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}}) - } - if stateFilter != "all" && stateFilter != "notindb" { - var queryState schema.SchedulerState = schema.SchedulerState(stateFilter) - queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState}) - } - // if healthFilter != "all" { - // filters = append(filters, &model.NodeFilter{HealthState: &healthFilter}) - // } - - // Special Case: Disable Paging for missing nodes filter, save IPP for later - var backupItems int - if stateFilter == "notindb" { - backupItems = page.ItemsPerPage - page.ItemsPerPage = -1 - } - - // Query Nodes From DB - nodeRepo := repository.GetNodeRepository() - rawNodes, serr := nodeRepo.QueryNodes(ctx, queryFilters, page, nil) // Order not Used - if serr != nil { - cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)") - return nil, serr - } - - // Intermediate Node Result Info - nodes := make([]string, 0) - stateMap := make(map[string]string) - for _, node := range rawNodes { - nodes = append(nodes, node.Hostname) - stateMap[node.Hostname] = string(node.NodeState) - } - - // Setup Vars - var countNodes int - var cerr error - var hasNextPage bool - - // Special Case: Find Nodes not in DB node table but in metricStore only - if stateFilter == "notindb" { - // Reapply Original Paging - page.ItemsPerPage = backupItems - // Get Nodes From Topology - var topoNodes []string - if subCluster != "" { - scNodes := archive.NodeLists[cluster][subCluster] - topoNodes = scNodes.PrintList() - } else { - subClusterNodeLists := archive.NodeLists[cluster] - for _, nodeList := range subClusterNodeLists { - topoNodes = append(topoNodes, nodeList.PrintList()...) - } - } - // Compare to all nodes from cluster/subcluster in DB - var missingNodes []string - for _, scanNode := range topoNodes { - if !slices.Contains(nodes, scanNode) { - missingNodes = append(missingNodes, scanNode) - } - } - // Filter nodes by name - if nodeFilter != "" { - filteredNodesByName := []string{} - for _, missingNode := range missingNodes { - if strings.Contains(missingNode, nodeFilter) { - filteredNodesByName = append(filteredNodesByName, missingNode) - } - } - missingNodes = filteredNodesByName - } - // Sort Missing Nodes Alphanumerically - slices.Sort(missingNodes) - // Total Missing - countNodes = len(missingNodes) - // Apply paging - if countNodes > page.ItemsPerPage { - start := (page.Page - 1) * page.ItemsPerPage - end := start + page.ItemsPerPage - if end > countNodes { - end = countNodes - hasNextPage = false - } else { - hasNextPage = true - } - nodes = missingNodes[start:end] - } else { - nodes = missingNodes - } - - } else { - // DB Nodes: Count and Find Next Page - countNodes, cerr = nodeRepo.CountNodes(ctx, queryFilters) - if cerr != nil { - cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)") - return nil, cerr - } - - // Example Page 4 @ 10 IpP : Does item 41 exist? - // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists. - nextPage := &model.PageRequest{ - ItemsPerPage: 1, - Page: ((page.Page * page.ItemsPerPage) + 1), - } - nextNodes, err := nodeRepo.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used - if err != nil { - cclog.Warn("Error while querying next nodes") - return nil, err - } - hasNextPage = len(nextNodes) == 1 - } - - // Load Metric Data For Specified Nodes Only data, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx) if err != nil { cclog.Warn("error while loading node data (Resolver.NodeMetricsList") return nil, err } - // Build Result nodeMetricsList := make([]*model.NodeMetrics, 0, len(data)) for hostname, metrics := range data { host := &model.NodeMetrics{ @@ -965,7 +849,6 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub nodeMetricsList = append(nodeMetricsList, host) } - // Final Return nodeMetricsListResult := &model.NodesResultList{ Items: nodeMetricsList, TotalNodes: &countNodes, diff --git a/internal/repository/node.go b/internal/repository/node.go index 9a1f3530..3b597eda 100644 --- a/internal/repository/node.go +++ b/internal/repository/node.go @@ -10,6 +10,8 @@ import ( "database/sql" "encoding/json" "fmt" + "slices" + "strings" "sync" "time" @@ -551,6 +553,140 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model. return timedStates, nil } +func (r *NodeRepository) GetNodesForList( + ctx context.Context, + cluster string, + subCluster string, + stateFilter string, + nodeFilter string, + page *model.PageRequest, +) ([]string, map[string]string, int, bool, error) { + + // Init Return Vars + nodes := make([]string, 0) + stateMap := make(map[string]string) + countNodes := 0 + hasNextPage := false + + // Build Filters + queryFilters := make([]*model.NodeFilter, 0) + if cluster != "" { + queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}}) + } + if subCluster != "" { + queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}}) + } + if nodeFilter != "" && stateFilter != "notindb" { + queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}}) + } + if stateFilter != "all" && stateFilter != "notindb" { + var queryState schema.SchedulerState = schema.SchedulerState(stateFilter) + queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState}) + } + // if healthFilter != "all" { + // filters = append(filters, &model.NodeFilter{HealthState: &healthFilter}) + // } + + // Special Case: Disable Paging for missing nodes filter, save IPP for later + var backupItems int + if stateFilter == "notindb" { + backupItems = page.ItemsPerPage + page.ItemsPerPage = -1 + } + + // Query Nodes From DB + rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used + if serr != nil { + cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)") + return nil, nil, 0, false, serr + } + + // Intermediate Node Result Info + for _, node := range rawNodes { + if node == nil { + continue + } + nodes = append(nodes, node.Hostname) + stateMap[node.Hostname] = string(node.NodeState) + } + + // Special Case: Find Nodes not in DB node table but in metricStore only + if stateFilter == "notindb" { + // Reapply Original Paging + page.ItemsPerPage = backupItems + // Get Nodes From Topology + var topoNodes []string + if subCluster != "" { + scNodes := archive.NodeLists[cluster][subCluster] + topoNodes = scNodes.PrintList() + } else { + subClusterNodeLists := archive.NodeLists[cluster] + for _, nodeList := range subClusterNodeLists { + topoNodes = append(topoNodes, nodeList.PrintList()...) + } + } + // Compare to all nodes from cluster/subcluster in DB + var missingNodes []string + for _, scanNode := range topoNodes { + if !slices.Contains(nodes, scanNode) { + missingNodes = append(missingNodes, scanNode) + } + } + // Filter nodes by name + if nodeFilter != "" { + filteredNodesByName := []string{} + for _, missingNode := range missingNodes { + if strings.Contains(missingNode, nodeFilter) { + filteredNodesByName = append(filteredNodesByName, missingNode) + } + } + missingNodes = filteredNodesByName + } + // Sort Missing Nodes Alphanumerically + slices.Sort(missingNodes) + // Total Missing + countNodes = len(missingNodes) + // Apply paging + if countNodes > page.ItemsPerPage { + start := (page.Page - 1) * page.ItemsPerPage + end := start + page.ItemsPerPage + if end > countNodes { + end = countNodes + hasNextPage = false + } else { + hasNextPage = true + } + nodes = missingNodes[start:end] + } else { + nodes = missingNodes + } + + } else { + // DB Nodes: Count and Find Next Page + var cerr error + countNodes, cerr = r.CountNodes(ctx, queryFilters) + if cerr != nil { + cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)") + return nil, nil, 0, false, cerr + } + + // Example Page 4 @ 10 IpP : Does item 41 exist? + // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists. + nextPage := &model.PageRequest{ + ItemsPerPage: 1, + Page: ((page.Page * page.ItemsPerPage) + 1), + } + nextNodes, err := r.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used + if err != nil { + cclog.Warn("Error while querying next nodes") + return nil, nil, 0, false, err + } + hasNextPage = len(nextNodes) == 1 + } + + return nodes, stateMap, countNodes, hasNextPage, nil +} + func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { user := GetUserFromContext(ctx) return AccessCheckWithUser(user, query) From e707fd08936ac683936e45af0155091e05ee7ced Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 18 Dec 2025 11:26:05 +0100 Subject: [PATCH 30/30] Provide fallback in archive manager in case fd is not available --- tools/archive-manager/main.go | 70 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/tools/archive-manager/main.go b/tools/archive-manager/main.go index 940c92d4..4972fe96 100644 --- a/tools/archive-manager/main.go +++ b/tools/archive-manager/main.go @@ -9,9 +9,11 @@ import ( "encoding/json" "flag" "fmt" + "io/fs" "os" "os/exec" "os/signal" + "path/filepath" "strconv" "strings" "sync" @@ -39,28 +41,47 @@ func parseDate(in string) int64 { return 0 } -// countJobs counts the total number of jobs in the source archive using external fd command. -// It requires the fd binary to be available in PATH. -// The srcConfig parameter should be the JSON configuration string containing the archive path. -func countJobs(srcConfig string) (int, error) { - fdPath, err := exec.LookPath("fd") - if err != nil { - return 0, fmt.Errorf("fd binary not found in PATH: %w", err) - } - +// parseArchivePath extracts the path from the source config JSON. +func parseArchivePath(srcConfig string) (string, error) { var config struct { Kind string `json:"kind"` Path string `json:"path"` } if err := json.Unmarshal([]byte(srcConfig), &config); err != nil { - return 0, fmt.Errorf("failed to parse source config: %w", err) + return "", fmt.Errorf("failed to parse source config: %w", err) } if config.Path == "" { - return 0, fmt.Errorf("no path found in source config") + return "", fmt.Errorf("no path found in source config") } - fdCmd := exec.Command(fdPath, "meta.json", config.Path) + return config.Path, nil +} + +// countJobsNative counts jobs using native Go filepath.WalkDir. +// This is used as a fallback when fd/fdfind is not available. +func countJobsNative(archivePath string) (int, error) { + count := 0 + err := filepath.WalkDir(archivePath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // Skip directories we can't access + } + if !d.IsDir() && d.Name() == "meta.json" { + count++ + } + return nil + }) + + if err != nil { + return 0, fmt.Errorf("failed to walk directory: %w", err) + } + + return count, nil +} + +// countJobsWithFd counts jobs using the external fd command. +func countJobsWithFd(fdPath, archivePath string) (int, error) { + fdCmd := exec.Command(fdPath, "meta.json", archivePath) wcCmd := exec.Command("wc", "-l") pipe, err := fdCmd.StdoutPipe() @@ -91,6 +112,31 @@ func countJobs(srcConfig string) (int, error) { return count, nil } +// countJobs counts the total number of jobs in the source archive. +// It tries to use external fd/fdfind command for speed, falling back to +// native Go filepath.WalkDir if neither is available. +// The srcConfig parameter should be the JSON configuration string containing the archive path. +func countJobs(srcConfig string) (int, error) { + archivePath, err := parseArchivePath(srcConfig) + if err != nil { + return 0, err + } + + // Try fd first (common name) + if fdPath, err := exec.LookPath("fd"); err == nil { + return countJobsWithFd(fdPath, archivePath) + } + + // Try fdfind (Debian/Ubuntu package name) + if fdPath, err := exec.LookPath("fdfind"); err == nil { + return countJobsWithFd(fdPath, archivePath) + } + + // Fall back to native Go implementation + cclog.Debug("fd/fdfind not found, using native Go file walker") + return countJobsNative(archivePath) +} + // formatDuration formats a duration as a human-readable string. func formatDuration(d time.Duration) string { if d < time.Minute {