ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2026-01-15 17:21:46 +01:00
Code Issues Packages Releases Activity

Update dependencies. Rebuild graphql and swagger

Browse Source
This commit is contained in:
Jan Eitzinger
2026-01-15 08:32:06 +01:00
parent 8f0bb907ff
commit e1efc68476
24 changed files with 3321 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/api/cluster.go
View File

@@ -27,7 +27,7 @@ type GetClustersAPIResponse struct {
// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter.
// @produce json
// @param cluster query string false "Job Cluster"
// @success 200 {object} api.GetClustersApiResponse "Array of clusters"
// @success 200 {object} api.GetClustersAPIResponse "Array of clusters"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"

1178
internal/api/docs.go
View File

File diff suppressed because it is too large Load Diff

27
internal/api/job.go
View File

@@ -104,7 +104,7 @@ type JobMetricWithName struct {
// @param items-per-page query int false "Items per page (Default: 25)"
// @param page query int false "Page Number (Default: 1)"
// @param with-metadata query bool false "Include metadata (e.g. jobScript) in response"
// @success 200 {object} api.GetJobsApiResponse "Job array and page info"
// @success 200 {object} api.GetJobsAPIResponse "Job array and page info"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -232,7 +232,7 @@ func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
// @produce json
// @param id path int true "Database ID of Job"
// @param all-metrics query bool false "Include all available metrics"
// @success 200 {object} api.GetJobApiResponse "Job resource"
// @success 200 {object} api.GetJobAPIResponse "Job resource"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -324,8 +324,8 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request)
// @accept json
// @produce json
// @param id path int true "Database ID of Job"
// @param request body api.GetJobApiRequest true "Array of metric names"
// @success 200 {object} api.GetJobApiResponse "Job resource"
// @param request body api.GetJobAPIRequest true "Array of metric names"
// @success 200 {object} api.GetJobAPIResponse "Job resource"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -478,7 +478,7 @@ func (api *RestAPI) editMeta(rw http.ResponseWriter, r *http.Request) {
// @accept json
// @produce json
// @param id path int true "Job Database ID"
// @param request body api.TagJobApiRequest true "Array of tag-objects to add"
// @param request body api.TagJobAPIRequest true "Array of tag-objects to add"
// @success 200 {object} schema.Job "Updated job resource"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
@@ -542,7 +542,7 @@ func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) {
// @accept json
// @produce json
// @param id path int true "Job Database ID"
// @param request body api.TagJobApiRequest true "Array of tag-objects to remove"
// @param request body api.TagJobAPIRequest true "Array of tag-objects to remove"
// @success 200 {object} schema.Job "Updated job resource"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
@@ -606,7 +606,7 @@ func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) {
// @description Tag wills be removed from respective archive files.
// @accept json
// @produce plain
// @param request body api.TagJobApiRequest true "Array of tag-objects to remove"
// @param request body api.TagJobAPIRequest true "Array of tag-objects to remove"
// @success 200 {string} string "Success Response"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
@@ -650,7 +650,7 @@ func (api *RestAPI) removeTags(rw http.ResponseWriter, r *http.Request) {
// @accept json
// @produce json
// @param request body schema.Job true "Job to add"
// @success 201 {object} api.DefaultApiResponse "Job added successfully"
// @success 201 {object} api.DefaultAPIResponse "Job added successfully"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -728,7 +728,7 @@ func (api *RestAPI) startJob(rw http.ResponseWriter, r *http.Request) {
// @description Job to stop is specified by request body. All fields are required in this case.
// @description Returns full job resource information according to 'Job' scheme.
// @produce json
// @param request body api.StopJobApiRequest true "All fields required"
// @param request body api.StopJobAPIRequest true "All fields required"
// @success 200 {object} schema.Job "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
@@ -754,7 +754,6 @@ func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
return
}
// cclog.Printf("loading db job for stopJobByRequest... : stopJobApiRequest=%v", req)
job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
if err != nil {
// Try cached jobs if not found in main repository
@@ -776,7 +775,7 @@ func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
// @description Job to remove is specified by database ID. This will not remove the job from the job archive.
// @produce json
// @param id path int true "Database ID of Job"
// @success 200 {object} api.DefaultApiResponse "Success message"
// @success 200 {object} api.DefaultAPIResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -820,8 +819,8 @@ func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) {
// @description Job to delete is specified by request body. All fields are required in this case.
// @accept json
// @produce json
// @param request body api.DeleteJobApiRequest true "All fields required"
// @success 200 {object} api.DefaultApiResponse "Success message"
// @param request body api.DeleteJobAPIRequest true "All fields required"
// @success 200 {object} api.DefaultAPIResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
@@ -873,7 +872,7 @@ func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request)
// @description Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.
// @produce json
// @param ts path int true "Unix epoch timestamp"
// @success 200 {object} api.DefaultApiResponse "Success message"
// @success 200 {object} api.DefaultAPIResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"

2
internal/api/node.go
View File

@@ -47,7 +47,7 @@ func determineState(states []string) schema.SchedulerState {
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce json
// @param request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success 200 {object} api.DefaultApiResponse "Success message"
// @success 200 {object} api.DefaultAPIResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"

2
internal/api/user.go
View File

@@ -31,7 +31,7 @@ type APIReturnedUser struct {
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce json
// @param not-just-user query bool true "If returned list should contain all users or only users with additional special roles"
// @success 200 {array} api.ApiReturnedUser "List of users returned successfully"
// @success 200 {array} api.APIReturnedUser "List of users returned successfully"
// @failure 400 {string} string "Bad Request"
// @failure 401 {string} string "Unauthorized"
// @failure 403 {string} string "Forbidden"

20
internal/graph/generated/generated.go
View File

@@ -10815,7 +10815,7 @@ func (ec *executionContext) _SubCluster_metricConfig(ctx context.Context, field
return obj.MetricConfig, nil
},
nil,
ec.marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ,
ec.marshalNMetricConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ,
true,
true,
)
@@ -18466,11 +18466,7 @@ func (ec *executionContext) marshalNJobsStatistics2ᚖgithubᚗcomᚋClusterCock
return ec._JobsStatistics(ctx, sel, v)
}
func (ec *executionContext) marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx context.Context, sel ast.SelectionSet, v schema.MetricConfig) graphql.Marshaler {
return ec._MetricConfig(ctx, sel, &v)
}
func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.MetricConfig) graphql.Marshaler {
func (ec *executionContext) marshalNMetricConfig2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfigᚄ(ctx context.Context, sel ast.SelectionSet, v []*schema.MetricConfig) graphql.Marshaler {
ret := make(graphql.Array, len(v))
var wg sync.WaitGroup
isLen1 := len(v) == 1
@@ -18494,7 +18490,7 @@ func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpi
if !isLen1 {
defer wg.Done()
}
ret[i] = ec.marshalNMetricConfig2githubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx, sel, v[i])
ret[i] = ec.marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx, sel, v[i])
}
if isLen1 {
f(i)
@@ -18514,6 +18510,16 @@ func (ec *executionContext) marshalNMetricConfig2ᚕgithubᚗcomᚋClusterCockpi
return ret
}
func (ec *executionContext) marshalNMetricConfig2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑlibᚋv2ᚋschemaᚐMetricConfig(ctx context.Context, sel ast.SelectionSet, v *schema.MetricConfig) graphql.Marshaler {
if v == nil {
if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) {
graphql.AddErrorf(ctx, "the requested element is null which the schema does not allow")
}
return graphql.Null
}
return ec._MetricConfig(ctx, sel, v)
}
func (ec *executionContext) marshalNMetricFootprints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricFootprintsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.MetricFootprints) graphql.Marshaler {
ret := make(graphql.Array, len(v))
var wg sync.WaitGroup

4
internal/graph/schema.resolvers.go
View File

@@ -3,7 +3,7 @@ package graph
// This file will be automatically regenerated based on the schema, any resolver
// implementations
// will be copied through when generating and any unknown code will be moved to the end.
// Code generated by github.com/99designs/gqlgen version v0.17.84
// Code generated by github.com/99designs/gqlgen version v0.17.85
import (
"context"
@@ -283,7 +283,7 @@ func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []strin
// Test Access: Admins && Admin Tag OR Everyone && Private Tag
if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope {
// Remove from DB
if err = r.Repo.RemoveTagById(tid); err != nil {
if err = r.Repo.RemoveTagByID(tid); err != nil {
cclog.Warn("Error while removing tag")
return nil, err
} else {

1
internal/importer/handleImport.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package importer
import (

1
internal/importer/normalize.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package importer
import (

4
internal/metricstore/metricstore.go
View File

@@ -74,7 +74,7 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) {
cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
// Helper function to add metric configuration
addMetricConfig := func(mc schema.MetricConfig) {
addMetricConfig := func(mc *schema.MetricConfig) {
agg, err := AssignAggregationStrategy(mc.Aggregation)
if err != nil {
cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
@@ -88,7 +88,7 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) {
for _, c := range archive.Clusters {
for _, mc := range c.MetricConfig {
addMetricConfig(*mc)
addMetricConfig(mc)
}
for _, sc := range c.SubClusters {

1
internal/repository/hooks.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
import (

2
internal/repository/job.go
View File

@@ -686,7 +686,6 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
return subclusters, nil
}
// FIXME: Set duration to requested walltime?
// StopJobsExceedingWalltimeBy marks running jobs as failed if they exceed their walltime limit.
// This is typically called periodically to clean up stuck or orphaned jobs.
//
@@ -762,7 +761,6 @@ func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
return jobIds, nil
}
// FIXME: Reconsider filtering short jobs with harcoded threshold
// FindRunningJobs returns all currently running jobs for a specific cluster.
// Filters out short-running jobs based on repoConfig.MinRunningJobDuration threshold.
//

1
internal/repository/jobHooks.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
import (

4
internal/repository/job_test.go
View File

@@ -90,13 +90,13 @@ func TestFindJobsBetween(t *testing.T) {
// 2. Create a tag
tagName := fmt.Sprintf("testtag_%d", time.Now().UnixNano())
tagId, err := r.CreateTag("testtype", tagName, "global")
tagID, err := r.CreateTag("testtype", tagName, "global")
if err != nil {
t.Fatal(err)
}
// 3. Link Tag (Manually to avoid archive dependency side-effects in unit test)
_, err = r.DB.Exec("INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)", *targetJob.ID, tagId)
_, err = r.DB.Exec("INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)", *targetJob.ID, tagID)
if err != nil {
t.Fatal(err)
}

2
internal/repository/node.go
View File

@@ -579,7 +579,7 @@ func (r *NodeRepository) GetNodesForList(
queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
}
if stateFilter != "all" && stateFilter != "notindb" {
var queryState schema.SchedulerState = schema.SchedulerState(stateFilter)
queryState := schema.SchedulerState(stateFilter)
queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
}
// if healthFilter != "all" {

8
internal/repository/repository_test.go
View File

@@ -46,7 +46,7 @@ func BenchmarkSelect1(b *testing.B) {
}
func BenchmarkDB_FindJobById(b *testing.B) {
var jobId int64 = 1677322
var jobID int64 = 1677322
b.Run("FindJobById", func(b *testing.B) {
db := setup(b)
@@ -55,7 +55,7 @@ func BenchmarkDB_FindJobById(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, err := db.FindByID(getContext(b), jobId)
_, err := db.FindByID(getContext(b), jobID)
noErr(b, err)
}
})
@@ -63,7 +63,7 @@ func BenchmarkDB_FindJobById(b *testing.B) {
}
func BenchmarkDB_FindJob(b *testing.B) {
var jobId int64 = 107266
var jobID int64 = 107266
var startTime int64 = 1657557241
cluster := "fritz"
@@ -74,7 +74,7 @@ func BenchmarkDB_FindJob(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, err := db.Find(&jobId, &cluster, &startTime)
_, err := db.Find(&jobID, &cluster, &startTime)
noErr(b, err)
}
})

356
internal/repository/stats.go
View File

@@ -2,6 +2,44 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// This file contains job statistics and histogram generation functionality for the JobRepository.
//
// # Job Statistics
//
// The statistics methods provide aggregated metrics about jobs including total jobs, users,
// walltime, and resource usage (nodes, cores, accelerators). Statistics can be computed:
// - Overall (JobsStats): Single aggregate across all matching jobs
// - Grouped (JobsStatsGrouped): Aggregated by user, project, cluster, or subcluster
// - Counts (JobCountGrouped, AddJobCount): Simple job counts with optional filtering
//
// All statistics methods support filtering via JobFilter and respect security contexts.
//
// # Histograms
//
// Histogram methods generate distribution data for visualization:
// - Duration, nodes, cores, accelerators (AddHistograms)
// - Job metrics like CPU load, memory usage (AddMetricHistograms)
//
// Histograms use intelligent binning:
// - Duration: Variable bin sizes (1m, 10m, 1h, 6h, 12h, 24h) with zero-padding
// - Resources: Natural value-based bins
// - Metrics: Normalized to peak values with configurable bin counts
//
// # Running vs. Completed Jobs
//
// Statistics handle running jobs specially:
// - Duration calculated as (now - start_time) for running jobs
// - Metric histograms for running jobs load data from metric backend instead of footprint
// - Job state filtering distinguishes running/completed jobs
//
// # Performance Considerations
//
// - All queries use prepared statements via stmtCache
// - Complex aggregations use SQL for efficiency
// - Histogram pre-initialization ensures consistent bin ranges
// - Metric histogram queries limited to 500 jobs for running job analysis
package repository
import (
@@ -19,7 +57,9 @@ import (
sq "github.com/Masterminds/squirrel"
)
// GraphQL validation should make sure that no unkown values can be specified.
// groupBy2column maps GraphQL Aggregate enum values to their corresponding database column names.
// Used by JobsStatsGrouped and JobCountGrouped to translate user-facing grouping dimensions
// into SQL GROUP BY clauses. GraphQL validation ensures only valid enum values are accepted.
var groupBy2column = map[model.Aggregate]string{
model.AggregateUser: "job.hpc_user",
model.AggregateProject: "job.project",
@@ -27,6 +67,9 @@ var groupBy2column = map[model.Aggregate]string{
model.AggregateSubcluster: "job.subcluster",
}
// sortBy2column maps GraphQL SortByAggregate enum values to their corresponding computed column names.
// Used by JobsStatsGrouped to translate sort preferences into SQL ORDER BY clauses.
// Column names match the AS aliases used in buildStatsQuery.
var sortBy2column = map[model.SortByAggregate]string{
model.SortByAggregateTotaljobs: "totalJobs",
model.SortByAggregateTotalusers: "totalUsers",
@@ -39,6 +82,21 @@ var sortBy2column = map[model.SortByAggregate]string{
model.SortByAggregateTotalacchours: "totalAccHours",
}
// buildCountQuery constructs a SQL query to count jobs with optional grouping and filtering.
//
// Parameters:
// - filter: Job filters to apply (cluster, user, time range, etc.)
// - kind: Special filter - "running" for running jobs only, "short" for jobs under threshold
// - col: Column name to GROUP BY; empty string for total count without grouping
//
// Returns a SelectBuilder that produces either:
// - Single count: COUNT(job.id) when col is empty
// - Grouped counts: col, COUNT(job.id) when col is specified
//
// The kind parameter enables counting specific job categories:
// - "running": Only jobs with job_state = 'running'
// - "short": Only jobs with duration < ShortRunningJobsDuration config value
// - empty: All jobs matching filters
func (r *JobRepository) buildCountQuery(
filter []*model.JobFilter,
kind string,
@@ -47,10 +105,8 @@ func (r *JobRepository) buildCountQuery(
var query sq.SelectBuilder
if col != "" {
// Scan columns: id, cnt
query = sq.Select(col, "COUNT(job.id)").From("job").GroupBy(col)
} else {
// Scan columns: cnt
query = sq.Select("COUNT(job.id)").From("job")
}
@@ -68,6 +124,27 @@ func (r *JobRepository) buildCountQuery(
return query
}
// buildStatsQuery constructs a SQL query to compute comprehensive job statistics with optional grouping.
//
// Parameters:
// - filter: Job filters to apply (cluster, user, time range, etc.)
// - col: Column name to GROUP BY; empty string for overall statistics without grouping
//
// Returns a SelectBuilder that produces comprehensive statistics:
// - totalJobs: Count of jobs
// - totalUsers: Count of distinct users (always 0 when grouping by user)
// - totalWalltime: Sum of job durations in hours
// - totalNodes: Sum of nodes used across all jobs
// - totalNodeHours: Sum of (duration × num_nodes) in hours
// - totalCores: Sum of hardware threads used across all jobs
// - totalCoreHours: Sum of (duration × num_hwthreads) in hours
// - totalAccs: Sum of accelerators used across all jobs
// - totalAccHours: Sum of (duration × num_acc) in hours
//
// Special handling:
// - Running jobs: Duration calculated as (now - start_time) instead of stored duration
// - Grouped queries: Also select grouping column and user's display name from hpc_user table
// - All time values converted from seconds to hours (÷ 3600) and rounded
func (r *JobRepository) buildStatsQuery(
filter []*model.JobFilter,
col string,
@@ -75,31 +152,29 @@ func (r *JobRepository) buildStatsQuery(
var query sq.SelectBuilder
if col != "" {
// Scan columns: id, name, totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
query = sq.Select(
col,
"name",
"COUNT(job.id) as totalJobs",
"COUNT(DISTINCT job.hpc_user) AS totalUsers",
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as int) as totalWalltime`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_nodes) as int) as totalNodes`),
`CAST(SUM(job.num_nodes) as int) as totalNodes`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as int) as totalNodeHours`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as int) as totalCores`),
`CAST(SUM(job.num_hwthreads) as int) as totalCores`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as int) as totalCoreHours`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_acc) as int) as totalAccs`),
`CAST(SUM(job.num_acc) as int) as totalAccs`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as int) as totalAccHours`, time.Now().Unix()),
).From("job").LeftJoin("hpc_user ON hpc_user.username = job.hpc_user").GroupBy(col)
} else {
// Scan columns: totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
query = sq.Select(
"COUNT(job.id) as totalJobs",
"COUNT(DISTINCT job.hpc_user) AS totalUsers",
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as int)`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_nodes) as int)`),
`CAST(SUM(job.num_nodes) as int)`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as int)`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as int)`),
`CAST(SUM(job.num_hwthreads) as int)`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as int)`, time.Now().Unix()),
fmt.Sprintf(`CAST(SUM(job.num_acc) as int)`),
`CAST(SUM(job.num_acc) as int)`,
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as int)`, time.Now().Unix()),
).From("job")
}
@@ -111,6 +186,25 @@ func (r *JobRepository) buildStatsQuery(
return query
}
// JobsStatsGrouped computes comprehensive job statistics grouped by a dimension (user, project, cluster, or subcluster).
//
// This is the primary method for generating aggregated statistics views in the UI, providing
// metrics like total jobs, walltime, and resource usage broken down by the specified grouping.
//
// Parameters:
// - ctx: Context for security checks and cancellation
// - filter: Filters to apply (time range, cluster, job state, etc.)
// - page: Optional pagination (ItemsPerPage: -1 disables pagination)
// - sortBy: Optional sort column (totalJobs, totalWalltime, totalCoreHours, etc.)
// - groupBy: Required grouping dimension (User, Project, Cluster, or Subcluster)
//
// Returns a slice of JobsStatistics, one per group, with:
// - ID: The group identifier (username, project name, cluster name, etc.)
// - Name: Display name (for users, from hpc_user.name; empty for other groups)
// - Statistics: totalJobs, totalUsers, totalWalltime, resource usage metrics
//
// Security: Respects user roles via SecurityCheck - users see only their own data unless admin/support.
// Performance: Results are sorted in SQL and pagination applied before scanning rows.
func (r *JobRepository) JobsStatsGrouped(
ctx context.Context,
filter []*model.JobFilter,
@@ -230,6 +324,21 @@ func (r *JobRepository) JobsStatsGrouped(
return stats, nil
}
// JobsStats computes overall job statistics across all matching jobs without grouping.
//
// This method provides a single aggregate view of job metrics, useful for dashboard
// summaries and overall system utilization reports.
//
// Parameters:
// - ctx: Context for security checks and cancellation
// - filter: Filters to apply (time range, cluster, job state, etc.)
//
// Returns a single-element slice containing aggregate statistics:
// - totalJobs, totalUsers, totalWalltime
// - totalNodeHours, totalCoreHours, totalAccHours
//
// Unlike JobsStatsGrouped, this returns overall totals without breaking down by dimension.
// Security checks are applied via SecurityCheck to respect user access levels.
func (r *JobRepository) JobsStats(
ctx context.Context,
filter []*model.JobFilter,
@@ -303,6 +412,17 @@ func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
return 0.0
}
// JobCountGrouped counts jobs grouped by a dimension without computing detailed statistics.
//
// This is a lightweight alternative to JobsStatsGrouped when only job counts are needed,
// avoiding the overhead of calculating walltime and resource usage metrics.
//
// Parameters:
// - ctx: Context for security checks
// - filter: Filters to apply
// - groupBy: Grouping dimension (User, Project, Cluster, or Subcluster)
//
// Returns JobsStatistics with only ID and TotalJobs populated for each group.
func (r *JobRepository) JobCountGrouped(
ctx context.Context,
filter []*model.JobFilter,
@@ -343,6 +463,20 @@ func (r *JobRepository) JobCountGrouped(
return stats, nil
}
// AddJobCountGrouped augments existing statistics with additional job counts by category.
//
// This method enriches JobsStatistics returned by JobsStatsGrouped or JobCountGrouped
// with counts of running or short-running jobs, matched by group ID.
//
// Parameters:
// - ctx: Context for security checks
// - filter: Filters to apply
// - groupBy: Grouping dimension (must match the dimension used for stats parameter)
// - stats: Existing statistics to augment (modified in-place by ID matching)
// - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
//
// Returns the same stats slice with RunningJobs or ShortJobs fields populated per group.
// Groups without matching jobs will have 0 for the added field.
func (r *JobRepository) AddJobCountGrouped(
ctx context.Context,
filter []*model.JobFilter,
@@ -392,6 +526,18 @@ func (r *JobRepository) AddJobCountGrouped(
return stats, nil
}
// AddJobCount augments existing overall statistics with additional job counts by category.
//
// Similar to AddJobCountGrouped but for ungrouped statistics. Applies the same count
// to all statistics entries (typically just one).
//
// Parameters:
// - ctx: Context for security checks
// - filter: Filters to apply
// - stats: Existing statistics to augment (modified in-place)
// - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
//
// Returns the same stats slice with RunningJobs or ShortJobs fields set to the total count.
func (r *JobRepository) AddJobCount(
ctx context.Context,
filter []*model.JobFilter,
@@ -437,6 +583,26 @@ func (r *JobRepository) AddJobCount(
return stats, nil
}
// AddHistograms augments statistics with distribution histograms for job properties.
//
// Generates histogram data for visualization of job duration, node count, core count,
// and accelerator count distributions. Duration histogram uses intelligent binning based
// on the requested resolution.
//
// Parameters:
// - ctx: Context for security checks
// - filter: Filters to apply to jobs included in histograms
// - stat: Statistics struct to augment (modified in-place)
// - durationBins: Bin size - "1m", "10m", "1h", "6h", "12h", or "24h" (default)
//
// Populates these fields in stat:
// - HistDuration: Job duration distribution (zero-padded bins)
// - HistNumNodes: Node count distribution
// - HistNumCores: Core (hwthread) count distribution
// - HistNumAccs: Accelerator count distribution
//
// Duration bins are pre-initialized with zeros to ensure consistent ranges for visualization.
// Bin size determines both the width and maximum duration displayed (e.g., "1h" = 48 bins × 1h = 48h max).
func (r *JobRepository) AddHistograms(
ctx context.Context,
filter []*model.JobFilter,
@@ -447,20 +613,20 @@ func (r *JobRepository) AddHistograms(
var targetBinCount int
var targetBinSize int
switch {
case *durationBins == "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
switch *durationBins {
case "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
targetBinCount = 60
targetBinSize = 60
case *durationBins == "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
case "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
targetBinCount = 72
targetBinSize = 600
case *durationBins == "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
case "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
targetBinCount = 48
targetBinSize = 3600
case *durationBins == "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
case "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
targetBinCount = 12
targetBinSize = 21600
case *durationBins == "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
case "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
targetBinCount = 14
targetBinSize = 43200
default: // 24h
@@ -499,7 +665,30 @@ func (r *JobRepository) AddHistograms(
return stat, nil
}
// Requires thresholds for metric from config for cluster? Of all clusters and use largest? split to 10 + 1 for artifacts?
// AddMetricHistograms augments statistics with distribution histograms for job metrics.
//
// Generates histogram data for metrics like CPU load, memory usage, etc. Handles running
// and completed jobs differently: running jobs load data from metric backend, completed jobs
// use footprint data from database.
//
// Parameters:
// - ctx: Context for security checks
// - filter: Filters to apply (MUST contain State filter for running jobs)
// - metrics: List of metric names to histogram (e.g., ["cpu_load", "mem_used"])
// - stat: Statistics struct to augment (modified in-place)
// - targetBinCount: Number of histogram bins (default: 10)
//
// Populates HistMetrics field in stat with MetricHistoPoints for each metric.
//
// Binning algorithm:
// - Values normalized to metric's peak value from cluster configuration
// - Bins evenly distributed from 0 to peak
// - Pre-initialized with zeros for consistent visualization
//
// Limitations:
// - Running jobs: Limited to 500 jobs for performance
// - Requires valid cluster configuration with metric peak values
// - Uses footprint statistic (avg/max/min) configured per metric
func (r *JobRepository) AddMetricHistograms(
ctx context.Context,
filter []*model.JobFilter,
@@ -534,7 +723,16 @@ func (r *JobRepository) AddMetricHistograms(
return stat, nil
}
// `value` must be the column grouped by, but renamed to "value"
// jobsStatisticsHistogram generates a simple histogram by grouping on a column value.
//
// Used for histograms where the column value directly represents the bin (e.g., node count, core count).
// Unlike duration/metric histograms, this doesn't pre-initialize bins with zeros.
//
// Parameters:
// - value: SQL expression that produces the histogram value, aliased as "value"
// - filters: Job filters to apply
//
// Returns histogram points with Value (from column) and Count (number of jobs).
func (r *JobRepository) jobsStatisticsHistogram(
ctx context.Context,
value string,
@@ -573,6 +771,26 @@ func (r *JobRepository) jobsStatisticsHistogram(
return points, nil
}
// jobsDurationStatisticsHistogram generates a duration histogram with pre-initialized bins.
//
// Bins are zero-padded to provide consistent ranges for visualization, unlike simple
// histograms which only return bins with data. The value parameter should compute
// the bin number from job duration.
//
// Parameters:
// - value: SQL expression computing bin number from duration, aliased as "value"
// - filters: Job filters to apply
// - binSizeSeconds: Width of each bin in seconds
// - targetBinCount: Number of bins to pre-initialize
//
// Returns histogram points with Value (bin_number × binSizeSeconds) and Count.
// All bins from 1 to targetBinCount are returned, with Count=0 for empty bins.
//
// Algorithm:
// 1. Pre-initialize targetBinCount bins with zero counts
// 2. Query database for actual counts per bin
// 3. Match query results to pre-initialized bins by value
// 4. Bins without matches remain at zero
func (r *JobRepository) jobsDurationStatisticsHistogram(
ctx context.Context,
value string,
@@ -588,7 +806,6 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
return nil, qerr
}
// Initialize histogram bins with zero counts
// Each bin represents a duration range: bin N = [N*binSizeSeconds, (N+1)*binSizeSeconds)
// Example: binSizeSeconds=3600 (1 hour), bin 1 = 0-1h, bin 2 = 1-2h, etc.
points := make([]*model.HistoPoint, 0)
@@ -607,8 +824,8 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
return nil, err
}
// Match query results to pre-initialized bins and fill counts
// Query returns raw duration values that need to be mapped to correct bins
// Match query results to pre-initialized bins.
// point.Value from query is the bin number; multiply by binSizeSeconds to match bin.Value.
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
@@ -616,13 +833,8 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
return nil, err
}
// Find matching bin and update count
// point.Value is multiplied by binSizeSeconds to match pre-calculated bin.Value
for _, e := range points {
if e.Value == (point.Value * binSizeSeconds) {
// Note: Matching on unmodified integer value (and multiplying point.Value
// by binSizeSeconds after match) causes frontend to loop into highest
// targetBinCount, due to zoom condition instantly being fulfilled (cause unknown)
e.Count = point.Count
break
}
@@ -633,13 +845,34 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
return points, nil
}
// jobsMetricStatisticsHistogram generates a metric histogram using footprint data from completed jobs.
//
// Values are normalized to the metric's peak value and distributed into bins. The algorithm
// is based on SQL histogram generation techniques, extracting metric values from JSON footprint
// and computing bin assignments in SQL.
//
// Parameters:
// - metric: Metric name (e.g., "cpu_load", "mem_used")
// - filters: Job filters to apply
// - bins: Number of bins to generate
//
// Returns MetricHistoPoints with metric name, unit, footprint stat type, and binned data.
//
// Algorithm:
// 1. Determine peak value from cluster configuration (filtered cluster or max across all)
// 2. Generate SQL that extracts footprint value, normalizes to [0,1], multiplies by bin count
// 3. Pre-initialize bins with min/max ranges based on peak value
// 4. Query database for counts per bin
// 5. Match results to pre-initialized bins
//
// Special handling: Values exactly equal to peak are forced into the last bin by multiplying
// peak by 0.999999999 to avoid creating an extra bin.
func (r *JobRepository) jobsMetricStatisticsHistogram(
ctx context.Context,
metric string,
filters []*model.JobFilter,
bins *int,
) (*model.MetricHistoPoints, error) {
// Determine the metric's peak value for histogram normalization
// Peak value defines the upper bound for binning: values are distributed across
// bins from 0 to peak. First try to get peak from filtered cluster, otherwise
// scan all clusters to find the maximum peak value.
@@ -679,18 +912,14 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
}
}
// Construct SQL histogram bins using normalized values
// Construct SQL histogram bins using normalized values.
// Algorithm based on: https://jereze.com/code/sql-histogram/ (modified)
start := time.Now()
// Calculate bin number for each job's metric value:
// 1. Extract metric value from JSON footprint
// 2. Normalize to [0,1] by dividing by peak
// 3. Multiply by number of bins to get bin number
// 4. Cast to integer for bin assignment
//
// Special case: Values exactly equal to peak would fall into bin N+1,
// so we multiply peak by 0.999999999 to force it into the last bin (bin N)
// Bin calculation formula:
// bin_number = CAST( (value / peak) * num_bins AS INTEGER ) + 1
// Special case: value == peak would create bin N+1, so we test for equality
// and multiply peak by 0.999999999 to force it into bin N.
binQuery := fmt.Sprintf(`CAST(
((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
* %v as INTEGER )`,
@@ -699,24 +928,19 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
mainQuery := sq.Select(
fmt.Sprintf(`%s + 1 as bin`, binQuery),
`count(*) as count`,
// For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * %s as min`, peak, *bins, binQuery),
// For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * (%s + 1) as max`, peak, *bins, binQuery),
).From("job").Where(
"JSON_VALID(footprint)",
).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
// Only accessible Jobs...
mainQuery, qerr := SecurityCheck(ctx, mainQuery)
if qerr != nil {
return nil, qerr
}
// Filters...
for _, f := range filters {
mainQuery = BuildWhereClause(f, mainQuery)
}
// Finalize query with Grouping and Ordering
mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
rows, err := mainQuery.RunWith(r.DB).Query()
@@ -725,8 +949,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
return nil, err
}
// Initialize histogram bins with calculated min/max ranges
// Each bin represents a range of metric values
// Pre-initialize bins with calculated min/max ranges.
// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
points := make([]*model.MetricHistoPoint, 0)
binStep := int(peak) / *bins
@@ -737,29 +960,18 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
points = append(points, &epoint)
}
// Fill counts from query results
// Query only returns bins that have jobs, so we match against pre-initialized bins
// Match query results to pre-initialized bins.
for rows.Next() {
rpoint := model.MetricHistoPoint{}
if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil { // Required for Debug: &rpoint.Min, &rpoint.Max
if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil {
cclog.Warnf("Error while scanning rows for %s", metric)
return nil, err // FIXME: Totally bricks cc-backend if returned and if all metrics requested?
return nil, err
}
// Match query result to pre-initialized bin and update count
for _, e := range points {
if e.Bin != nil && rpoint.Bin != nil {
if *e.Bin == *rpoint.Bin {
e.Count = rpoint.Count
// Only Required For Debug: Check DB returned Min/Max against Backend Init above
// if rpoint.Min != nil {
// cclog.Warnf(">>>> Bin %d Min Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Min, *e.Min)
// }
// if rpoint.Max != nil {
// cclog.Warnf(">>>> Bin %d Max Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Max, *e.Max)
// }
break
}
if e.Bin != nil && rpoint.Bin != nil && *e.Bin == *rpoint.Bin {
e.Count = rpoint.Count
break
}
}
}
@@ -770,6 +982,28 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
return &result, nil
}
// runningJobsMetricStatisticsHistogram generates metric histograms for running jobs using live data.
//
// Unlike completed jobs which use footprint data from the database, running jobs require
// fetching current metric averages from the metric backend (via metricdispatch).
//
// Parameters:
// - metrics: List of metric names
// - filters: Job filters (should filter to running jobs only)
// - bins: Number of histogram bins
//
// Returns slice of MetricHistoPoints, one per metric.
//
// Limitations:
// - Maximum 500 jobs (returns nil if more jobs match)
// - Requires metric backend availability
// - Bins based on metric peak values from cluster configuration
//
// Algorithm:
// 1. Query first 501 jobs to check count limit
// 2. Load metric averages for all jobs via metricdispatch
// 3. For each metric, create bins based on peak value
// 4. Iterate averages and count jobs per bin
func (r *JobRepository) runningJobsMetricStatisticsHistogram(
ctx context.Context,
metrics []string,

197
internal/repository/tags.go
View File

@@ -3,6 +3,34 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// Package repository provides data access and persistence layer for ClusterCockpit.
//
// This file implements tag management functionality for job categorization and classification.
// Tags support both manual assignment (via REST/GraphQL APIs) and automatic detection
// (via tagger plugins). The implementation includes role-based access control through
// tag scopes and maintains bidirectional consistency between the SQL database and
// the file-based job archive.
//
// Database Schema:
//
// CREATE TABLE tag (
// id INTEGER PRIMARY KEY AUTOINCREMENT,
// tag_type VARCHAR(255) NOT NULL,
// tag_name VARCHAR(255) NOT NULL,
// tag_scope VARCHAR(255) NOT NULL DEFAULT "global",
// CONSTRAINT tag_unique UNIQUE (tag_type, tag_name, tag_scope)
// );
//
// CREATE TABLE jobtag (
// job_id INTEGER,
// tag_id INTEGER,
// PRIMARY KEY (job_id, tag_id),
// FOREIGN KEY (job_id) REFERENCES job(id) ON DELETE CASCADE,
// FOREIGN KEY (tag_id) REFERENCES tag(id) ON DELETE CASCADE
// );
//
// The jobtag junction table enables many-to-many relationships between jobs and tags.
// CASCADE deletion ensures referential integrity when jobs or tags are removed.
package repository
import (
@@ -73,7 +101,7 @@ func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error
func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) {
j, err := r.FindByIDWithUser(user, job)
if err != nil {
cclog.Warn("Error while finding job by id")
cclog.Warnf("Error while finding job %d for user %s during tag removal: %v", job, user.Username, err)
return nil, err
}
@@ -93,7 +121,7 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.
archiveTags, err := r.getArchiveTags(&job)
if err != nil {
cclog.Warn("Error while getting tags for job")
cclog.Warnf("Error while getting archive tags for job %d in RemoveTag: %v", job, err)
return nil, err
}
@@ -104,7 +132,7 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.
// Requires user authentication for security checks. Used by REST API.
func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) {
// Get Tag ID to delete
tagID, exists := r.TagId(tagType, tagName, tagScope)
tagID, exists := r.TagID(tagType, tagName, tagScope)
if !exists {
cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
return nil, fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
@@ -113,7 +141,7 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT
// Get Job
j, err := r.FindByIDWithUser(user, job)
if err != nil {
cclog.Warn("Error while finding job by id")
cclog.Warnf("Error while finding job %d for user %s during tag removal by request: %v", job, user.Username, err)
return nil, err
}
@@ -128,19 +156,30 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT
tags, err := r.GetTags(user, &job)
if err != nil {
cclog.Warn("Error while getting tags for job")
cclog.Warnf("Error while getting tags for job %d in RemoveJobTagByRequest: %v", job, err)
return nil, err
}
archiveTags, err := r.getArchiveTags(&job)
if err != nil {
cclog.Warn("Error while getting tags for job")
cclog.Warnf("Error while getting archive tags for job %d in RemoveJobTagByRequest: %v", job, err)
return nil, err
}
return tags, archive.UpdateTags(j, archiveTags)
}
// removeTagFromArchiveJobs updates the job archive for all affected jobs after a tag deletion.
//
// This function is called asynchronously (via goroutine) after removing a tag from the database
// to synchronize the file-based job archive with the database state. Errors are logged but not
// returned since this runs in the background.
//
// Parameters:
// - jobIds: Database IDs of all jobs that had the deleted tag
//
// Implementation note: Each job is processed individually to handle partial failures gracefully.
// If one job fails to update, others will still be processed.
func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
for _, j := range jobIds {
tags, err := r.getArchiveTags(&j)
@@ -163,18 +202,18 @@ func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
// Used by REST API. Does not update tagged jobs in Job archive.
func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagScope string) error {
// Get Tag ID to delete
tagID, exists := r.TagId(tagType, tagName, tagScope)
tagID, exists := r.TagID(tagType, tagName, tagScope)
if !exists {
cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
return fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
}
return r.RemoveTagById(tagID)
return r.RemoveTagByID(tagID)
}
// Removes a tag from db by tag id
// Used by GraphQL API.
func (r *JobRepository) RemoveTagById(tagID int64) error {
func (r *JobRepository) RemoveTagByID(tagID int64) error {
jobIds, err := r.FindJobIdsByTag(tagID)
if err != nil {
return err
@@ -213,7 +252,7 @@ func (r *JobRepository) RemoveTagById(tagID int64) error {
// Example:
//
// tagID, err := repo.CreateTag("performance", "high-memory", "global")
func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) {
func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagID int64, err error) {
// Default to "Global" scope if none defined
if tagScope == "" {
tagScope = "global"
@@ -300,13 +339,13 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
for rows.Next() {
var tagType string
var tagName string
var tagId int
var tagID int
var count int
if err = rows.Scan(&tagType, &tagName, &tagId, &count); err != nil {
if err = rows.Scan(&tagType, &tagName, &tagID, &count); err != nil {
return nil, nil, err
}
// Use tagId as second Map-Key component to differentiate tags with identical names
counts[fmt.Sprint(tagType, tagName, tagId)] = count
counts[fmt.Sprint(tagType, tagName, tagID)] = count
}
err = rows.Err()
@@ -314,18 +353,44 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
}
var (
ErrTagNotFound = errors.New("the tag does not exist")
ErrJobNotOwned = errors.New("user is not owner of job")
ErrTagNoAccess = errors.New("user not permitted to use that tag")
ErrTagPrivateScope = errors.New("tag is private to another user")
ErrTagAdminScope = errors.New("tag requires admin privileges")
// ErrTagNotFound is returned when a tag ID or tag identifier (type, name, scope) does not exist in the database.
ErrTagNotFound = errors.New("the tag does not exist")
// ErrJobNotOwned is returned when a user attempts to tag a job they do not have permission to access.
ErrJobNotOwned = errors.New("user is not owner of job")
// ErrTagNoAccess is returned when a user attempts to use a tag they cannot access due to scope restrictions.
ErrTagNoAccess = errors.New("user not permitted to use that tag")
// ErrTagPrivateScope is returned when a user attempts to access another user's private tag.
ErrTagPrivateScope = errors.New("tag is private to another user")
// ErrTagAdminScope is returned when a non-admin user attempts to use an admin-scoped tag.
ErrTagAdminScope = errors.New("tag requires admin privileges")
// ErrTagsIncompatScopes is returned when attempting to combine admin and non-admin scoped tags in a single operation.
ErrTagsIncompatScopes = errors.New("combining admin and non-admin scoped tags not allowed")
)
// addJobTag is a helper function that inserts a job-tag association and updates the archive.
// Returns the updated tag list for the job.
func (r *JobRepository) addJobTag(jobId int64, tagId int64, job *schema.Job, getTags func() ([]*schema.Tag, error)) ([]*schema.Tag, error) {
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobId, tagId)
//
// This function performs three operations atomically:
// 1. Inserts the job-tag association into the jobtag junction table
// 2. Retrieves the updated tag list for the job (using the provided getTags callback)
// 3. Updates the job archive with the new tags to maintain database-archive consistency
//
// Parameters:
// - jobId: Database ID of the job
// - tagId: Database ID of the tag to associate
// - job: Full job object needed for archive update
// - getTags: Callback function to retrieve updated tags (allows different security contexts)
//
// Returns the complete updated tag list for the job or an error.
//
// Note: This function does NOT validate tag scope permissions - callers must perform
// authorization checks before invoking this helper.
func (r *JobRepository) addJobTag(jobID int64, tagID int64, job *schema.Job, getTags func() ([]*schema.Tag, error)) ([]*schema.Tag, error) {
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobID, tagID)
if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
s, _, _ := q.ToSql()
@@ -335,13 +400,13 @@ func (r *JobRepository) addJobTag(jobId int64, tagId int64, job *schema.Job, get
tags, err := getTags()
if err != nil {
cclog.Warnf("Error getting tags for job %d: %v", jobId, err)
cclog.Warnf("Error getting tags for job %d: %v", jobID, err)
return nil, err
}
archiveTags, err := r.getArchiveTags(&jobId)
archiveTags, err := r.getArchiveTags(&jobID)
if err != nil {
cclog.Warnf("Error getting archive tags for job %d: %v", jobId, err)
cclog.Warnf("Error getting archive tags for job %d: %v", jobID, err)
return nil, err
}
@@ -350,7 +415,7 @@ func (r *JobRepository) addJobTag(jobId int64, tagId int64, job *schema.Job, get
// AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`.
// If such a tag does not yet exist, it is created.
func (r *JobRepository) AddTagOrCreate(user *schema.User, jobId int64, tagType string, tagName string, tagScope string) (tagId int64, err error) {
func (r *JobRepository) AddTagOrCreate(user *schema.User, jobID int64, tagType string, tagName string, tagScope string) (tagID int64, err error) {
// Default to "Global" scope if none defined
if tagScope == "" {
tagScope = "global"
@@ -364,44 +429,43 @@ func (r *JobRepository) AddTagOrCreate(user *schema.User, jobId int64, tagType s
return 0, fmt.Errorf("cannot write tag scope with current authorization")
}
tagId, exists := r.TagId(tagType, tagName, tagScope)
tagID, exists := r.TagID(tagType, tagName, tagScope)
if !exists {
tagId, err = r.CreateTag(tagType, tagName, tagScope)
tagID, err = r.CreateTag(tagType, tagName, tagScope)
if err != nil {
return 0, err
}
}
if _, err := r.AddTag(user, jobId, tagId); err != nil {
if _, err := r.AddTag(user, jobID, tagID); err != nil {
return 0, err
}
return tagId, nil
return tagID, nil
}
// used in auto tagger plugins
func (r *JobRepository) AddTagOrCreateDirect(jobId int64, tagType string, tagName string) (tagId int64, err error) {
func (r *JobRepository) AddTagOrCreateDirect(jobID int64, tagType string, tagName string) (tagID int64, err error) {
tagScope := "global"
tagId, exists := r.TagId(tagType, tagName, tagScope)
tagID, exists := r.TagID(tagType, tagName, tagScope)
if !exists {
tagId, err = r.CreateTag(tagType, tagName, tagScope)
tagID, err = r.CreateTag(tagType, tagName, tagScope)
if err != nil {
return 0, err
}
}
if _, err := r.AddTagDirect(jobId, tagId); err != nil {
if _, err := r.AddTagDirect(jobID, tagID); err != nil {
return 0, err
}
return tagId, nil
return tagID, nil
}
func (r *JobRepository) HasTag(jobId int64, tagType string, tagName string) bool {
func (r *JobRepository) HasTag(jobID int64, tagType string, tagName string) bool {
var id int64
q := sq.Select("id").From("tag").Join("jobtag ON jobtag.tag_id = tag.id").
Where("jobtag.job_id = ?", jobId).Where("tag.tag_type = ?", tagType).
Where("jobtag.job_id = ?", jobID).Where("tag.tag_type = ?", tagType).
Where("tag.tag_name = ?", tagName)
err := q.RunWith(r.stmtCache).QueryRow().Scan(&id)
if err != nil {
@@ -411,21 +475,21 @@ func (r *JobRepository) HasTag(jobId int64, tagType string, tagName string) bool
}
}
// TagId returns the database id of the tag with the specified type and name.
func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) (tagId int64, exists bool) {
// TagID returns the database id of the tag with the specified type and name.
func (r *JobRepository) TagID(tagType string, tagName string, tagScope string) (tagID int64, exists bool) {
exists = true
if err := sq.Select("id").From("tag").
Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).Where("tag.tag_scope = ?", tagScope).
RunWith(r.stmtCache).QueryRow().Scan(&tagId); err != nil {
RunWith(r.stmtCache).QueryRow().Scan(&tagID); err != nil {
exists = false
}
return
}
// TagInfo returns the database infos of the tag with the specified id.
func (r *JobRepository) TagInfo(tagId int64) (tagType string, tagName string, tagScope string, exists bool) {
func (r *JobRepository) TagInfo(tagID int64) (tagType string, tagName string, tagScope string, exists bool) {
exists = true
if err := sq.Select("tag.tag_type", "tag.tag_name", "tag.tag_scope").From("tag").Where("tag.id = ?", tagId).
if err := sq.Select("tag.tag_type", "tag.tag_name", "tag.tag_scope").From("tag").Where("tag.id = ?", tagID).
RunWith(r.stmtCache).QueryRow().Scan(&tagType, &tagName, &tagScope); err != nil {
exists = false
}
@@ -450,7 +514,7 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
for rows.Next() {
tag := &schema.Tag{}
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
cclog.Warn("Error while scanning rows")
cclog.Warnf("Error while scanning tag rows in GetTags: %v", err)
return nil, err
}
// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
@@ -483,7 +547,7 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
for rows.Next() {
tag := &schema.Tag{}
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
cclog.Warn("Error while scanning rows")
cclog.Warnf("Error while scanning tag rows in GetTagsDirect: %v", err)
return nil, err
}
tags = append(tags, tag)
@@ -492,7 +556,18 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
return tags, nil
}
// GetArchiveTags returns a list of all tags *regardless of scope* for archiving if job is nil or of the tags that the job with that database ID has.
// getArchiveTags returns all tags for a job WITHOUT applying scope-based filtering.
//
// This internal function is used exclusively for job archive synchronization where we need
// to store all tags regardless of the current user's permissions. Unlike GetTags() which
// filters by scope, this returns the complete unfiltered tag list.
//
// Parameters:
// - job: Pointer to job database ID, or nil to return all tags in the system
//
// Returns all tags without scope filtering, used only for archive operations.
//
// WARNING: Do NOT expose this function to user-facing APIs as it bypasses authorization.
func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
if job != nil {
@@ -510,7 +585,7 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
for rows.Next() {
tag := &schema.Tag{}
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
cclog.Warn("Error while scanning rows")
cclog.Warnf("Error while scanning tag rows in getArchiveTags: %v", err)
return nil, err
}
tags = append(tags, tag)
@@ -519,18 +594,18 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
return tags, nil
}
func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, tagScope string) (err error) {
func (r *JobRepository) ImportTag(jobID int64, tagType string, tagName string, tagScope string) (err error) {
// Import has no scope ctx, only import from metafile to DB (No recursive archive update required), only returns err
tagId, exists := r.TagId(tagType, tagName, tagScope)
tagID, exists := r.TagID(tagType, tagName, tagScope)
if !exists {
tagId, err = r.CreateTag(tagType, tagName, tagScope)
tagID, err = r.CreateTag(tagType, tagName, tagScope)
if err != nil {
return err
}
}
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobId, tagId)
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobID, tagID)
if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
s, _, _ := q.ToSql()
@@ -541,6 +616,28 @@ func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, t
return nil
}
// checkScopeAuth validates whether a user is authorized to perform an operation on a tag with the given scope.
//
// This function implements the tag scope authorization matrix:
//
// Scope | Read Access | Write Access
// -------------|----------------------------------|----------------------------------
// "global" | All users | Admin, Support, API-only
// "admin" | Admin, Support | Admin, API-only
// <username> | Owner only | Owner only (private tags)
//
// Parameters:
// - user: User attempting the operation (must not be nil)
// - operation: Either "read" or "write"
// - scope: Tag scope value ("global", "admin", or username for private tags)
//
// Returns:
// - pass: true if authorized, false if denied
// - err: error only if operation is invalid or user is nil
//
// Special cases:
// - API-only users (single role: RoleApi) can write to admin and global scopes for automation
// - Private tags use the username as scope, granting exclusive access to that user
func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scope string) (pass bool, err error) {
if user != nil {
switch {