ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-26 05:19:05 +01:00
Code Issues Packages Releases Activity

Start to convert to new footprint layout

Browse Source
This commit is contained in:
Jan Eitzinger 2024-06-28 16:48:10 +02:00
parent e1c1c06fb2
commit 786770f56a
Signed by: moebiusband
GPG Key ID: 2574BA29B90D6DD5
5 changed files with 78 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
internal/repository/job.go
View File

@ -59,7 +59,7 @@ func GetJobRepository() *JobRepository {
var jobColumns []string = []string{ var jobColumns []string = []string{
"job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.partition", "job.array_job_id", "job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.partition", "job.array_job_id",
"job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state", "job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state",
"job.duration", "job.walltime", "job.resources", "job.mem_used_max", "job.flops_any_avg", "job.mem_bw_avg", "job.load_avg", // "job.meta_data", "job.duration", "job.walltime", "job.resources", "job.footprint", // "job.meta_data",
} }
func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
@ -67,15 +67,22 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
if err := row.Scan( if err := row.Scan(
&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId,
&job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State,
&job.Duration, &job.Walltime, &job.RawResources, &job.MemUsedMax, &job.FlopsAnyAvg, &job.MemBwAvg, &job.LoadAvg /*&job.RawMetaData*/); err != nil { &job.Duration, &job.Walltime, &job.RawResources, &job.RawFootprint /*&job.RawMetaData*/); err != nil {
log.Warnf("Error while scanning rows (Job): %v", err) log.Warnf("Error while scanning rows (Job): %v", err)
return nil, err return nil, err
} }
if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil { if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil {
log.Warn("Error while unmarhsaling raw resources json") log.Warn("Error while unmarshaling raw resources json")
return nil, err return nil, err
} }
job.RawResources = nil
if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
log.Warn("Error while unmarshaling raw footprint json")
return nil, err
}
job.RawFootprint = nil
// if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { // if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
// return nil, err // return nil, err
@ -86,7 +93,6 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
job.Duration = int32(time.Since(job.StartTime).Seconds()) job.Duration = int32(time.Since(job.StartTime).Seconds())
} }
job.RawResources = nil
return job, nil return job, nil
} }

2
internal/repository/migration.go
View File

@ -16,7 +16,7 @@ import (
"github.com/golang-migrate/migrate/v4/source/iofs" "github.com/golang-migrate/migrate/v4/source/iofs"
) )
const Version uint = 7 const Version uint = 8
//go:embed migrations/* //go:embed migrations/*
var migrationFiles embed.FS var migrationFiles embed.FS

18
internal/repository/query.go
View File

@ -22,8 +22,8 @@ func (r *JobRepository) QueryJobs(
ctx context.Context, ctx context.Context,
filters []*model.JobFilter, filters []*model.JobFilter,
page *model.PageRequest, page *model.PageRequest,
order *model.OrderByInput) ([]*schema.Job, error) { order *model.OrderByInput,
) ([]*schema.Job, error) {
query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
if qerr != nil { if qerr != nil {
return nil, qerr return nil, qerr
@ -73,8 +73,8 @@ func (r *JobRepository) QueryJobs(
func (r *JobRepository) CountJobs( func (r *JobRepository) CountJobs(
ctx context.Context, ctx context.Context,
filters []*model.JobFilter) (int, error) { filters []*model.JobFilter,
) (int, error) {
query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
if qerr != nil { if qerr != nil {
return 0, qerr return 0, qerr
@ -227,9 +227,7 @@ func buildStringCondition(field string, cond *model.StringInput, query sq.Select
} }
if cond.In != nil { if cond.In != nil {
queryElements := make([]string, len(cond.In)) queryElements := make([]string, len(cond.In))
for i, val := range cond.In { copy(queryElements, cond.In)
queryElements[i] = val
}
return query.Where(sq.Or{sq.Eq{field: queryElements}}) return query.Where(sq.Or{sq.Eq{field: queryElements}})
} }
return query return query
@ -257,8 +255,10 @@ func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.
return query return query
} }
var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") var (
var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
)
func toSnakeCase(str string) string { func toSnakeCase(str string) string {
for _, c := range str { for _, c := range str {

38
pkg/schema/cluster.go
View File

@ -33,35 +33,37 @@ type SubCluster struct {
Name string `json:"name"` Name string `json:"name"`
Nodes string `json:"nodes"` Nodes string `json:"nodes"`
ProcessorType string `json:"processorType"` ProcessorType string `json:"processorType"`
SocketsPerNode int `json:"socketsPerNode"` Topology Topology `json:"topology"`
CoresPerSocket int `json:"coresPerSocket"`
ThreadsPerCore int `json:"threadsPerCore"`
FlopRateScalar MetricValue `json:"flopRateScalar"` FlopRateScalar MetricValue `json:"flopRateScalar"`
FlopRateSimd MetricValue `json:"flopRateSimd"` FlopRateSimd MetricValue `json:"flopRateSimd"`
MemoryBandwidth MetricValue `json:"memoryBandwidth"` MemoryBandwidth MetricValue `json:"memoryBandwidth"`
Topology Topology `json:"topology"` SocketsPerNode int `json:"socketsPerNode"`
CoresPerSocket int `json:"coresPerSocket"`
ThreadsPerCore int `json:"threadsPerCore"`
} }
type SubClusterConfig struct { type SubClusterConfig struct {
Name string `json:"name"` Name string `json:"name"`
Peak float64 `json:"peak"` Peak float64 `json:"peak"`
Normal float64 `json:"normal"` Normal float64 `json:"normal"`
Caution float64 `json:"caution"` Caution float64 `json:"caution"`
Alert float64 `json:"alert"` Alert float64 `json:"alert"`
Remove bool `json:"remove"` Footprint bool `json:"footprint"`
Remove bool `json:"remove"`
} }
type MetricConfig struct { type MetricConfig struct {
Name string `json:"name"`
Unit Unit `json:"unit"` Unit Unit `json:"unit"`
Name string `json:"name"`
Scope MetricScope `json:"scope"` Scope MetricScope `json:"scope"`
Aggregation string `json:"aggregation"` Aggregation string `json:"aggregation"`
SubClusters []*SubClusterConfig `json:"subClusters,omitempty"`
Timestep int `json:"timestep"` Timestep int `json:"timestep"`
Peak float64 `json:"peak"` Peak float64 `json:"peak"`
Normal float64 `json:"normal"` Normal float64 `json:"normal"`
Caution float64 `json:"caution"` Caution float64 `json:"caution"`
Alert float64 `json:"alert"` Alert float64 `json:"alert"`
SubClusters []*SubClusterConfig `json:"subClusters,omitempty"` Footprint bool `json:"footprint"`
} }
type Cluster struct { type Cluster struct {
@ -76,8 +78,8 @@ type Cluster struct {
// return value, return true as the second value. TODO: Optimize this, there // return value, return true as the second value. TODO: Optimize this, there
// must be a more efficient way/algorithm. // must be a more efficient way/algorithm.
func (topo *Topology) GetSocketsFromHWThreads( func (topo *Topology) GetSocketsFromHWThreads(
hwthreads []int) (sockets []int, exclusive bool) { hwthreads []int,
) (sockets []int, exclusive bool) {
socketsMap := map[int]int{} socketsMap := map[int]int{}
for _, hwthread := range hwthreads { for _, hwthread := range hwthreads {
for socket, hwthreadsInSocket := range topo.Socket { for socket, hwthreadsInSocket := range topo.Socket {
@ -106,8 +108,8 @@ func (topo *Topology) GetSocketsFromHWThreads(
// return value, return true as the second value. TODO: Optimize this, there // return value, return true as the second value. TODO: Optimize this, there
// must be a more efficient way/algorithm. // must be a more efficient way/algorithm.
func (topo *Topology) GetCoresFromHWThreads( func (topo *Topology) GetCoresFromHWThreads(
hwthreads []int) (cores []int, exclusive bool) { hwthreads []int,
) (cores []int, exclusive bool) {
coresMap := map[int]int{} coresMap := map[int]int{}
for _, hwthread := range hwthreads { for _, hwthread := range hwthreads {
for core, hwthreadsInCore := range topo.Core { for core, hwthreadsInCore := range topo.Core {
@ -136,8 +138,8 @@ func (topo *Topology) GetCoresFromHWThreads(
// memory domains in the first return value, return true as the second value. // memory domains in the first return value, return true as the second value.
// TODO: Optimize this, there must be a more efficient way/algorithm. // TODO: Optimize this, there must be a more efficient way/algorithm.
func (topo *Topology) GetMemoryDomainsFromHWThreads( func (topo *Topology) GetMemoryDomainsFromHWThreads(
hwthreads []int) (memDoms []int, exclusive bool) { hwthreads []int,
) (memDoms []int, exclusive bool) {
memDomsMap := map[int]int{} memDomsMap := map[int]int{}
for _, hwthread := range hwthreads { for _, hwthread := range hwthreads {
for memDom, hwthreadsInmemDom := range topo.MemoryDomain { for memDom, hwthreadsInmemDom := range topo.MemoryDomain {

85
pkg/schema/job.go
View File

@ -16,30 +16,31 @@ import (
// Common subset of Job and JobMeta. Use one of those, not this type directly. // Common subset of Job and JobMeta. Use one of those, not this type directly.
type BaseJob struct { type BaseJob struct {
// The unique identifier of a job Footprint map[string]float64 `json:"footPrint"`
JobID int64 `json:"jobId" db:"job_id" example:"123000"` MetaData map[string]string `json:"metaData"`
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user Cluster string `json:"cluster" db:"cluster" example:"fritz"`
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project SubCluster string `json:"subCluster" db:"subcluster" example:"main"`
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster Partition string `json:"partition,omitempty" db:"partition" example:"main"`
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster Project string `json:"project" db:"project" example:"abcd200"`
Partition string `json:"partition,omitempty" db:"partition" example:"main"` // The Slurm partition to which the job was submitted User string `json:"user" db:"user" example:"abcd100h"`
ArrayJobId int64 `json:"arrayJobId,omitempty" db:"array_job_id" example:"123000"` // The unique identifier of an array job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"`
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) Tags []*Tag `json:"tags,omitempty"`
// NumCores int32 `json:"numCores" db:"num_cores" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) RawFootprint []byte `json:"-" db:"footprint"`
NumHWThreads int32 `json:"numHwthreads,omitempty" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) RawMetaData []byte `json:"-" db:"meta_data"`
NumAcc int32 `json:"numAcc,omitempty" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) Resources []*Resource `json:"resources"`
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user RawResources []byte `json:"-" db:"resources"`
MonitoringStatus int32 `json:"monitoringStatus,omitempty" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull ConcurrentJobs JobLinkResultList `json:"concurrentJobs"`
SMT int32 `json:"smt,omitempty" db:"smt" example:"4"` // SMT threads used by job Energy float64 `json:"energy"`
State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job ArrayJobId int64 `json:"arrayJobId,omitempty" db:"array_job_id" example:"123000"`
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) Walltime int64 `json:"walltime,omitempty" db:"walltime" example:"86400" minimum:"1"`
Walltime int64 `json:"walltime,omitempty" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) JobID int64 `json:"jobId" db:"job_id" example:"123000"`
Tags []*Tag `json:"tags,omitempty"` // List of tags Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"`
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] SMT int32 `json:"smt,omitempty" db:"smt" example:"4"`
Resources []*Resource `json:"resources"` // Resources used by job MonitoringStatus int32 `json:"monitoringStatus,omitempty" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"`
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"`
MetaData map[string]string `json:"metaData"` // Additional information about the job NumAcc int32 `json:"numAcc,omitempty" db:"num_acc" example:"2" minimum:"1"`
ConcurrentJobs JobLinkResultList `json:"concurrentJobs"` NumHWThreads int32 `json:"numHwthreads,omitempty" db:"num_hwthreads" example:"20" minimum:"1"`
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"`
} }
// Job struct type // Job struct type
@ -49,19 +50,10 @@ type BaseJob struct {
// Job model // Job model
// @Description Information of a HPC job. // @Description Information of a HPC job.
type Job struct { type Job struct {
// The unique identifier of a job in the database StartTime time.Time `json:"startTime"`
ID int64 `json:"id" db:"id"`
BaseJob BaseJob
StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds ID int64 `json:"id" db:"id"`
StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"`
MemUsedMax float64 `json:"memUsedMax" db:"mem_used_max"` // MemUsedMax as Float64
FlopsAnyAvg float64 `json:"flopsAnyAvg" db:"flops_any_avg"` // FlopsAnyAvg as Float64
MemBwAvg float64 `json:"memBwAvg" db:"mem_bw_avg"` // MemBwAvg as Float64
LoadAvg float64 `json:"loadAvg" db:"load_avg"` // LoadAvg as Float64
NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64
NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64
FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64
FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64
} }
// JobMeta struct type // JobMeta struct type
@ -88,11 +80,10 @@ type JobLinkResultList struct {
// JobMeta model // JobMeta model
// @Description Meta data information of a HPC job. // @Description Meta data information of a HPC job.
type JobMeta struct { type JobMeta struct {
// The unique identifier of a job in the database ID *int64 `json:"id,omitempty"`
ID *int64 `json:"id,omitempty"` Statistics map[string]JobStatistics `json:"statistics"`
BaseJob BaseJob
StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"`
Statistics map[string]JobStatistics `json:"statistics"` // Metric statistics of job
} }
const ( const (
@ -124,18 +115,18 @@ type JobStatistics struct {
// Tag model // Tag model
// @Description Defines a tag using name and type. // @Description Defines a tag using name and type.
type Tag struct { type Tag struct {
ID int64 `json:"id" db:"id"` // The unique DB identifier of a tag Type string `json:"type" db:"tag_type" example:"Debug"`
Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type Name string `json:"name" db:"tag_name" example:"Testjob"`
Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name ID int64 `json:"id" db:"id"`
} }
// Resource model // Resource model
// @Description A resource used by a job // @Description A resource used by a job
type Resource struct { type Resource struct {
Hostname string `json:"hostname"` // Name of the host (= node) Hostname string `json:"hostname"`
HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids Configuration string `json:"configuration,omitempty"`
Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids HWThreads []int `json:"hwthreads,omitempty"`
Configuration string `json:"configuration,omitempty"` // The configuration options of the node Accelerators []string `json:"accelerators,omitempty"`
} }
type JobState string type JobState string