ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-05-13 20:51:43 +02:00
Code Issues Packages Releases Activity

initial branch commit, add job compare switch, add gql resolver

Browse Source
This commit is contained in:
Christoph Kluge 2025-04-29 15:10:06 +02:00
parent f65e122f8d
commit df497d5952
9 changed files with 1050 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
api/schema.graphqls
View File

@ -158,7 +158,7 @@ type StatsSeries {
max: [NullableFloat!]!
}
type JobStatsWithScope {
type NamedStatsWithScope {
name: String!
scope: MetricScope!
stats: [ScopedStats!]!
@ -171,8 +171,13 @@ type ScopedStats {
}
type JobStats {
name: String!
stats: MetricStatistics!
jobId: Int!
stats: [NamedStats!]!
}
type NamedStats {
name: String!
data: MetricStatistics!
}
type Unit {
@ -259,12 +264,13 @@ type Query {
job(id: ID!): Job
jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]!
jobStats(id: ID!, metrics: [String!]): [JobStats!]!
scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]!
jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [NamedStatsWithScope!]!
jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]!
jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
@ -288,6 +294,7 @@ type TimeRangeOutput { range: String, from: Time!, to: Time! }
input JobFilter {
tags: [ID!]
jobId: StringInput
jobIds: [ID!]
arrayJobId: Int
user: StringInput
project: StringInput

1153
internal/graph/generated/generated.go
View File

File diff suppressed because it is too large Load Diff

22
internal/graph/model/models_gen.go
View File

@ -51,6 +51,7 @@ type IntRangeOutput struct {
type JobFilter struct {
Tags []string `json:"tags,omitempty"`
JobID *StringInput `json:"jobId,omitempty"`
JobIds []string `json:"jobIds,omitempty"`
ArrayJobID *int `json:"arrayJobId,omitempty"`
User *StringInput `json:"user,omitempty"`
Project *StringInput `json:"project,omitempty"`
@ -96,14 +97,8 @@ type JobResultList struct {
}
type JobStats struct {
Name string `json:"name"`
Stats *schema.MetricStatistics `json:"stats"`
}
type JobStatsWithScope struct {
Name string `json:"name"`
Scope schema.MetricScope `json:"scope"`
Stats []*ScopedStats `json:"stats"`
JobID int `json:"jobId"`
Stats []*NamedStats `json:"stats"`
}
type JobsStatistics struct {
@ -153,6 +148,17 @@ type MetricStatItem struct {
type Mutation struct {
}
type NamedStats struct {
Name string `json:"name"`
Data *schema.MetricStatistics `json:"data"`
}
type NamedStatsWithScope struct {
Name string `json:"name"`
Scope schema.MetricScope `json:"scope"`
Stats []*ScopedStats `json:"stats"`
}
type NodeMetrics struct {
Host string `json:"host"`
SubCluster string `json:"subCluster"`

68
internal/graph/schema.resolvers.go
View File

@ -400,7 +400,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
}
// JobStats is the resolver for the jobStats field.
func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) {
func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) {
job, err := r.Query().Job(ctx, id)
if err != nil {
log.Warnf("Error while querying job %s for metadata", id)
@ -413,11 +413,11 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin
return nil, err
}
res := []*model.JobStats{}
res := []*model.NamedStats{}
for name, md := range data {
res = append(res, &model.JobStats{
Name: name,
Stats: &md,
res = append(res, &model.NamedStats{
Name: name,
Data: &md,
})
}
@ -425,7 +425,7 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin
}
// ScopedJobStats is the resolver for the scopedJobStats field.
func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) {
func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) {
job, err := r.Query().Job(ctx, id)
if err != nil {
log.Warnf("Error while querying job %s for metadata", id)
@ -438,7 +438,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
return nil, err
}
res := make([]*model.JobStatsWithScope, 0)
res := make([]*model.NamedStatsWithScope, 0)
for name, scoped := range data {
for scope, stats := range scoped {
@ -451,7 +451,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
})
}
res = append(res, &model.JobStatsWithScope{
res = append(res, &model.NamedStatsWithScope{
Name: name,
Scope: scope,
Stats: mdlStats,
@ -462,12 +462,6 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
return res, nil
}
// JobsFootprints is the resolver for the jobsFootprints field.
func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
return r.jobsFootprints(ctx, filter, metrics)
}
// Jobs is the resolver for the jobs field.
func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) {
if page == nil {
@ -589,6 +583,52 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
return stats, nil
}
// JobsMetricStats is the resolver for the jobsMetricStats field.
func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) {
// No Paging, Fixed Order by StartTime ASC
order := &model.OrderByInput{
Field: "startTime",
Type: "col",
Order: "ASC",
}
jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order)
if err != nil {
log.Warn("Error while querying jobs for comparison")
return nil, err
}
res := []*model.JobStats{}
for _, job := range jobs {
data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
if err != nil {
log.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID)
continue
// return nil, err
}
sres := []*model.NamedStats{}
for name, md := range data {
sres = append(sres, &model.NamedStats{
Name: name,
Data: &md,
})
}
res = append(res, &model.JobStats{
JobID: int(job.JobID),
Stats: sres,
})
}
return res, err
}
// JobsFootprints is the resolver for the jobsFootprints field.
func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
return r.jobsFootprints(ctx, filter, metrics)
}
// RooflineHeatmap is the resolver for the rooflineHeatmap field.
func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY)

62
web/frontend/src/Jobs.root.svelte
View File

@ -21,6 +21,7 @@
import { init } from "./generic/utils.js";
import Filters from "./generic/Filters.svelte";
import JobList from "./generic/JobList.svelte";
import JobCompare from "./generic/JobCompare.svelte";
import TextFilter from "./generic/helper/TextFilter.svelte";
import Refresher from "./generic/helper/Refresher.svelte";
import Sorting from "./generic/select/SortSelection.svelte";
@ -36,7 +37,9 @@
let filterComponent; // see why here: https://stackoverflow.com/questions/58287729/how-can-i-export-a-function-from-a-svelte-component-that-changes-a-value-in-the
let jobList,
matchedJobs = null;
jobCompare,
matchedListJobs,
matchedCompareJobs = null;
let sorting = { field: "startTime", type: "col", order: "DESC" },
isSortingOpen = false,
isMetricsSelectionOpen = false;
@ -49,6 +52,7 @@
: !!ccconfig.plot_list_showFootprint;
let selectedCluster = filterPresets?.cluster ? filterPresets.cluster : null;
let presetProject = filterPresets?.project ? filterPresets.project : ""
let showCompare = false;
// The filterPresets are handled by the Filters component,
// so we need to wait for it to be ready before we can start a query.
@ -72,7 +76,7 @@
{/if}
<!-- ROW2: Tools-->
<Row cols={{ xs: 1, md: 2, lg: 4}} class="mb-3">
<Row cols={{ xs: 1, md: 2, lg: 5}} class="mb-3">
<Col lg="2" class="mb-2 mb-lg-0">
<ButtonGroup class="w-100">
<Button outline color="primary" on:click={() => (isSortingOpen = true)}>
@ -87,20 +91,24 @@
</Button>
</ButtonGroup>
</Col>
<Col lg="4" xl="{(presetProject !== '') ? 5 : 6}" class="mb-1 mb-lg-0">
<Col lg="4" class="mb-1 mb-lg-0">
<Filters
{filterPresets}
{matchedJobs}
matchedJobs={showCompare? matchedCompareJobs: matchedListJobs}
bind:this={filterComponent}
on:update-filters={({ detail }) => {
selectedCluster = detail.filters[0]?.cluster
? detail.filters[0].cluster.eq
: null;
jobList.queryJobs(detail.filters);
if (showCompare) {
jobCompare.queryJobs(detail.filters);
} else {
jobList.queryJobs(detail.filters);
}
}}
/>
</Col>
<Col lg="3" xl="{(presetProject !== '') ? 3 : 2}" class="mb-2 mb-lg-0">
<Col lg="2" class="mb-2 mb-lg-0">
<TextFilter
{presetProject}
bind:authlevel
@ -108,24 +116,44 @@
on:set-filter={({ detail }) => filterComponent.updateFilters(detail)}
/>
</Col>
<Col lg="3" xl="2" class="mb-1 mb-lg-0">
<Col lg="2" class="mb-1 mb-lg-0">
<Refresher on:refresh={() => {
jobList.refreshJobs()
jobList.refreshAllMetrics()
if (showCompare) {
jobCompare.refreshJobs()
jobCompare.refreshAllMetrics()
} else {
jobList.refreshJobs()
jobList.refreshAllMetrics()
}
}} />
</Col>
<Col lg="2" class="mb-2 mb-lg-0">
<Button color="primary" on:click={() => {
showCompare = !showCompare
}} >
{showCompare ? 'Compare' : 'List'} Jobs
</Button>
</Col>
</Row>
<!-- ROW3: Job List-->
<!-- ROW3: Job List / Job Compare-->
<Row>
<Col>
<JobList
bind:this={jobList}
bind:metrics
bind:sorting
bind:matchedJobs
bind:showFootprint
/>
{#if !showCompare}
<JobList
bind:this={jobList}
bind:metrics
bind:sorting
bind:matchedListJobs
bind:showFootprint
/>
{:else}
<JobCompare
bind:this={jobCompare}
bind:metrics
bind:matchedCompareJobs
/>
{/if}
</Col>
</Row>

156
web/frontend/src/generic/JobCompare.svelte Normal file
View File

@ -0,0 +1,156 @@
<!--
@component jobCompare component; compares jobs according to set filters
Properties:
- `sorting Object?`: Currently active sorting [Default: {field: "startTime", type: "col", order: "DESC"}]
- `matchedJobs Number?`: Number of matched jobs for selected filters [Default: 0]
- `metrics [String]?`: The currently selected metrics [Default: User-Configured Selection]
- `showFootprint Bool`: If to display the jobFootprint component
Functions:
- `refreshJobs()`: Load jobs data with unchanged parameters and 'network-only' keyword
- `refreshAllMetrics()`: Trigger downstream refresh of all running jobs' metric data
- `queryJobs(filters?: [JobFilter])`: Load jobs data with new filters, starts from page 1
-->
<script>
import { getContext } from "svelte";
import {
queryStore,
gql,
getContextClient,
mutationStore,
} from "@urql/svelte";
import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap";
const ccconfig = getContext("cc-config"),
initialized = getContext("initialized"),
globalMetrics = getContext("globalMetrics");
const equalsCheck = (a, b) => {
return JSON.stringify(a) === JSON.stringify(b);
}
export let matchedCompareJobs = 0;
export let metrics = ccconfig.plot_list_selectedMetrics;
let filter = [];
const sorting = { field: "startTime", type: "col", order: "DESC" };
/* GQL */
const client = getContextClient();
// Pull All Series For Metrics Statistics Only On Node Scope
const compareQuery = gql`
query ($filter: [JobFilter!]!, $metrics: [String!]!) {
jobsMetricStats(filter: $filter, metrics: $metrics) {
jobId
stats {
name
data {
min
avg
max
}
}
}
}
`;
$: compareData = queryStore({
client: client,
query: compareQuery,
variables:{ filter, metrics },
});
$: matchedCompareJobs = $compareData.data != null ? $compareData.data.jobsMetricStats.length : -1;
/* FUNCTIONS */
// Force refresh list with existing unchanged variables (== usually would not trigger reactivity)
export function refreshJobs() {
compareData = queryStore({
client: client,
query: compareQuery,
variables: { filter, metrics },
requestPolicy: "network-only",
});
}
export function refreshAllMetrics() {
// Refresh Job Metrics (Downstream will only query for running jobs)
triggerMetricRefresh = true
setTimeout(function () {
triggerMetricRefresh = false;
}, 100);
}
// (Re-)query and optionally set new filters; Query will be started reactively.
export function queryJobs(filters) {
if (filters != null) {
let minRunningFor = ccconfig.plot_list_hideShortRunningJobs;
if (minRunningFor && minRunningFor > 0) {
filters.push({ minRunningFor });
}
filter = filters;
}
}
// Adapt for Persisting Job Selections in DB later down the line
// const updateConfigurationMutation = ({ name, value }) => {
// return mutationStore({
// client: client,
// query: gql`
// mutation ($name: String!, $value: String!) {
// updateConfiguration(name: $name, value: $value)
// }
// `,
// variables: { name, value },
// });
// };
// function updateConfiguration(value, page) {
// updateConfigurationMutation({
// name: "plot_list_jobsPerPage",
// value: value,
// }).subscribe((res) => {
// if (res.fetching === false && !res.error) {
// jobs = [] // Empty List
// paging = { itemsPerPage: value, page: page }; // Trigger reload of jobList
// } else if (res.fetching === false && res.error) {
// throw res.error;
// }
// });
// }
</script>
{#if $compareData.fetching}
<Row>
<Col>
<Spinner secondary />
</Col>
</Row>
{:else if $compareData.error}
<Row>
<Col>
<Card body color="danger" class="mb-3"
><h2>{$compareData.error.message}</h2></Card
>
</Col>
</Row>
{:else}
{#each $compareData.data.jobsMetricStats as job (job.jobId)}
<Row>
<Col><b><i>{job.jobId}</i></b></Col>
{#each job.stats as stat (stat.name)}
<Col><b>{stat.name}</b></Col>
<Col>Min {stat.data.min}</Col>
<Col>Avg {stat.data.avg}</Col>
<Col>Max {stat.data.max}</Col>
{/each}
</Row>
<hr/>
{:else}
<div> No jobs found </div>
{/each}
{/if}

6
web/frontend/src/generic/JobList.svelte
View File

@ -35,7 +35,7 @@
}
export let sorting = { field: "startTime", type: "col", order: "DESC" };
export let matchedJobs = 0;
export let matchedListJobs = 0;
export let metrics = ccconfig.plot_list_selectedMetrics;
export let showFootprint;
@ -141,7 +141,7 @@
}
}
$: matchedJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1;
$: matchedListJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1;
// Force refresh list with existing unchanged variables (== usually would not trigger reactivity)
export function refreshJobs() {
@ -310,7 +310,7 @@
bind:page
{itemsPerPage}
itemText="Jobs"
totalItems={matchedJobs}
totalItems={matchedListJobs}
on:update-paging={({ detail }) => {
if (detail.itemsPerPage != itemsPerPage) {
updateConfiguration(detail.itemsPerPage.toString(), detail.page);

2
web/frontend/src/generic/plots/Polar.svelte
View File

@ -55,7 +55,7 @@
const getValues = (type) => labels.map(name => {
// Peak is adapted and scaled for job shared state
const peak = polarMetrics.find(m => m?.name == name)?.peak
const metric = polarData.find(m => m?.name == name)?.stats
const metric = polarData.find(m => m?.name == name)?.data
const value = (peak && metric) ? (metric[type] / peak) : 0
return value <= 1. ? value : 1.
})

2
web/frontend/src/job/jobsummary/JobFootprintPolar.svelte
View File

@ -42,7 +42,7 @@
query ($dbid: ID!, $selectedMetrics: [String!]!) {
jobStats(id: $dbid, metrics: $selectedMetrics) {
name
stats {
data {
min
avg
max