initial branch commit, add job compare switch, add gql resolver

api/schema.graphqls

@@ -158,7 +158,7 @@ type StatsSeries {
   max:    [NullableFloat!]!
 }
 
-type JobStatsWithScope {
+type NamedStatsWithScope {
   name:   String!
   scope:  MetricScope!
   stats:  [ScopedStats!]!
@@ -171,8 +171,13 @@ type ScopedStats {
 }
 
 type JobStats {
-  name:   String!
+  jobId: Int!
-  stats:  MetricStatistics!
+  stats: [NamedStats!]!
+}
+
+type NamedStats {
+  name:  String!
+  data:  MetricStatistics!
 }
 
 type Unit {
@@ -259,12 +264,13 @@ type Query {
 
   job(id: ID!): Job
   jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]!
-  jobStats(id: ID!, metrics: [String!]): [JobStats!]!
+  jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
-  scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobStatsWithScope!]!
+  scopedJobStats(id: ID!, metrics: [String!], scopes: [MetricScope!]): [NamedStatsWithScope!]!
-  jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
 
   jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
   jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
+  jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]!
+  jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
 
   rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
 
@@ -288,6 +294,7 @@ type TimeRangeOutput { range: String, from: Time!, to: Time! }
 input JobFilter {
   tags:        [ID!]
   jobId:       StringInput
+  jobIds:      [ID!]
   arrayJobId:  Int
   user:        StringInput
   project:     StringInput

internal/graph/model/models_gen.go

@@ -51,6 +51,7 @@ type IntRangeOutput struct {
 type JobFilter struct {
 	Tags            []string          `json:"tags,omitempty"`
 	JobID           *StringInput      `json:"jobId,omitempty"`
+	JobIds          []string          `json:"jobIds,omitempty"`
 	ArrayJobID      *int              `json:"arrayJobId,omitempty"`
 	User            *StringInput      `json:"user,omitempty"`
 	Project         *StringInput      `json:"project,omitempty"`
@@ -96,14 +97,8 @@ type JobResultList struct {
 }
 
 type JobStats struct {
-	Name  string                   `json:"name"`
+	JobID int           `json:"jobId"`
-	Stats *schema.MetricStatistics `json:"stats"`
+	Stats []*NamedStats `json:"stats"`
-}
-
-type JobStatsWithScope struct {
-	Name  string             `json:"name"`
-	Scope schema.MetricScope `json:"scope"`
-	Stats []*ScopedStats     `json:"stats"`
 }
 
 type JobsStatistics struct {
@@ -153,6 +148,17 @@ type MetricStatItem struct {
 type Mutation struct {
 }
 
+type NamedStats struct {
+	Name string                   `json:"name"`
+	Data *schema.MetricStatistics `json:"data"`
+}
+
+type NamedStatsWithScope struct {
+	Name  string             `json:"name"`
+	Scope schema.MetricScope `json:"scope"`
+	Stats []*ScopedStats     `json:"stats"`
+}
+
 type NodeMetrics struct {
 	Host       string               `json:"host"`
 	SubCluster string               `json:"subCluster"`

internal/graph/schema.resolvers.go

@@ -400,7 +400,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 }
 
 // JobStats is the resolver for the jobStats field.
-func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.JobStats, error) {
+func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) {
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
 		log.Warnf("Error while querying job %s for metadata", id)
@@ -413,11 +413,11 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin
 		return nil, err
 	}
 
-	res := []*model.JobStats{}
+	res := []*model.NamedStats{}
 	for name, md := range data {
-		res = append(res, &model.JobStats{
+		res = append(res, &model.NamedStats{
-			Name:  name,
+			Name: name,
-			Stats: &md,
+			Data: &md,
 		})
 	}
 
@@ -425,7 +425,7 @@ func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []strin
 }
 
 // ScopedJobStats is the resolver for the scopedJobStats field.
-func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobStatsWithScope, error) {
+func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) {
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
 		log.Warnf("Error while querying job %s for metadata", id)
@@ -438,7 +438,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
 		return nil, err
 	}
 
-	res := make([]*model.JobStatsWithScope, 0)
+	res := make([]*model.NamedStatsWithScope, 0)
 	for name, scoped := range data {
 		for scope, stats := range scoped {
 
@@ -451,7 +451,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
 				})
 			}
 
-			res = append(res, &model.JobStatsWithScope{
+			res = append(res, &model.NamedStatsWithScope{
 				Name:  name,
 				Scope: scope,
 				Stats: mdlStats,
@@ -462,12 +462,6 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
 	return res, nil
 }
 
-// JobsFootprints is the resolver for the jobsFootprints field.
-func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
-	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
-	return r.jobsFootprints(ctx, filter, metrics)
-}
-
 // Jobs is the resolver for the jobs field.
 func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) {
 	if page == nil {
@@ -589,6 +583,52 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	return stats, nil
 }
 
+// JobsMetricStats is the resolver for the jobsMetricStats field.
+func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) {
+	// No Paging, Fixed Order by StartTime ASC
+	order := &model.OrderByInput{
+		Field: "startTime",
+		Type:  "col",
+		Order: "ASC",
+	}
+
+	jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order)
+	if err != nil {
+		log.Warn("Error while querying jobs for comparison")
+		return nil, err
+	}
+
+	res := []*model.JobStats{}
+	for _, job := range jobs {
+		data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
+		if err != nil {
+			log.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID)
+			continue
+			// return nil, err
+		}
+
+		sres := []*model.NamedStats{}
+		for name, md := range data {
+			sres = append(sres, &model.NamedStats{
+				Name: name,
+				Data: &md,
+			})
+		}
+
+		res = append(res, &model.JobStats{
+			JobID: int(job.JobID),
+			Stats: sres,
+		})
+	}
+	return res, err
+}
+
+// JobsFootprints is the resolver for the jobsFootprints field.
+func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
+	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
+	return r.jobsFootprints(ctx, filter, metrics)
+}
+
 // RooflineHeatmap is the resolver for the rooflineHeatmap field.
 func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
 	return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY)

web/frontend/src/Jobs.root.svelte

@@ -21,6 +21,7 @@
   import { init } from "./generic/utils.js";
   import Filters from "./generic/Filters.svelte";
   import JobList from "./generic/JobList.svelte";
+  import JobCompare from "./generic/JobCompare.svelte";
   import TextFilter from "./generic/helper/TextFilter.svelte";
   import Refresher from "./generic/helper/Refresher.svelte";
   import Sorting from "./generic/select/SortSelection.svelte";
@@ -36,7 +37,9 @@
 
   let filterComponent; // see why here: https://stackoverflow.com/questions/58287729/how-can-i-export-a-function-from-a-svelte-component-that-changes-a-value-in-the
   let jobList,
-    matchedJobs = null;
+      jobCompare,
+    matchedListJobs,
+    matchedCompareJobs = null;
   let sorting = { field: "startTime", type: "col", order: "DESC" },
     isSortingOpen = false,
     isMetricsSelectionOpen = false;
@@ -49,6 +52,7 @@
     : !!ccconfig.plot_list_showFootprint;
   let selectedCluster = filterPresets?.cluster ? filterPresets.cluster : null;
   let presetProject = filterPresets?.project ? filterPresets.project : ""
+  let showCompare = false;
 
   // The filterPresets are handled by the Filters component,
   // so we need to wait for it to be ready before we can start a query.
@@ -72,7 +76,7 @@
 {/if}
 
 <!-- ROW2: Tools-->
-<Row cols={{ xs: 1, md: 2, lg: 4}} class="mb-3">
+<Row cols={{ xs: 1, md: 2, lg: 5}} class="mb-3">
   <Col lg="2" class="mb-2 mb-lg-0">
     <ButtonGroup class="w-100">
       <Button outline color="primary" on:click={() => (isSortingOpen = true)}>
@@ -87,20 +91,24 @@
       </Button>
     </ButtonGroup>
   </Col>
-  <Col lg="4" xl="{(presetProject !== '') ? 5 : 6}" class="mb-1 mb-lg-0">
+  <Col lg="4" class="mb-1 mb-lg-0">
     <Filters
       {filterPresets}
-      {matchedJobs}
+      matchedJobs={showCompare? matchedCompareJobs: matchedListJobs}
       bind:this={filterComponent}
       on:update-filters={({ detail }) => {
         selectedCluster = detail.filters[0]?.cluster
           ? detail.filters[0].cluster.eq
           : null;
-        jobList.queryJobs(detail.filters);
+        if (showCompare) {
+          jobCompare.queryJobs(detail.filters);
+        } else {
+          jobList.queryJobs(detail.filters);
+        }
       }}
     />
   </Col>
-  <Col lg="3" xl="{(presetProject !== '') ? 3 : 2}" class="mb-2 mb-lg-0">
+  <Col lg="2" class="mb-2 mb-lg-0">
     <TextFilter
       {presetProject}
       bind:authlevel
@@ -108,24 +116,44 @@
       on:set-filter={({ detail }) => filterComponent.updateFilters(detail)}
     />
   </Col>
-  <Col lg="3" xl="2" class="mb-1 mb-lg-0">
+  <Col lg="2" class="mb-1 mb-lg-0">
     <Refresher on:refresh={() => {
-      jobList.refreshJobs()
+      if (showCompare) {
-      jobList.refreshAllMetrics()
+        jobCompare.refreshJobs()
+        jobCompare.refreshAllMetrics()
+      } else {
+        jobList.refreshJobs()
+        jobList.refreshAllMetrics()
+      }
     }} />
   </Col>
+  <Col lg="2" class="mb-2 mb-lg-0">
+    <Button color="primary" on:click={() => {
+      showCompare = !showCompare
+    }} >
+      {showCompare ? 'Compare' : 'List'} Jobs
+    </Button>
+  </Col>
 </Row>
 
-<!-- ROW3: Job List-->
+<!-- ROW3: Job List / Job Compare-->
 <Row>
   <Col>
-    <JobList
+    {#if !showCompare}
-      bind:this={jobList}
+      <JobList
-      bind:metrics
+        bind:this={jobList}
-      bind:sorting
+        bind:metrics
-      bind:matchedJobs
+        bind:sorting
-      bind:showFootprint
+        bind:matchedListJobs
-    />
+        bind:showFootprint
+      />
+    {:else}
+      <JobCompare
+        bind:this={jobCompare}
+        bind:metrics
+        bind:matchedCompareJobs
+      />
+    {/if}
   </Col>
 </Row>
 

web/frontend/src/generic/JobCompare.svelte

@@ -0,0 +1,156 @@
+<!--
+    @component jobCompare component; compares jobs according to set filters
+
+    Properties:
+    - `sorting Object?`: Currently active sorting [Default: {field: "startTime", type: "col", order: "DESC"}]
+    - `matchedJobs Number?`: Number of matched jobs for selected filters [Default: 0]
+    - `metrics [String]?`: The currently selected metrics [Default: User-Configured Selection]
+    - `showFootprint Bool`: If to display the jobFootprint component
+
+    Functions:
+    - `refreshJobs()`: Load jobs data with unchanged parameters and 'network-only' keyword
+    - `refreshAllMetrics()`: Trigger downstream refresh of all running jobs' metric data
+    - `queryJobs(filters?: [JobFilter])`: Load jobs data with new filters, starts from page 1
+ -->
+
+<script>
+  import { getContext } from "svelte";
+  import {
+    queryStore,
+    gql,
+    getContextClient,
+    mutationStore,
+  } from "@urql/svelte";
+  import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap";
+
+  const ccconfig = getContext("cc-config"),
+    initialized = getContext("initialized"),
+    globalMetrics = getContext("globalMetrics");
+
+  const equalsCheck = (a, b) => {
+    return JSON.stringify(a) === JSON.stringify(b);
+  }
+
+  export let matchedCompareJobs = 0;
+  export let metrics = ccconfig.plot_list_selectedMetrics;
+
+  let filter = [];
+  const sorting = { field: "startTime", type: "col", order: "DESC" };
+
+  /* GQL */
+
+  const client = getContextClient();
+  // Pull All Series For Metrics Statistics Only On Node Scope
+  const compareQuery = gql`
+  query ($filter: [JobFilter!]!, $metrics: [String!]!) {
+    jobsMetricStats(filter: $filter, metrics: $metrics) {
+      jobId
+      stats {
+        name
+        data {
+          min
+          avg
+          max
+        }
+      }
+    }
+  }
+  `;
+
+  $: compareData = queryStore({
+    client: client,
+    query: compareQuery,
+    variables:{ filter, metrics },
+  });
+
+  $: matchedCompareJobs = $compareData.data != null ? $compareData.data.jobsMetricStats.length : -1;
+
+ /* FUNCTIONS */
+   // Force refresh list with existing unchanged variables (== usually would not trigger reactivity)
+   export function refreshJobs() {
+    compareData = queryStore({
+      client: client,
+      query: compareQuery,
+      variables: { filter, metrics },
+      requestPolicy: "network-only",
+    });
+  }
+
+  export function refreshAllMetrics() {
+    // Refresh Job Metrics (Downstream will only query for running jobs)
+    triggerMetricRefresh = true
+    setTimeout(function () {
+      triggerMetricRefresh = false;
+    }, 100);
+  }
+
+  // (Re-)query and optionally set new filters; Query will be started reactively.
+  export function queryJobs(filters) {
+    if (filters != null) {
+      let minRunningFor = ccconfig.plot_list_hideShortRunningJobs;
+      if (minRunningFor && minRunningFor > 0) {
+        filters.push({ minRunningFor });
+      }
+      filter = filters;
+    }
+  }
+
+  // Adapt for Persisting Job Selections in DB later down the line
+  // const updateConfigurationMutation = ({ name, value }) => {
+  //   return mutationStore({
+  //     client: client,
+  //     query: gql`
+  //       mutation ($name: String!, $value: String!) {
+  //         updateConfiguration(name: $name, value: $value)
+  //       }
+  //     `,
+  //     variables: { name, value },
+  //   });
+  // };
+
+  // function updateConfiguration(value, page) {
+  //   updateConfigurationMutation({
+  //     name: "plot_list_jobsPerPage",
+  //     value: value,
+  //   }).subscribe((res) => {
+  //     if (res.fetching === false && !res.error) {
+  //       jobs = [] // Empty List
+  //       paging = { itemsPerPage: value, page: page }; // Trigger reload of jobList
+  //     } else if (res.fetching === false && res.error) {
+  //       throw res.error;
+  //     }
+  //   });
+  // }
+
+</script>
+
+{#if $compareData.fetching}
+  <Row>
+    <Col>
+      <Spinner secondary />
+    </Col>
+  </Row>
+{:else if $compareData.error}
+  <Row>
+    <Col>
+      <Card body color="danger" class="mb-3"
+        ><h2>{$compareData.error.message}</h2></Card
+      >
+    </Col>
+  </Row>
+{:else}
+  {#each $compareData.data.jobsMetricStats as job (job.jobId)}
+    <Row>
+      <Col><b><i>{job.jobId}</i></b></Col>
+      {#each job.stats as stat (stat.name)}
+        <Col><b>{stat.name}</b></Col>
+        <Col>Min {stat.data.min}</Col>
+        <Col>Avg {stat.data.avg}</Col>
+        <Col>Max {stat.data.max}</Col>
+      {/each}
+    </Row>
+    <hr/>
+  {:else}
+    <div> No jobs found </div>
+  {/each}
+{/if}

web/frontend/src/generic/JobList.svelte

@@ -35,7 +35,7 @@
   }
 
   export let sorting = { field: "startTime", type: "col", order: "DESC" };
-  export let matchedJobs = 0;
+  export let matchedListJobs = 0;
   export let metrics = ccconfig.plot_list_selectedMetrics;
   export let showFootprint;
 
@@ -141,7 +141,7 @@
     }
   }
 
-  $: matchedJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1;
+  $: matchedListJobs = $jobsStore.data != null ? $jobsStore.data.jobs.count : -1;
 
   // Force refresh list with existing unchanged variables (== usually would not trigger reactivity)
   export function refreshJobs() {
@@ -310,7 +310,7 @@
     bind:page
     {itemsPerPage}
     itemText="Jobs"
-    totalItems={matchedJobs}
+    totalItems={matchedListJobs}
     on:update-paging={({ detail }) => {
       if (detail.itemsPerPage != itemsPerPage) {
         updateConfiguration(detail.itemsPerPage.toString(), detail.page);

web/frontend/src/generic/plots/Polar.svelte

@@ -55,7 +55,7 @@
     const getValues = (type) => labels.map(name => {
         // Peak is adapted and scaled for job shared state
         const peak = polarMetrics.find(m => m?.name == name)?.peak
-        const metric = polarData.find(m => m?.name == name)?.stats
+        const metric = polarData.find(m => m?.name == name)?.data
         const value = (peak && metric) ? (metric[type] / peak) : 0
         return value <= 1. ? value : 1.
     })

web/frontend/src/job/jobsummary/JobFootprintPolar.svelte

@@ -42,7 +42,7 @@
     query ($dbid: ID!, $selectedMetrics: [String!]!) {
       jobStats(id: $dbid, metrics: $selectedMetrics) {
         name
-        stats {
+        data {
           min
           avg
           max