add scopes, paging and backend filtering to nodeList

web/frontend/src/Systems.root.svelte

@@ -10,7 +10,6 @@
 
 <script>
   import { getContext } from "svelte";
-  import { queryStore, gql, getContextClient } from "@urql/svelte"
   import {
     Row,
     Col,
@@ -20,10 +19,9 @@
     InputGroupText,
     Icon,
     Button,
-    Spinner,
   } from "@sveltestrap/sveltestrap";
 
-  import { init, checkMetricsDisabled } from "./generic/utils.js";
+  import { init } from "./generic/utils.js";
   import NodeOverview from "./systems/NodeOverview.svelte";
   import NodeList from "./systems/NodeList.svelte";
   import MetricSelection from "./generic/select/MetricSelection.svelte";
@@ -32,6 +30,7 @@
 
   export let displayType;
   export let cluster;
+  export let subCluster = "";
   export let from = null;
   export let to = null;
 
@@ -45,7 +44,7 @@
   if (from == null || to == null) {
     to = new Date(Date.now());
     from = new Date(to.getTime());
-    from.setHours(from.getHours() - 2);
+    from.setHours(from.getHours() - 12);
   }
 
   const initialized = getContext("initialized");
@@ -58,79 +57,15 @@
   let selectedMetrics = ccconfig[`node_list_selectedMetrics:${cluster}`] || [ccconfig.system_view_selectedMetric];
   let isMetricsSelectionOpen = false;
 
-  // New Jan 2025
   /*
-  - Toss "add_resolution_node_systems" branch OR include/merge here if resolutions in node-overview useful
-  - Add single object field for nodeData query to CCMS query: "nodeDataQuery"
-    - Contains following fields:
-      - metrics: [String]  // List of metrics to query
-      - page: Int          // Page number
-      - itemsPerPage: Int  // Number of items per page
-      - resolution: Int    // Requested Resolution for all returned data
-      - nodeFilter: String // (partial) hostname string
-    - With this, all use-cases except "scopes" can be handled, if nodeFilter is "" (empty) all nodes are returned by default
-    - Is basically a stepped up version of the "forAllNodes" property, as "these metrics for all nodes" is still the base idea
-  - Required: Handling in CCMS, co-develop in close contact with Aditya
-  - Question: How and where to handle scope queries? (e.g. "node" vs "accelerator") -> NOT handled in ccms!
-  - NOtes: "Sorting" as use-case ignored for now, probably default to alphanumerical on hostnames of cluster
+    Note 1: Scope Selector or Auto-Scoped?
+    Note 2: "Sorting" as use-case ignored for now, probably default to alphanumerical on hostnames of cluster
+    Note 3: Add Idle State Filter (== No allocated Jobs) [Frontend?] : Cannot be handled by CCMS, requires secondary job query and refiltering of visible nodes
   */
 
-  // Todo: Add Idle State Filter (== No allocated Jobs) [Frontend?] : Cannot be handled by CCMS, requires secondary job query and refiltering of visible nodes
-  // Todo: NodeList: Mindestens Accelerator Scope ... "Show Detail" Switch?
-  // Todo: Rework GQL Query: Add Paging (Scrollable / Paging Configbar), Add Nodes Filter (see jobs-onthefly-userfilter: ccms inkompatibel!), add scopes
-  //       All three issues need either new features in ccms (paging, filter) or new implementation of ccms node queries with scopes (currently very job-specific)
-  // Todo: Review performance // observed high client-side load frequency
-  //       Is Svelte {#each} -> <MetricPlot/> -> onMount() related : Cannot be skipped ...
-  //       Will be solved as soon as dedicated paging, itemLimits and filtering is implemented in ccms
-  // ==> Skip for Q4/24 Release, build from ccms upgrade (paging/filter) up
-  
-  const client = getContextClient();
-  const nodeQuery = gql`
-    query ($cluster: String!, $metrics: [String!], $from: Time!, $to: Time!) {
-      nodeMetrics(
-        cluster: $cluster
-        metrics: $metrics
-        from: $from
-        to: $to
-      ) {
-        host
-        subCluster
-        metrics {
-          name
-          scope
-          metric {
-            timestep
-            unit {
-              base
-              prefix
-            }
-            series {
-              statistics {
-                min
-                avg
-                max
-              }
-              data
-            }
-          }
-        }
-      }
-    }
-  `
-
-  $: nodesQuery = queryStore({
-    client: client,
-    query: nodeQuery,
-    variables: {
-      cluster: cluster,
-      metrics: selectedMetrics,
-      from: from.toISOString(),
-      to: to.toISOString(),
-    },
-  });
-
   let systemMetrics = [];
   let systemUnits = {};
+
   function loadMetrics(isInitialized) {
     if (!isInitialized) return
     systemMetrics = [...globalMetrics.filter((gm) => gm?.availability.find((av) => av.cluster == cluster))]
@@ -145,43 +80,6 @@
     selectedMetrics = [selectedMetric]
   }
 
-  let rawData = []
-  $: if ($initq.data && $nodesQuery?.data) {
-    rawData = $nodesQuery?.data?.nodeMetrics.filter((h) => {
-      if (h.subCluster === '') { // Exclude nodes with empty subCluster field
-        console.warn('subCluster not configured for node', h.host)
-        return false
-      } else {
-        return h.metrics.some(
-          (m) => selectedMetrics.includes(m.name) && m.scope == "node",
-        )
-      }
-    })
-  }
-
-  let mappedData = []
-  $: if (rawData?.length > 0) {
-    mappedData = rawData.map((h) => ({
-      host: h.host,
-      subCluster: h.subCluster,
-      data: h.metrics.filter(
-        (m) => selectedMetrics.includes(m.name) && m.scope == "node",
-      ),
-      disabled: checkMetricsDisabled(
-        selectedMetrics,
-        cluster,
-        h.subCluster,
-      ),
-    }))
-    .sort((a, b) => a.host.localeCompare(b.host))
-  }
-
-  let filteredData = []
-  $: if (mappedData?.length > 0) {
-    filteredData = mappedData.filter((h) =>
-      h.host.includes(hostnameFilter)
-    )
-  }
 </script>
 
 <!-- ROW1: Tools-->
@@ -255,25 +153,13 @@
       <Card body color="danger">Unknown displayList type! </Card>
     </Col>
   </Row>
-{:else if $nodesQuery.error}
-  <Row>
-    <Col>
-      <Card body color="danger">{$nodesQuery.error.message}</Card>
-    </Col>
-  </Row>
-{:else if $nodesQuery.fetching }
-  <Row>
-    <Col>
-      <Spinner />
-    </Col>
-  </Row>
-{:else if filteredData?.length > 0}
+{:else}
   {#if displayNodeOverview}
     <!-- ROW2-1: Node Overview (Grid Included)-->
-    <NodeOverview {cluster} {ccconfig} data={filteredData}/>
+    <NodeOverview {cluster} {subCluster} {ccconfig} {selectedMetrics} {from} {to} {hostnameFilter}/>
   {:else}
     <!-- ROW2-2: Node List (Grid Included)-->
-    <NodeList {cluster} {selectedMetrics} {systemUnits} data={filteredData} bind:selectedMetric/>
+    <NodeList {cluster} {subCluster} {ccconfig} {selectedMetrics} {hostnameFilter} {from} {to} {systemUnits}/>
   {/if}
 {/if}
 

web/frontend/src/generic/plots/MetricPlot.svelte

@@ -553,7 +553,7 @@
 </script>
 
 <!-- Define $width Wrapper and NoData Card -->
-{#if series[0].data.length > 0}
+{#if series[0]?.data && series[0].data.length > 0}
   <div bind:this={plotWrapper} bind:clientWidth={width}
         style="background-color: {backgroundColor()};" class={forNode ? 'py-2 rounded' : 'rounded'}
   />

web/frontend/src/systems.entrypoint.js

@@ -6,6 +6,7 @@ new Systems({
     props: {
         displayType: displayType,
         cluster: infos.cluster,
+        subCluster: infos.subCluster,
         from: infos.from,
         to: infos.to
     },

web/frontend/src/systems/NodeList.svelte

@@ -3,68 +3,168 @@
 
     Properties:
     - `cluster String`: The nodes' cluster
-    - `data [Object]`: The node data array for all nodes
+    - `subCluster String`: The nodes' subCluster
+    - `ccconfig Object?`: The ClusterCockpit Config Context [Default: null]
     - `selectedMetrics [String]`: The array of selected metrics
-    - `selectedMetrics Object`: The object of metric units
+    - `systemUnits Object`: The object of metric units
  -->
 
 <script>
-  import { Row, Table } from "@sveltestrap/sveltestrap";
-  import {
-    stickyHeader 
-  } from "../generic/utils.js";
+  import { queryStore, gql, getContextClient } from "@urql/svelte";
+  import { Row, Col, Card, Table, Spinner } from "@sveltestrap/sveltestrap";
+  import { init, stickyHeader } from "../generic/utils.js";
   import NodeListRow from "./nodelist/NodeListRow.svelte";
+  import Pagination from "../generic/joblist/Pagination.svelte";
 
   export let cluster;
-  export let data = null;
+  export let subCluster = "";
+  export const ccconfig = null;
   export let selectedMetrics = [];
+  export let hostnameFilter = "";
   export let systemUnits = null;
+  export let from = null;
+  export let to = null;
 
-  // Always use ONE BIG list, but: Make copyable markers -> Nodeinfo ! (like in markdown)
+  // let usePaging = ccconfig.node_list_usePaging
+  let itemsPerPage = 10 // usePaging ? ccconfig.node_list_jobsPerPage : 10;
+  let page = 1;
+  let paging = { itemsPerPage, page };
 
   let headerPaddingTop = 0;
   stickyHeader(
     ".cc-table-wrapper > table.table >thead > tr > th.position-sticky:nth-child(1)",
     (x) => (headerPaddingTop = x),
   );
+
+  const { query: initq } = init();
+  const client = getContextClient();
+  const nodeListQuery = gql`
+    query ($cluster: String!, $subCluster: String!, $nodeFilter: String!, $metrics: [String!], $scopes: [MetricScope!]!, $from: Time!, $to: Time!, $paging: PageRequest!) {
+      nodeMetricsList(
+        cluster: $cluster
+        subCluster: $subCluster
+        nodeFilter: $nodeFilter
+        scopes: $scopes
+        metrics: $metrics
+        from: $from
+        to: $to
+        page: $paging
+      ) {
+        items {
+          host
+          subCluster
+          metrics {
+            name
+            scope
+            metric {
+              timestep
+              unit {
+                base
+                prefix
+              }
+              series {
+                statistics {
+                  min
+                  avg
+                  max
+                }
+                data
+              }
+            }
+          }
+        }
+        totalNodes
+        hasNextPage
+      }
+    }
+  `
+
+  $: nodesQuery = queryStore({
+    client: client,
+    query: nodeListQuery,
+    variables: {
+      cluster: cluster,
+      subCluster: subCluster,
+      nodeFilter: hostnameFilter,
+      scopes: ["core", "accelerator"],
+      metrics: selectedMetrics,
+      from: from.toISOString(),
+      to: to.toISOString(),
+      paging: paging,
+    },
+  });
+
+  $: matchedNodes = $nodesQuery.data?.nodeMetricsList.totalNodes || 0;
 </script>
 
-<Row>
-  <div class="col cc-table-wrapper">
-    <Table cellspacing="0px" cellpadding="0px">
-      <thead>
-        <tr>
-          <th
-            class="position-sticky top-0 text-capitalize"
-            scope="col"
-            style="padding-top: {headerPaddingTop}px;"
-          >
-            {cluster} Node Info
-          </th>
-
-          {#each selectedMetrics as metric (metric)}
-            <th
-              class="position-sticky top-0 text-center"
-              scope="col"
-              style="padding-top: {headerPaddingTop}px"
-            >
-              {metric} ({systemUnits[metric]})
-            </th>
-          {/each}
-        </tr>
-      </thead>
-      <tbody>
-        {#each data as nodeData (nodeData.host)}
-          <NodeListRow {nodeData} {cluster} {selectedMetrics}/>
-        {:else}
+{#if $nodesQuery.error}
+  <Row>
+    <Col>
+      <Card body color="danger">{$nodesQuery.error.message}</Card>
+    </Col>
+  </Row>
+{:else if $nodesQuery.fetching }
+  <Row>
+    <Col>
+      <Spinner />
+    </Col>
+  </Row>
+{:else if $initq?.data && $nodesQuery?.data}
+  <Row>
+    <div class="col cc-table-wrapper">
+      <Table cellspacing="0px" cellpadding="0px">
+        <thead>
           <tr>
-            <td>No nodes found </td>
+            <th
+              class="position-sticky top-0 text-capitalize"
+              scope="col"
+              style="padding-top: {headerPaddingTop}px;"
+            >
+              {cluster} Node Info
+            </th>
+
+            {#each selectedMetrics as metric (metric)}
+              <th
+                class="position-sticky top-0 text-center"
+                scope="col"
+                style="padding-top: {headerPaddingTop}px"
+              >
+                {metric} ({systemUnits[metric]})
+              </th>
+            {/each}
           </tr>
-        {/each}
-      </tbody>
-    </Table>
-  </div>
-</Row>
+        </thead>
+        <tbody>
+          {#each $nodesQuery.data.nodeMetricsList.items as nodeData (nodeData.host)}
+            <NodeListRow {nodeData} {cluster} {selectedMetrics}/>
+          {:else}
+            <tr>
+              <td>No nodes found </td>
+            </tr>
+          {/each}
+        </tbody>
+      </Table>
+    </div>
+  </Row>
+{/if}
+
+{#if true} <!-- usePaging -->
+  <Pagination
+    bind:page
+    {itemsPerPage}
+    itemText="Nodes"
+    totalItems={matchedNodes}
+    on:update-paging={({ detail }) => {
+      paging = { itemsPerPage: detail.itemsPerPage, page: detail.page }
+      // if (detail.itemsPerPage != itemsPerPage) {
+      //   updateConfiguration(detail.itemsPerPage.toString(), detail.page);
+      // } else {
+      //   // nodes = []
+      //   paging = { itemsPerPage: detail.itemsPerPage, page: detail.page };
+      // }
+    }}
+  />
+{/if}
 
 <style>
   .cc-table-wrapper {

web/frontend/src/systems/NodeOverview.svelte

@@ -3,50 +3,153 @@
 
     Properties:
     - `ccconfig Object?`: The ClusterCockpit Config Context [Default: null]
-    - `data Object?`: The GQL nodeMetrics data [Default: null]
     - `cluster String`: The cluster to show status information for
     - `selectedMetric String?`: The selectedMetric input [Default: ""]
  -->
 
  <script>
-  import { Row, Col, Card } from "@sveltestrap/sveltestrap";
+  import { queryStore, gql, getContextClient } from "@urql/svelte";
+  import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap";
+  import { init, checkMetricsDisabled } from "../generic/utils.js";
   import MetricPlot from "../generic/plots/MetricPlot.svelte";
 
   export let ccconfig = null;
-  export let data = null;
   export let cluster = "";
-  export let selectedMetric = "";
+  export const subCluster = "";
+  export let selectedMetrics = null;
+  export let hostnameFilter = "";
+  export let from = null;
+  export let to = null;
 
+  const { query: initq } = init();
+  const client = getContextClient();
+  const nodeQuery = gql`
+    query ($cluster: String!, $metrics: [String!], $from: Time!, $to: Time!) {
+      nodeMetrics(
+        cluster: $cluster
+        metrics: $metrics
+        from: $from
+        to: $to
+      ) {
+        host
+        subCluster
+        metrics {
+          name
+          scope
+          metric {
+            timestep
+            unit {
+              base
+              prefix
+            }
+            series {
+              statistics {
+                min
+                avg
+                max
+              }
+              data
+            }
+          }
+        }
+      }
+    }
+  `
+
+  $: selectedMetric = selectedMetrics[0] ? selectedMetrics[0] : "";
+
+  $: nodesQuery = queryStore({
+    client: client,
+    query: nodeQuery,
+    variables: {
+      cluster: cluster,
+      metrics: selectedMetrics,
+      from: from.toISOString(),
+      to: to.toISOString(),
+    },
+  });
+
+  let rawData = []
+  $: if ($initq.data && $nodesQuery?.data) {
+    rawData = $nodesQuery?.data?.nodeMetrics.filter((h) => {
+      if (h.subCluster === '') { // Exclude nodes with empty subCluster field
+        console.warn('subCluster not configured for node', h.host)
+        return false
+      } else {
+        return h.metrics.some(
+          (m) => selectedMetrics.includes(m.name) && m.scope == "node",
+        )
+      }
+    })
+  }
+
+  let mappedData = []
+  $: if (rawData?.length > 0) {
+    mappedData = rawData.map((h) => ({
+      host: h.host,
+      subCluster: h.subCluster,
+      data: h.metrics.filter(
+        (m) => selectedMetrics.includes(m.name) && m.scope == "node",
+      ),
+      disabled: checkMetricsDisabled(
+        selectedMetrics,
+        cluster,
+        h.subCluster,
+      ),
+    }))
+    .sort((a, b) => a.host.localeCompare(b.host))
+  }
+
+  let filteredData = []
+  $: if (mappedData?.length > 0) {
+    filteredData = mappedData.filter((h) =>
+      h.host.includes(hostnameFilter)
+    )
+  }
 </script>
 
-<!-- PlotGrid flattened into this component -->
-<Row cols={{ xs: 1, sm: 2, md: 3, lg: ccconfig.plot_view_plotsPerRow}}>
-  {#each data as item (item.host)}
-    <Col class="px-1">
-      <h4 style="width: 100%; text-align: center;">
-        <a
-          style="display: block;padding-top: 15px;"
-          href="/monitoring/node/{cluster}/{item.host}"
-          >{item.host} ({item.subCluster})</a
-        >
-      </h4>
-      {#if item?.disabled[selectedMetric]}
-        <Card body class="mx-3" color="info"
-          >Metric disabled for subcluster <code
-            >{selectedMetric}:{item.subCluster}</code
-          ></Card
-        >
-      {:else}
-        <!-- "No Data"-Warning included in MetricPlot-Component -->
-        <MetricPlot
-          timestep={item.data[0].metric.timestep}
-          series={item.data[0].metric.series}
-          metric={item.data[0].name}
-          {cluster}
-          subCluster={item.subCluster}
-          forNode
-        />
-      {/if}
+{#if $nodesQuery.error}
+  <Row>
+    <Col>
+      <Card body color="danger">{$nodesQuery.error.message}</Card>
     </Col>
-  {/each}
-</Row>
+  </Row>
+{:else if $nodesQuery.fetching }
+  <Row>
+    <Col>
+      <Spinner />
+    </Col>
+  </Row>
+{:else if filteredData?.length > 0}
+  <!-- PlotGrid flattened into this component -->
+  <Row cols={{ xs: 1, sm: 2, md: 3, lg: ccconfig.plot_view_plotsPerRow}}>
+    {#each filteredData as item (item.host)}
+      <Col class="px-1">
+        <h4 style="width: 100%; text-align: center;">
+          <a
+            style="display: block;padding-top: 15px;"
+            href="/monitoring/node/{cluster}/{item.host}"
+            >{item.host} ({item.subCluster})</a
+          >
+        </h4>
+        {#if item?.disabled[selectedMetric]}
+          <Card body class="mx-3" color="info"
+            >Metric disabled for subcluster <code
+              >{selectedMetric}:{item.subCluster}</code
+            ></Card
+          >
+        {:else}
+          <!-- "No Data"-Warning included in MetricPlot-Component -->
+          <MetricPlot
+            timestep={item.data[0].metric.timestep}
+            series={item.data[0].metric.series}
+            metric={item.data[0].name}
+            {cluster}
+            subCluster={item.subCluster}
+            forNode
+          />
+        {/if}
+      </Col>
+    {/each}
+  </Row>
+{/if}

web/frontend/src/systems/nodelist/NodeListRow.svelte

@@ -9,6 +9,7 @@
 
 <script>
   import { Card } from "@sveltestrap/sveltestrap";
+  import { maxScope, checkMetricDisabled } from "../../generic/utils.js";
   import MetricPlot from "../../generic/plots/MetricPlot.svelte";
   import NodeInfo from "./NodeInfo.svelte";
 
@@ -16,28 +17,55 @@
   export let nodeData;
   export let selectedMetrics;
 
-  const sortOrder = (nodeMetrics) =>
-    selectedMetrics.map((name) => nodeMetrics.find((nodeMetric) => nodeMetric.name == name));
+  // Helper
+  const selectScope = (nodeMetrics) =>
+    nodeMetrics.reduce(
+      (a, b) =>
+        maxScope([a.scope, b.scope]) == a.scope ? b : a,
+      nodeMetrics[0],
+    );
+
+  const sortAndSelectScope = (allNodeMetrics) =>
+    selectedMetrics
+      .map((selectedName) => allNodeMetrics.filter((nodeMetric) => nodeMetric.name == selectedName))
+      .map((matchedNodeMetrics) => ({
+        disabled: false,
+        data: matchedNodeMetrics.length > 0 ? selectScope(matchedNodeMetrics) : null,
+      }))
+      .map((scopedNodeMetric) => {
+        if (scopedNodeMetric?.data) {
+          return {
+            disabled: checkMetricDisabled(
+              scopedNodeMetric.data.name,
+              cluster,
+              nodeData.subCluster,
+            ),
+            data: scopedNodeMetric.data,
+          };
+        } else {
+          return scopedNodeMetric;
+        }
+      });
 </script>
 
 <tr>
   <td>
     <NodeInfo {cluster} subCluster={nodeData.subCluster} hostname={nodeData.host} />
   </td>
-  {#each sortOrder(nodeData?.data) as metricData (metricData.name)}
+  {#each sortAndSelectScope(nodeData?.metrics) as metricData (metricData.data.name)}
     <td>
-      {#if nodeData?.disabled[metricData.name]}
+      {#if metricData?.disabled}
         <Card body class="mx-3" color="info"
           >Metric disabled for subcluster <code
-            >{metricData.name}:{nodeData.subCluster}</code
+            >{metricData.data.name}:{nodeData.subCluster}</code
           ></Card
         >
       {:else}
         <!-- "No Data"-Warning included in MetricPlot-Component -->
         <MetricPlot
-          timestep={metricData.metric.timestep}
-          series={metricData.metric.series}
-          metric={metricData.name}
+          timestep={metricData.data.metric.timestep}
+          series={metricData.data.metric.series}
+          metric={metricData.data.name}
           {cluster}
           subCluster={nodeData.subCluster}
           forNode