Merge pull request #172 from ClusterCockpit/hotfix

Revert check for null in concurrent job list
This commit is contained in:
Jan Eitzinger 2023-07-03 09:40:15 +02:00 committed by GitHub
commit 185f7144b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -1,21 +1,37 @@
<script> <script>
import { init, groupByScope, fetchMetricsStore, checkMetricDisabled } from './utils.js' import {
import { Row, Col, Card, Spinner, TabContent, TabPane, init,
CardBody, CardHeader, CardTitle, Button, Icon } from 'sveltestrap' groupByScope,
import PlotTable from './PlotTable.svelte' fetchMetricsStore,
import Metric from './Metric.svelte' checkMetricDisabled,
import PolarPlot from './plots/Polar.svelte' } from "./utils.js";
import Roofline from './plots/Roofline.svelte' import {
import JobInfo from './joblist/JobInfo.svelte' Row,
import TagManagement from './TagManagement.svelte' Col,
import MetricSelection from './MetricSelection.svelte' Card,
import Zoom from './Zoom.svelte' Spinner,
import StatsTable from './StatsTable.svelte' TabContent,
import { getContext } from 'svelte' TabPane,
CardBody,
CardHeader,
CardTitle,
Button,
Icon,
} from "sveltestrap";
import PlotTable from "./PlotTable.svelte";
import Metric from "./Metric.svelte";
import PolarPlot from "./plots/Polar.svelte";
import Roofline from "./plots/Roofline.svelte";
import JobInfo from "./joblist/JobInfo.svelte";
import TagManagement from "./TagManagement.svelte";
import MetricSelection from "./MetricSelection.svelte";
import Zoom from "./Zoom.svelte";
import StatsTable from "./StatsTable.svelte";
import { getContext } from "svelte";
export let dbid export let dbid;
export let authlevel export let authlevel;
export let roles export let roles;
const { query: initq } = init(` const { query: initq } = init(`
job(id: "${dbid}") { job(id: "${dbid}") {
@ -29,146 +45,237 @@
userData { name, email }, userData { name, email },
concurrentJobs { items { id, jobId }, count, listQuery } concurrentJobs { items { id, jobId }, count, listQuery }
} }
`) `);
const ccconfig = getContext('cc-config'), const ccconfig = getContext("cc-config"),
clusters = getContext('clusters') clusters = getContext("clusters");
let isMetricsSelectionOpen = false, selectedMetrics = [], isFetched = new Set() let isMetricsSelectionOpen = false,
const [jobMetrics, startFetching] = fetchMetricsStore() selectedMetrics = [],
getContext('on-init')(() => { isFetched = new Set();
let job = $initq.data.job const [jobMetrics, startFetching] = fetchMetricsStore();
if (!job) getContext("on-init")(() => {
return let job = $initq.data.job;
if (!job) return;
selectedMetrics = ccconfig[`job_view_selectedMetrics:${job.cluster}`] selectedMetrics =
|| clusters.find(c => c.name == job.cluster).metricConfig.map(mc => mc.name) ccconfig[`job_view_selectedMetrics:${job.cluster}`] ||
clusters
.find((c) => c.name == job.cluster)
.metricConfig.map((mc) => mc.name);
let toFetch = new Set([ let toFetch = new Set([
'flops_any', 'mem_bw', "flops_any",
"mem_bw",
...selectedMetrics, ...selectedMetrics,
...(ccconfig[`job_view_polarPlotMetrics:${job.cluster}`] || ccconfig[`job_view_polarPlotMetrics`]), ...(ccconfig[`job_view_polarPlotMetrics:${job.cluster}`] ||
...(ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] || ccconfig[`job_view_nodestats_selectedMetrics`])]) ccconfig[`job_view_polarPlotMetrics`]),
...(ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] ||
ccconfig[`job_view_nodestats_selectedMetrics`]),
]);
// Select default Scopes to load // Select default Scopes to load
if (job.numAcc === 0) { // No Accels if (job.numAcc === 0) {
startFetching(job, [...toFetch], job.numNodes > 2 ? ["node"] : ["node", "core"]) // No Accels
} else { // Accels startFetching(
startFetching(job, [...toFetch], job.numNodes > 2 ? ["node", "accelerator"] : ["node", "accelerator", "core"]) job,
[...toFetch],
job.numNodes > 2 ? ["node"] : ["node", "core"]
);
} else {
// Accels
startFetching(
job,
[...toFetch],
job.numNodes > 2
? ["node", "accelerator"]
: ["node", "accelerator", "core"]
);
} }
isFetched = toFetch isFetched = toFetch;
}) });
const lazyFetchMoreMetrics = () => { const lazyFetchMoreMetrics = () => {
let notYetFetched = new Set() let notYetFetched = new Set();
for (let m of selectedMetrics) { for (let m of selectedMetrics) {
if (!isFetched.has(m)) { if (!isFetched.has(m)) {
notYetFetched.add(m) notYetFetched.add(m);
isFetched.add(m) isFetched.add(m);
} }
} }
if (notYetFetched.size > 0) if (notYetFetched.size > 0)
startFetching($initq.data.job, [...notYetFetched], $initq.data.job.numNodes > 2 ? ["node"] : ["node", "core"]) startFetching(
} $initq.data.job,
[...notYetFetched],
$initq.data.job.numNodes > 2 ? ["node"] : ["node", "core"]
);
};
// Fetch more data once required: // Fetch more data once required:
$: if ($initq.data && $jobMetrics.data && selectedMetrics) lazyFetchMoreMetrics(); $: if ($initq.data && $jobMetrics.data && selectedMetrics)
lazyFetchMoreMetrics();
let plots = {}, jobTags, fullWidth, statsTable let plots = {},
$: polarPlotSize = Math.min(fullWidth / 3 - 10, 300) jobTags,
$: document.title = $initq.fetching ? 'Loading...' : ($initq.error ? 'Error' : `Job ${$initq.data.job.jobId} - ClusterCockpit`) fullWidth,
statsTable;
$: polarPlotSize = Math.min(fullWidth / 3 - 10, 300);
$: document.title = $initq.fetching
? "Loading..."
: $initq.error
? "Error"
: `Job ${$initq.data.job.jobId} - ClusterCockpit`;
// Find out what metrics or hosts are missing: // Find out what metrics or hosts are missing:
let missingMetrics = [], missingHosts = [], somethingMissing = false let missingMetrics = [],
missingHosts = [],
somethingMissing = false;
$: if ($initq.data && $jobMetrics.data) { $: if ($initq.data && $jobMetrics.data) {
let job = $initq.data.job, let job = $initq.data.job,
metrics = $jobMetrics.data.jobMetrics, metrics = $jobMetrics.data.jobMetrics,
metricNames = clusters.find(c => c.name == job.cluster).metricConfig.map(mc => mc.name) metricNames = clusters
.find((c) => c.name == job.cluster)
.metricConfig.map((mc) => mc.name);
// Metric not found in JobMetrics && Metric not explicitly disabled: Was expected, but is Missing // Metric not found in JobMetrics && Metric not explicitly disabled: Was expected, but is Missing
missingMetrics = metricNames.filter(metric => (!metrics.some(jm => jm.name == metric) && !checkMetricDisabled(metric, $initq.data.job.cluster, $initq.data.job.subCluster))) missingMetrics = metricNames.filter(
missingHosts = job.resources.map(({ hostname }) => ({ (metric) =>
hostname: hostname, !metrics.some((jm) => jm.name == metric) &&
metrics: metricNames.filter(metric => !metrics.some(jm => jm.scope == 'node' && jm.metric.series.some(series => series.hostname == hostname))) !checkMetricDisabled(
})).filter(({ metrics }) => metrics.length > 0) metric,
somethingMissing = missingMetrics.length > 0 || missingHosts.length > 0 $initq.data.job.cluster,
$initq.data.job.subCluster
)
);
missingHosts = job.resources
.map(({ hostname }) => ({
hostname: hostname,
metrics: metricNames.filter(
(metric) =>
!metrics.some(
(jm) =>
jm.scope == "node" &&
jm.metric.series.some(
(series) => series.hostname == hostname
)
)
),
}))
.filter(({ metrics }) => metrics.length > 0);
somethingMissing = missingMetrics.length > 0 || missingHosts.length > 0;
} }
const orderAndMap = (grouped, selectedMetrics) => const orderAndMap = (grouped, selectedMetrics) =>
selectedMetrics.map(metric => ({ selectedMetrics.map((metric) => ({
metric: metric, metric: metric,
data: grouped.find((group) => data: grouped.find((group) => group[0].name == metric),
group[0].name == metric disabled: checkMetricDisabled(
metric,
$initq.data.job.cluster,
$initq.data.job.subCluster
), ),
disabled: checkMetricDisabled(metric, $initq.data.job.cluster, $initq.data.job.subCluster) }));
}))
</script> </script>
<div class="row" bind:clientWidth={fullWidth}></div> <div class="row" bind:clientWidth={fullWidth} />
<Row> <Row>
<Col> <Col>
{#if $initq.error} {#if $initq.error}
<Card body color="danger">{$initq.error.message}</Card> <Card body color="danger">{$initq.error.message}</Card>
{:else if $initq.data} {:else if $initq.data}
<JobInfo job={$initq.data.job} jobTags={jobTags}/> <JobInfo job={$initq.data.job} {jobTags} />
{:else} {:else}
<Spinner secondary/> <Spinner secondary />
{/if} {/if}
</Col> </Col>
{#if $jobMetrics.data && $initq.data} {#if $jobMetrics.data && $initq.data}
{#if $initq.data.job.concurrentJobs.items.length != 0} {#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0}
{#if authlevel > roles.manager} {#if authlevel > roles.manager}
<Col> <Col>
<h5>Concurrent Jobs <Icon name="info-circle" style="cursor:help;" title="Shared jobs running on the same node with overlapping runtimes"/></h5> <h5>
Concurrent Jobs <Icon
name="info-circle"
style="cursor:help;"
title="Shared jobs running on the same node with overlapping runtimes"
/>
</h5>
<ul> <ul>
<li><a href="/monitoring/jobs/?{$initq.data.job.concurrentJobs.listQuery}" target="_blank">See All</a></li> <li>
{#each $initq.data.job.concurrentJobs.items as pjob, index} <a
<li><a href="/monitoring/job/{pjob.id}" target="_blank">{pjob.jobId}</a></li> href="/monitoring/jobs/?{$initq.data.job
{/each} .concurrentJobs.listQuery}"
target="_blank">See All</a
>
</li>
{#each $initq.data.job.concurrentJobs.items as pjob, index}
<li>
<a
href="/monitoring/job/{pjob.id}"
target="_blank">{pjob.jobId}</a
>
</li>
{/each}
</ul> </ul>
</Col> </Col>
{:else} {:else}
<Col> <Col>
<h5>{$initq.data.job.concurrentJobs.items.length} Concurrent Jobs</h5> <h5>
<p>Number of shared jobs on the same node with overlapping runtimes.</p> {$initq.data.job.concurrentJobs.items.length} Concurrent
Jobs
</h5>
<p>
Number of shared jobs on the same node with overlapping
runtimes.
</p>
</Col> </Col>
{/if} {/if}
{/if} {/if}
<Col> <Col>
<PolarPlot <PolarPlot
width={polarPlotSize} height={polarPlotSize} width={polarPlotSize}
metrics={ccconfig[`job_view_polarPlotMetrics:${$initq.data.job.cluster}`] || ccconfig[`job_view_polarPlotMetrics`]} height={polarPlotSize}
metrics={ccconfig[
`job_view_polarPlotMetrics:${$initq.data.job.cluster}`
] || ccconfig[`job_view_polarPlotMetrics`]}
cluster={$initq.data.job.cluster} cluster={$initq.data.job.cluster}
jobMetrics={$jobMetrics.data.jobMetrics}/> jobMetrics={$jobMetrics.data.jobMetrics}
/>
</Col> </Col>
<Col> <Col>
<Roofline <Roofline
width={fullWidth / 3 - 10} height={polarPlotSize} width={fullWidth / 3 - 10}
height={polarPlotSize}
cluster={clusters cluster={clusters
.find(c => c.name == $initq.data.job.cluster).subClusters .find((c) => c.name == $initq.data.job.cluster)
.find(sc => sc.name == $initq.data.job.subCluster)} .subClusters.find(
flopsAny={$jobMetrics.data.jobMetrics.find(m => m.name == 'flops_any' && m.scope == 'node')} (sc) => sc.name == $initq.data.job.subCluster
memBw={$jobMetrics.data.jobMetrics.find(m => m.name == 'mem_bw' && m.scope == 'node')} /> )}
flopsAny={$jobMetrics.data.jobMetrics.find(
(m) => m.name == "flops_any" && m.scope == "node"
)}
memBw={$jobMetrics.data.jobMetrics.find(
(m) => m.name == "mem_bw" && m.scope == "node"
)}
/>
</Col> </Col>
{:else} {:else}
<Col></Col> <Col />
<Col></Col> <Col />
{/if} {/if}
</Row> </Row>
<br/> <br />
<Row> <Row>
<Col xs="auto"> <Col xs="auto">
{#if $initq.data} {#if $initq.data}
<TagManagement job={$initq.data.job} bind:jobTags={jobTags}/> <TagManagement job={$initq.data.job} bind:jobTags />
{/if} {/if}
</Col> </Col>
<Col xs="auto"> <Col xs="auto">
{#if $initq.data} {#if $initq.data}
<Button outline <Button outline on:click={() => (isMetricsSelectionOpen = true)}>
on:click={() => (isMetricsSelectionOpen = true)}> <Icon name="graph-up" /> Metrics
<Icon name="graph-up"/> Metrics
</Button> </Button>
{/if} {/if}
</Col> </Col>
@ -176,97 +283,139 @@
<Zoom timeseriesPlots={plots} /> <Zoom timeseriesPlots={plots} />
</Col> </Col>
</Row> </Row>
<br/> <br />
<Row> <Row>
<Col> <Col>
{#if $jobMetrics.error} {#if $jobMetrics.error}
{#if $initq.data.job.monitoringStatus == 0 || $initq.data.job.monitoringStatus == 2} {#if $initq.data.job.monitoringStatus == 0 || $initq.data.job.monitoringStatus == 2}
<Card body color="warning">Not monitored or archiving failed</Card> <Card body color="warning"
<br/> >Not monitored or archiving failed</Card
>
<br />
{/if} {/if}
<Card body color="danger">{$jobMetrics.error.message}</Card> <Card body color="danger">{$jobMetrics.error.message}</Card>
{:else if $jobMetrics.fetching} {:else if $jobMetrics.fetching}
<Spinner secondary/> <Spinner secondary />
{:else if $jobMetrics.data && $initq.data} {:else if $jobMetrics.data && $initq.data}
<PlotTable <PlotTable
let:item let:item
let:width let:width
renderFor="job" renderFor="job"
items={orderAndMap(groupByScope($jobMetrics.data.jobMetrics), selectedMetrics)} items={orderAndMap(
itemsPerRow={ccconfig.plot_view_plotsPerRow}> groupByScope($jobMetrics.data.jobMetrics),
selectedMetrics
)}
itemsPerRow={ccconfig.plot_view_plotsPerRow}
>
{#if item.data} {#if item.data}
<Metric <Metric
bind:this={plots[item.metric]} bind:this={plots[item.metric]}
on:more-loaded={({ detail }) => statsTable.moreLoaded(detail)} on:more-loaded={({ detail }) =>
statsTable.moreLoaded(detail)}
job={$initq.data.job} job={$initq.data.job}
metricName={item.metric} metricName={item.metric}
rawData={item.data.map(x => x.metric)} rawData={item.data.map((x) => x.metric)}
scopes={item.data.map(x => x.scope)} scopes={item.data.map((x) => x.scope)}
width={width} {width}
isShared={($initq.data.job.exclusive != 1)}/> isShared={$initq.data.job.exclusive != 1}
/>
{:else} {:else}
<Card body color="warning">No dataset returned for <code>{item.metric}</code></Card> <Card body color="warning"
>No dataset returned for <code>{item.metric}</code
></Card
>
{/if} {/if}
</PlotTable> </PlotTable>
{/if} {/if}
</Col> </Col>
</Row> </Row>
<br/> <br />
<Row> <Row>
<Col> <Col>
{#if $initq.data} {#if $initq.data}
<TabContent> <TabContent>
{#if somethingMissing} {#if somethingMissing}
<TabPane tabId="resources" tab="Resources" active={somethingMissing}> <TabPane
<div style="margin: 10px;"><Card color="warning"> tabId="resources"
<CardHeader> tab="Resources"
<CardTitle>Missing Metrics/Reseources</CardTitle> active={somethingMissing}
</CardHeader> >
<CardBody> <div style="margin: 10px;">
{#if missingMetrics.length > 0} <Card color="warning">
<p>No data at all is available for the metrics: {missingMetrics.join(', ')}</p> <CardHeader>
{/if} <CardTitle
{#if missingHosts.length > 0} >Missing Metrics/Reseources</CardTitle
<p>Some metrics are missing for the following hosts:</p> >
<ul> </CardHeader>
{#each missingHosts as missing} <CardBody>
<li>{missing.hostname}: {missing.metrics.join(', ')}</li> {#if missingMetrics.length > 0}
{/each} <p>
</ul> No data at all is available for the
{/if} metrics: {missingMetrics.join(", ")}
</CardBody> </p>
</Card></div> {/if}
</TabPane> {#if missingHosts.length > 0}
{/if} <p>
<TabPane tabId="stats" tab="Statistics Table" active={!somethingMissing}> Some metrics are missing for the
{#if $jobMetrics.data} following hosts:
{#key $jobMetrics.data} </p>
<StatsTable <ul>
bind:this={statsTable} {#each missingHosts as missing}
job={$initq.data.job} <li>
jobMetrics={$jobMetrics.data.jobMetrics} /> {missing.hostname}: {missing.metrics.join(
{/key} ", "
)}
</li>
{/each}
</ul>
{/if}
</CardBody>
</Card>
</div>
</TabPane>
{/if} {/if}
</TabPane> <TabPane
<TabPane tabId="job-script" tab="Job Script"> tabId="stats"
<div class="pre-wrapper"> tab="Statistics Table"
{#if $initq.data.job.metaData?.jobScript} active={!somethingMissing}
<pre><code>{$initq.data.job.metaData?.jobScript}</code></pre> >
{:else} {#if $jobMetrics.data}
<Card body color="warning">No job script available</Card> {#key $jobMetrics.data}
<StatsTable
bind:this={statsTable}
job={$initq.data.job}
jobMetrics={$jobMetrics.data.jobMetrics}
/>
{/key}
{/if} {/if}
</div> </TabPane>
</TabPane> <TabPane tabId="job-script" tab="Job Script">
<TabPane tabId="slurm-info" tab="Slurm Info"> <div class="pre-wrapper">
<div class="pre-wrapper"> {#if $initq.data.job.metaData?.jobScript}
{#if $initq.data.job.metaData?.slurmInfo} <pre><code
<pre><code>{$initq.data.job.metaData?.slurmInfo}</code></pre> >{$initq.data.job.metaData?.jobScript}</code
{:else} ></pre>
<Card body color="warning">No additional slurm information available</Card> {:else}
{/if} <Card body color="warning"
</div> >No job script available</Card
</TabPane> >
</TabContent> {/if}
</div>
</TabPane>
<TabPane tabId="slurm-info" tab="Slurm Info">
<div class="pre-wrapper">
{#if $initq.data.job.metaData?.slurmInfo}
<pre><code
>{$initq.data.job.metaData?.slurmInfo}</code
></pre>
{:else}
<Card body color="warning"
>No additional slurm information available</Card
>
{/if}
</div>
</TabPane>
</TabContent>
{/if} {/if}
</Col> </Col>
</Row> </Row>
@ -276,7 +425,8 @@
cluster={$initq.data.job.cluster} cluster={$initq.data.job.cluster}
configName="job_view_selectedMetrics" configName="job_view_selectedMetrics"
bind:metrics={selectedMetrics} bind:metrics={selectedMetrics}
bind:isOpen={isMetricsSelectionOpen} /> bind:isOpen={isMetricsSelectionOpen}
/>
{/if} {/if}
<style> <style>