mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2026-08-31 00:47:15 +02:00
Merge pull request #569 from ClusterCockpit/metric-store-tickets
Metric store tickets
This commit is contained in:
@@ -1,84 +0,0 @@
|
|||||||
{
|
|
||||||
"hooks": {
|
|
||||||
"PostToolUse": [
|
|
||||||
{
|
|
||||||
"matcher": "Task",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code post-task"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"matcher": "TodoWrite",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code post-todo"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"PreToolUse": [
|
|
||||||
{
|
|
||||||
"matcher": "Task",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code pre-task"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"SessionEnd": [
|
|
||||||
{
|
|
||||||
"matcher": "",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code session-end"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"SessionStart": [
|
|
||||||
{
|
|
||||||
"matcher": "",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code session-start"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"Stop": [
|
|
||||||
{
|
|
||||||
"matcher": "",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code stop"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"UserPromptSubmit": [
|
|
||||||
{
|
|
||||||
"matcher": "",
|
|
||||||
"hooks": [
|
|
||||||
{
|
|
||||||
"type": "command",
|
|
||||||
"command": "entire hooks claude-code user-prompt-submit"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"permissions": {
|
|
||||||
"deny": [
|
|
||||||
"Read(./.entire/metadata/**)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
tmp/
|
|
||||||
settings.local.json
|
|
||||||
metadata/
|
|
||||||
logs/
|
|
||||||
redactors/local/
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"enabled": true,
|
|
||||||
"telemetry": true
|
|
||||||
}
|
|
||||||
@@ -30,3 +30,9 @@ dist/
|
|||||||
.idea
|
.idea
|
||||||
tools/archive-migration/archive-migration
|
tools/archive-migration/archive-migration
|
||||||
tools/archive-manager/archive-manager
|
tools/archive-manager/archive-manager
|
||||||
|
|
||||||
|
docs
|
||||||
|
|
||||||
|
|
||||||
|
.claude
|
||||||
|
.superpowers
|
||||||
|
|||||||
@@ -30,10 +30,12 @@ based on a GraphQL API. The web frontend is also served by the backend using
|
|||||||
[Bootstrap Icons](https://icons.getbootstrap.com/).
|
[Bootstrap Icons](https://icons.getbootstrap.com/).
|
||||||
|
|
||||||
The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database.
|
The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database.
|
||||||
While there are metric data backends for the InfluxDB and Prometheus time series
|
Time-series metric data is kept separately: cc-backend ships with a built-in
|
||||||
databases, the only tested and supported setup is to use cc-metric-store as the
|
in-memory metric store (configured via the `metric-store` section) that persists
|
||||||
metric data backend. Documentation on how to integrate ClusterCockpit with other
|
its buffers with a write-ahead log and periodic binary checkpoints. Alternatively
|
||||||
time series databases will be added in the future.
|
(or in addition, routed per cluster) it can query one or more external
|
||||||
|
[cc-metric-store](https://github.com/ClusterCockpit/cc-metric-store) instances via
|
||||||
|
the `metric-store-external` section. At least one of the two must be configured.
|
||||||
|
|
||||||
For real-time integration with HPC systems, the backend can subscribe to
|
For real-time integration with HPC systems, the backend can subscribe to
|
||||||
[NATS](https://nats.io/) subjects to receive job start/stop events and node
|
[NATS](https://nats.io/) subjects to receive job start/stop events and node
|
||||||
@@ -42,8 +44,9 @@ state updates, providing an alternative to REST API polling.
|
|||||||
Completed batch jobs are stored in a file-based job archive according to
|
Completed batch jobs are stored in a file-based job archive according to
|
||||||
[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/main/job-archive).
|
[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/main/job-archive).
|
||||||
The backend supports authentication via local accounts, an external LDAP
|
The backend supports authentication via local accounts, an external LDAP
|
||||||
directory, and JWT tokens. Authorization for APIs is implemented with
|
directory, an OpenID Connect provider, and JWT tokens. Authorization for APIs is
|
||||||
[JWT](https://jwt.io/) tokens created with public/private key encryption.
|
implemented with [JWT](https://jwt.io/) tokens created with public/private key
|
||||||
|
encryption.
|
||||||
|
|
||||||
You find a detailed documentation on the [ClusterCockpit
|
You find a detailed documentation on the [ClusterCockpit
|
||||||
Webpage](https://clustercockpit.org).
|
Webpage](https://clustercockpit.org).
|
||||||
@@ -77,13 +80,16 @@ Execute the following steps:
|
|||||||
|
|
||||||
```shell
|
```shell
|
||||||
./cc-backend -init
|
./cc-backend -init
|
||||||
vim config.json (Add a second cluster entry and name the clusters alex and fritz)
|
|
||||||
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
|
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
|
||||||
tar xf job-archive-demo.tar
|
tar xf job-archive-demo.tar
|
||||||
./cc-backend -init-db -add-user demo:admin:demo -loglevel info
|
./cc-backend -init-db -add-user demo:admin:demo -loglevel info
|
||||||
./cc-backend -server -dev -loglevel info
|
./cc-backend -server -dev -loglevel info
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The clusters are read from the job archive (one directory with a `cluster.json`
|
||||||
|
per cluster), so no cluster list has to be maintained in `config.json`. The demo
|
||||||
|
archive contains the clusters `alex` and `fritz`.
|
||||||
|
|
||||||
You can access the web interface at [http://localhost:8080](http://localhost:8080).
|
You can access the web interface at [http://localhost:8080](http://localhost:8080).
|
||||||
Credentials for login are `demo:demo`.
|
Credentials for login are `demo:demo`.
|
||||||
Please note that some views do not work without a metric backend (e.g., the
|
Please note that some views do not work without a metric backend (e.g., the
|
||||||
@@ -130,6 +136,8 @@ cd ./cc-backend/
|
|||||||
make
|
make
|
||||||
|
|
||||||
cp configs/config.json .
|
cp configs/config.json .
|
||||||
|
# config.json references the UI defaults via "ui-file", so copy that too:
|
||||||
|
cp configs/uiConfig.json .
|
||||||
# EDIT config.json BEFORE YOU DEPLOY: change the secrets under "auth.jwts"
|
# EDIT config.json BEFORE YOU DEPLOY: change the secrets under "auth.jwts"
|
||||||
# ("public-key"/"private-key"). Each secret can also be supplied via an
|
# ("public-key"/"private-key"). Each secret can also be supplied via an
|
||||||
# environment variable (e.g. JWT_PUBLIC_KEY), which takes precedence over the
|
# environment variable (e.g. JWT_PUBLIC_KEY), which takes precedence over the
|
||||||
@@ -172,6 +180,42 @@ For production deployments, serve cc-backend over HTTPS so the session cookie is
|
|||||||
marked `Secure`. If you terminate TLS at a reverse proxy, prefer letting
|
marked `Secure`. If you terminate TLS at a reverse proxy, prefer letting
|
||||||
cc-backend serve HTTPS directly for now so the flag is applied.
|
cc-backend serve HTTPS directly for now so the flag is applied.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
cc-backend reads a single JSON configuration file (`./config.json` by default,
|
||||||
|
override with `-config`). Every top-level key is one configuration section:
|
||||||
|
|
||||||
|
| Section | Required | Description |
|
||||||
|
| ----------------------- | -------- | ----------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `main` | yes | Server address, TLS, database path and tuning, resampling defaults, footer links, job taggers, retention of node states, NATS API subjects. |
|
||||||
|
| `auth` | no | Local, LDAP, OIDC and JWT authentication including all secrets. |
|
||||||
|
| `nats` | no | Connection to the NATS broker (address, credentials). Required for the NATS API and for NATS metric ingestion. |
|
||||||
|
| `archive` | no | Job archive backend (`file`, `s3` or `sqlite`), plus archive retention and compression. Defaults to `./var/job-archive`. |
|
||||||
|
| `metric-store` | no* | Built-in in-memory metric store: retention, memory cap, checkpointing, cleanup and NATS subscriptions. |
|
||||||
|
| `metric-store-external` | no* | List of external cc-metric-store endpoints with a `scope` (cluster name or `*`) used to route queries. |
|
||||||
|
| `cron` | no | Intervals of the background workers (commit-job, duration, footprint). |
|
||||||
|
| `ui` | no | Defaults for the web frontend shown to new users (job list, job view, metric selections, plot options). |
|
||||||
|
|
||||||
|
\* At least one of `metric-store` and `metric-store-external` must be present,
|
||||||
|
otherwise the server refuses to start.
|
||||||
|
|
||||||
|
Any section can be moved into its own file by appending `-file` to the key and
|
||||||
|
giving a path instead of an object, e.g. `"ui-file": "uiConfig.json"` loads the
|
||||||
|
`ui` section from `uiConfig.json`. Unknown top-level keys are ignored with a
|
||||||
|
warning. Unknown keys *inside* a section are always rejected by the parser: for
|
||||||
|
`main` and `metric-store` this aborts startup, for the other sections it is
|
||||||
|
logged as an error and the section is applied only up to the offending key — so
|
||||||
|
watch the log after a configuration change.
|
||||||
|
|
||||||
|
Annotated examples are in [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/main/configs):
|
||||||
|
`config.json` (full featured, S3 archive, external metric stores),
|
||||||
|
`config-large.json` (large installation with LDAP/OIDC and DB tuning),
|
||||||
|
`config-demo.json` (minimal) and `uiConfig.json`.
|
||||||
|
|
||||||
|
Cluster topology and metric definitions are **not** part of `config.json`. They
|
||||||
|
are read from the `cluster.json` file inside each cluster's job archive
|
||||||
|
directory; `configs/cluster.json` is a documented example.
|
||||||
|
|
||||||
## Database Configuration
|
## Database Configuration
|
||||||
|
|
||||||
cc-backend uses SQLite as its database. For large installations, SQLite memory
|
cc-backend uses SQLite as its database. For large installations, SQLite memory
|
||||||
@@ -272,8 +316,9 @@ The effective configuration is logged at startup for verification.
|
|||||||
- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/main/cmd/cc-backend)
|
- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/main/cmd/cc-backend)
|
||||||
contains the main application entry point and CLI implementation.
|
contains the main application entry point and CLI implementation.
|
||||||
- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/main/configs)
|
- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/main/configs)
|
||||||
contains documentation about configuration and command line options and required
|
contains sample configuration files (`config.json`, `config-demo.json`,
|
||||||
environment variables. Sample configuration files are provided.
|
`config-large.json`, `uiConfig.json`, `cluster.json`), example REST API
|
||||||
|
payloads, and the example rule sets for the job taggers.
|
||||||
- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/main/init)
|
- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/main/init)
|
||||||
contains an example of setting up systemd for production use.
|
contains an example of setting up systemd for production use.
|
||||||
- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal)
|
- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal)
|
||||||
@@ -318,10 +363,14 @@ The effective configuration is logged at startup for verification.
|
|||||||
- [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/convert-pem-pubkey)
|
- [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/convert-pem-pubkey)
|
||||||
Tool to convert external pubkey for use in `cc-backend`.
|
Tool to convert external pubkey for use in `cc-backend`.
|
||||||
- [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/gen-keypair)
|
- [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/gen-keypair)
|
||||||
contains a small application to generate a compatible JWT keypair. You find
|
contains a small application to generate a compatible Ed25519 JWT keypair for
|
||||||
documentation on how to use it
|
the `auth.jwts` configuration.
|
||||||
[here](https://github.com/ClusterCockpit/cc-backend/blob/main/docs/JWT-Handling.md).
|
- [`binaryCheckpointReader`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/binaryCheckpointReader)
|
||||||
- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/main/web)
|
Dumps the contents of a metric store binary checkpoint file for inspection.
|
||||||
|
- `dataGenerator.sh` and `grepCCLog.pl`
|
||||||
|
Helper scripts to generate metric line protocol test data and to filter
|
||||||
|
cc-backend log output.
|
||||||
|
- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/main/web)
|
||||||
Server-side templates and frontend-related files:
|
Server-side templates and frontend-related files:
|
||||||
- [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/main/web/frontend)
|
- [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/main/web/frontend)
|
||||||
Svelte components and static assets for the frontend UI
|
Svelte components and static assets for the frontend UI
|
||||||
|
|||||||
+124
-6
@@ -6,9 +6,20 @@ This release replaces the browser session implementation and requires a database
|
|||||||
migration to version 12. Run `./cc-backend -migrate-db` after upgrading; the new
|
migration to version 12. Run `./cc-backend -migrate-db` after upgrading; the new
|
||||||
`sessions` table is created automatically (a fresh `-init-db` also creates it).
|
`sessions` table is created automatically (a fresh `-init-db` also creates it).
|
||||||
Existing login sessions are invalidated by the upgrade, so users have to log in
|
Existing login sessions are invalidated by the upgrade, so users have to log in
|
||||||
again once. See the behavior changes below regarding the session cookie `Secure`
|
again once.
|
||||||
flag and the removal of the `SESSION_KEY` environment variable.
|
|
||||||
For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
|
**Two configuration changes are mandatory before starting 1.6.0:**
|
||||||
|
|
||||||
|
- The `.env` file is gone. Move every secret into the `auth` section of
|
||||||
|
`config.json` (or export the corresponding environment variable).
|
||||||
|
- The `main.resampling` section no longer accepts `trigger`, `resolutions` and
|
||||||
|
`minimum-points`. cc-backend rejects unknown keys inside `main` and aborts on
|
||||||
|
startup if they are still present. Replace them with `default-policy` and
|
||||||
|
`default-algo`.
|
||||||
|
|
||||||
|
See the behavior changes below for the details, as well as for the session
|
||||||
|
cookie `Secure` flag and the removal of the `SESSION_KEY` environment variable.
|
||||||
|
For release specific notes visit the [ClusterCockpit Documentation](https://clustercockpit.org/docs/release/).
|
||||||
|
|
||||||
## Changes in 1.6.0
|
## Changes in 1.6.0
|
||||||
|
|
||||||
@@ -28,14 +39,117 @@ For release specific notes visit the [ClusterCockpit Documentation](https://clus
|
|||||||
terminate TLS at a reverse proxy and want the `Secure` flag should serve
|
terminate TLS at a reverse proxy and want the `Secure` flag should serve
|
||||||
cc-backend over HTTPS directly for now.
|
cc-backend over HTTPS directly for now.
|
||||||
- **`SESSION_KEY` removed**: The `SESSION_KEY` environment variable is no longer
|
- **`SESSION_KEY` removed**: The `SESSION_KEY` environment variable is no longer
|
||||||
used and should be removed from your `.env`. Server-side sessions use random
|
used and can be dropped from your environment. Server-side sessions use random
|
||||||
tokens, so no cookie-signing secret is required. It is ignored if left in place.
|
tokens, so no cookie-signing secret is required. It is ignored if left in place.
|
||||||
|
- **`.env` support removed (breaking)**: The `godotenv` bootstrap, the `.env`
|
||||||
|
file and `configs/env-template.txt` are gone. Secrets (JWT public/private key,
|
||||||
|
LDAP sync password, OIDC client id/secret, cross-login keys) now live in
|
||||||
|
`config.json` under `auth` next to the option that uses them. Each secret can
|
||||||
|
still be supplied through its existing environment variable
|
||||||
|
(`JWT_PUBLIC_KEY`, `JWT_PRIVATE_KEY`, `LDAP_ADMIN_PASSWORD`, `OID_CLIENT_ID`,
|
||||||
|
`OID_CLIENT_SECRET`, `CROSS_LOGIN_JWT_PUBLIC_KEY`, `CROSS_LOGIN_JWT_HS512_KEY`),
|
||||||
|
which takes precedence over the config value. `-init` now writes the demo JWT
|
||||||
|
keys into `config.json`.
|
||||||
|
- **Resampling configuration replaced by policies (breaking)**: The explicit
|
||||||
|
`trigger`, `resolutions` and `minimum-points` keys under `main.resampling` are
|
||||||
|
removed; `main` rejects unknown keys, so a config still carrying them aborts
|
||||||
|
startup. Configure `default-policy` (`low` ≈ 200, `medium` ≈ 500, `high` ≈ 1000
|
||||||
|
target points) and `default-algo` instead. Both are also exposed as per-user
|
||||||
|
settings, and the policy is now the single source for the requested resolution
|
||||||
|
and for the resampler's minimum-points threshold.
|
||||||
|
- **Default downsampling algorithm is now `average`**: Previously an unset
|
||||||
|
algorithm fell through to cc-lib's LTTB. Average consolidation makes every
|
||||||
|
plotted point the true arithmetic mean of its interval; LTTB deliberately keeps
|
||||||
|
extremes and cannot be read as interval means. Set
|
||||||
|
`main.resampling.default-algo` to `lttb` to restore the old behavior.
|
||||||
|
- **OIDC roles require an explicit mapping (breaking for role-based setups)**:
|
||||||
|
The hardcoded translation of token roles (which only recognised the literal
|
||||||
|
names `user`/`admin`/`manager`/`support` and silently dropped `api`) is
|
||||||
|
replaced by `auth.oidc.role-mapping`. The mapping is the sole source of roles:
|
||||||
|
unmapped token roles are ignored and users without a mapped role get the base
|
||||||
|
`user` role. Deployments whose IdP emits literal CC role names must now map
|
||||||
|
them explicitly (e.g. `"admin": "admin"`). Mapping targets are validated at
|
||||||
|
startup.
|
||||||
|
- **LDAP can assign elevated roles**: The new optional `auth.ldap.role-filters`
|
||||||
|
map (role → LDAP filter) grants `admin`/`support`/`api`/`manager` to accounts
|
||||||
|
matching a filter. LDAP is authoritative for the roles listed there — sync and
|
||||||
|
login both add *and remove* them to match group membership — while roles not
|
||||||
|
listed are preserved (a managed `manager` with assigned projects is never
|
||||||
|
stripped). Without `role-filters` the behavior is unchanged.
|
||||||
|
- **Unrecognized top-level config sections are reported**: cc-backend now logs a
|
||||||
|
warning for unknown top-level keys in `config.json`, which most often means a
|
||||||
|
setting was nested at the wrong level (e.g. `resampling` next to `main`
|
||||||
|
instead of inside it) and was silently ignored before.
|
||||||
|
- **The `auth` section is validated again**: Its JSON schema was missing the
|
||||||
|
enclosing `type`/`properties` wrapper, so it accepted every input and no auth
|
||||||
|
option was ever checked. With the schema fixed, an incomplete auth section now
|
||||||
|
aborts startup — `jwts` requires `max-age`, `oidc` requires `provider`, and
|
||||||
|
`ldap` requires `url`, `user-base`, `search-dn`, `user-bind` and `user-filter`.
|
||||||
|
Each of the three subsections stays optional on its own.
|
||||||
|
- **`archive.retention` target keys**: The retention target is configured with
|
||||||
|
`target-kind` and `target-path` (or the `target-*` S3 keys). The `location` key
|
||||||
|
used by the old example configuration was never read by the code, so a `move`
|
||||||
|
or `copy` policy configured that way ran against an empty target path.
|
||||||
|
|
||||||
|
### New features
|
||||||
|
|
||||||
|
- **Configurable footer links**: `main.footer-links` lets the Imprint and
|
||||||
|
Privacy Policy footer entries point at internal pages (default) or external
|
||||||
|
URLs. External `http(s)` targets open in a new tab; empty or unset values fall
|
||||||
|
back to the built-in `/imprint` and `/privacy` routes, and the existing
|
||||||
|
`./var/*.tmpl` override mechanism keeps working.
|
||||||
|
- **Display smoothing for metric plots**: A centered, NaN-aware moving average
|
||||||
|
can be applied in the plot layer, after whatever downsampling the backend
|
||||||
|
performed. It preserves series length and index alignment, skips NaN/null
|
||||||
|
samples, keeps all-NaN windows as gaps, and shrinks the window at the series
|
||||||
|
edges. The window is a per-user setting in data points with a site default in
|
||||||
|
`ui.plot-configuration.smoothing-window` (0 disables it). It is display-only:
|
||||||
|
reported statistics and job footprints are unaffected.
|
||||||
|
- **Metric tooltips**: `metricConfig` entries in `cluster.json` accept an
|
||||||
|
optional `tooltip` that is rendered in the metric selection dialog.
|
||||||
|
- **Metric store statistics from cached aggregates**: Buffers maintain running
|
||||||
|
sum/min/max/count while data is written and compute them eagerly on checkpoint
|
||||||
|
load, so a full-buffer statistics request is served from the cached aggregate
|
||||||
|
instead of rescanning the samples.
|
||||||
|
- **Checkpoint retention aware of running jobs**: The metric store receives a
|
||||||
|
`NodeProvider` (the job repository) at `Init`. Checkpoint cleanup now skips
|
||||||
|
nodes that still have running jobs, the full checkpoint history is loaded for
|
||||||
|
those nodes at startup, and this also applies to the `-cleanup-checkpoints`
|
||||||
|
CLI path.
|
||||||
|
|
||||||
|
### Bug fixes
|
||||||
|
|
||||||
|
- **Metric plots and the policy-based resample config**: `MetricPlot` still read
|
||||||
|
the removed `trigger`/`resolutions` values, so the array-based resolution
|
||||||
|
branch was unreachable and the zoom guard always fired. The zoom trigger is now
|
||||||
|
derived from the policy's target point count.
|
||||||
|
- **Resample algorithm for running jobs**: The selected algorithm was not
|
||||||
|
forwarded to the metric store when loading data for running jobs.
|
||||||
|
- **Partially covered buffers**: Requests that reach beyond the stored data now
|
||||||
|
get NaN padding at the start/end instead of misaligned series
|
||||||
|
(`dataNotAligned`).
|
||||||
|
- **Data race in the statistics read path**: Reading buffer statistics no longer
|
||||||
|
mutates the buffer, and the fast path is guarded against empty buffers.
|
||||||
|
- **Empty node levels leaked**: The retention free path now prunes node levels
|
||||||
|
that no longer hold any data, and no longer force-frees buffers that are
|
||||||
|
retained for running jobs.
|
||||||
|
- **Plot render options in the wrong place**: `PlotRenderOptions` is only shown
|
||||||
|
in the user settings again.
|
||||||
|
- **Invisible navbar**: Fixed a stuck animation that hid the navbar with certain
|
||||||
|
browser settings.
|
||||||
|
- **Subcluster dropdown**: The subcluster selection is size-limited and scrolls
|
||||||
|
instead of overflowing.
|
||||||
|
- **OIDC login session handling**: Fixed session handling on the OIDC login
|
||||||
|
route.
|
||||||
|
|
||||||
### Dependencies
|
### Dependencies
|
||||||
|
|
||||||
- **Added** `github.com/alexedwards/scs/v2` and
|
- **Added** `github.com/alexedwards/scs/v2` and
|
||||||
`github.com/alexedwards/scs/sqlite3store`.
|
`github.com/alexedwards/scs/sqlite3store`.
|
||||||
- **Removed** `github.com/gorilla/sessions` (and `github.com/gorilla/securecookie`).
|
- **Removed** `github.com/gorilla/sessions` (and `github.com/gorilla/securecookie`),
|
||||||
|
and `github.com/joho/godotenv` together with the `.env` support.
|
||||||
|
- **Upgraded** `cc-lib` (GraphQL code regenerated) and refreshed the remaining Go
|
||||||
|
module dependencies.
|
||||||
|
|
||||||
## Changes in 1.5.4
|
## Changes in 1.5.4
|
||||||
|
|
||||||
@@ -286,7 +400,11 @@ This is also the default.
|
|||||||
|
|
||||||
- **cc-lib upgraded**: Updated to latest cc-lib version.
|
- **cc-lib upgraded**: Updated to latest cc-lib version.
|
||||||
|
|
||||||
## Known issues
|
## Known issues (as of 1.5.2)
|
||||||
|
|
||||||
|
These were the open issues at the time of the 1.5.2 release and are kept here for
|
||||||
|
that release's record; they are not necessarily still open. Note that the
|
||||||
|
`ui-config` section referenced below is now called `ui`.
|
||||||
|
|
||||||
- The new dynamic memory management is not bullet proof yet across restarts.
|
- The new dynamic memory management is not bullet proof yet across restarts.
|
||||||
Buffers that are kept outside the retention period may be lost across a
|
Buffers that are kept outside the retention period may be lost across a
|
||||||
|
|||||||
@@ -143,6 +143,7 @@ type MetricConfig {
|
|||||||
alert: Float!
|
alert: Float!
|
||||||
lowerIsBetter: Boolean
|
lowerIsBetter: Boolean
|
||||||
subClusters: [SubClusterConfig!]!
|
subClusters: [SubClusterConfig!]!
|
||||||
|
tooltip: String
|
||||||
}
|
}
|
||||||
|
|
||||||
type Tag {
|
type Tag {
|
||||||
@@ -250,6 +251,12 @@ type TimeWeights {
|
|||||||
coreHours: [NullableFloat!]!
|
coreHours: [NullableFloat!]!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum ResampleAlgo {
|
||||||
|
LTTB
|
||||||
|
AVERAGE
|
||||||
|
SIMPLE
|
||||||
|
}
|
||||||
|
|
||||||
enum Aggregate {
|
enum Aggregate {
|
||||||
USER
|
USER
|
||||||
PROJECT
|
PROJECT
|
||||||
@@ -300,6 +307,7 @@ type GlobalMetricListItem {
|
|||||||
unit: Unit!
|
unit: Unit!
|
||||||
scope: MetricScope!
|
scope: MetricScope!
|
||||||
footprint: String
|
footprint: String
|
||||||
|
tooltip: String
|
||||||
availability: [ClusterSupport!]!
|
availability: [ClusterSupport!]!
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -340,6 +348,7 @@ type Query {
|
|||||||
metrics: [String!]
|
metrics: [String!]
|
||||||
scopes: [MetricScope!]
|
scopes: [MetricScope!]
|
||||||
resolution: Int
|
resolution: Int
|
||||||
|
resampleAlgo: ResampleAlgo
|
||||||
): [JobMetricWithName!]!
|
): [JobMetricWithName!]!
|
||||||
|
|
||||||
jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
|
jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
|
||||||
@@ -399,6 +408,7 @@ type Query {
|
|||||||
to: Time!
|
to: Time!
|
||||||
page: PageRequest
|
page: PageRequest
|
||||||
resolution: Int
|
resolution: Int
|
||||||
|
resampleAlgo: ResampleAlgo
|
||||||
): NodesResultList!
|
): NodesResultList!
|
||||||
|
|
||||||
clusterMetrics(
|
clusterMetrics(
|
||||||
|
|||||||
+10
-6
@@ -24,12 +24,8 @@ const configString = `
|
|||||||
"addr": "127.0.0.1:8080",
|
"addr": "127.0.0.1:8080",
|
||||||
"short-running-jobs-duration": 300,
|
"short-running-jobs-duration": 300,
|
||||||
"resampling": {
|
"resampling": {
|
||||||
"minimum-points": 600,
|
"default-policy": "medium",
|
||||||
"trigger": 300,
|
"default-algo": "average"
|
||||||
"resolutions": [
|
|
||||||
240,
|
|
||||||
60
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"api-allowed-ips": [
|
"api-allowed-ips": [
|
||||||
"*"
|
"*"
|
||||||
@@ -51,6 +47,14 @@ const configString = `
|
|||||||
"public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=",
|
"public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=",
|
||||||
"private-key": "dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q=="
|
"private-key": "dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q=="
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"metric-store": {
|
||||||
|
"retention-in-memory": "48h",
|
||||||
|
"memory-cap": 8,
|
||||||
|
"checkpoints": {
|
||||||
|
"file-format": "wal",
|
||||||
|
"directory": "./var/checkpoints"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
`
|
`
|
||||||
|
|||||||
+22
-6
@@ -91,6 +91,19 @@ func initGops() error {
|
|||||||
func initConfiguration() error {
|
func initConfiguration() error {
|
||||||
ccconf.Init(flagConfigFile)
|
ccconf.Init(flagConfigFile)
|
||||||
|
|
||||||
|
// Warn about unrecognized top-level config sections. A common mistake is
|
||||||
|
// nesting a setting at the wrong level (e.g. placing "resampling" next to
|
||||||
|
// "main" instead of inside it), which is otherwise silently ignored.
|
||||||
|
knownSections := map[string]bool{
|
||||||
|
"main": true, "auth": true, "nats": true, "archive": true,
|
||||||
|
"metric-store": true, "metric-store-external": true, "cron": true, "ui": true,
|
||||||
|
}
|
||||||
|
for _, k := range ccconf.GetKeys() {
|
||||||
|
if !knownSections[k] {
|
||||||
|
cclog.Warnf("ignoring unrecognized top-level config section %q (check nesting in config file)", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
cfg := ccconf.GetPackageConfig("main")
|
cfg := ccconf.GetPackageConfig("main")
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return fmt.Errorf("main configuration must be present")
|
return fmt.Errorf("main configuration must be present")
|
||||||
@@ -354,13 +367,11 @@ func runServer(ctx context.Context) error {
|
|||||||
haveMetricstore := false
|
haveMetricstore := false
|
||||||
mscfg := ccconf.GetPackageConfig("metric-store")
|
mscfg := ccconf.GetPackageConfig("metric-store")
|
||||||
if mscfg != nil {
|
if mscfg != nil {
|
||||||
metrics := metricstore.BuildMetricList()
|
// The repository is injected as NodeProvider (breaking the import
|
||||||
metricstore.Init(mscfg, metrics, &wg)
|
// cycle) so the checkpoint load inside Init can fetch the full
|
||||||
|
// history for nodes with running jobs.
|
||||||
|
metricstore.Init(mscfg, metricstore.BuildMetricList(), repository.GetJobRepository(), &wg)
|
||||||
|
|
||||||
// Inject repository as NodeProvider to break import cycle
|
|
||||||
ms := metricstore.GetMemoryStore()
|
|
||||||
jobRepo := repository.GetJobRepository()
|
|
||||||
ms.SetNodeProvider(jobRepo)
|
|
||||||
metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
|
metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
|
||||||
haveMetricstore = true
|
haveMetricstore = true
|
||||||
} else {
|
} else {
|
||||||
@@ -573,6 +584,11 @@ func run() error {
|
|||||||
cleanupDir = metricstore.Keys.Cleanup.RootDir
|
cleanupDir = metricstore.Keys.Cleanup.RootDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wire the job repository as NodeProvider so cleanup skips hosts
|
||||||
|
// with running jobs (same injection as runServer).
|
||||||
|
metricstore.InitMetrics(metricstore.BuildMetricList())
|
||||||
|
metricstore.GetMemoryStore().SetNodeProvider(repository.GetJobRepository())
|
||||||
|
|
||||||
cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
|
cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
|
||||||
n, err := metricstore.CleanupCheckpoints(
|
n, err := metricstore.CleanupCheckpoints(
|
||||||
metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
|
metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
|
||||||
|
|||||||
@@ -189,7 +189,14 @@ func (s *Server) init() error {
|
|||||||
|
|
||||||
if auth.Keys.OpenIDConfig != nil {
|
if auth.Keys.OpenIDConfig != nil {
|
||||||
openIDConnect := auth.NewOIDC(authHandle)
|
openIDConnect := auth.NewOIDC(authHandle)
|
||||||
openIDConnect.RegisterEndpoints(s.router)
|
// The OIDC callback mutates the session (SaveSession renews the token and
|
||||||
|
// writes the session data), so these routes have to run inside
|
||||||
|
// scs.LoadAndSave just like /login and /logout. Without it the session
|
||||||
|
// is missing from the request context and scs panics.
|
||||||
|
s.router.Group(func(r chi.Router) {
|
||||||
|
r.Use(sessionManager.LoadAndSave)
|
||||||
|
openIDConnect.RegisterEndpoints(r)
|
||||||
|
})
|
||||||
info["hasOpenIDConnect"] = true
|
info["hasOpenIDConnect"] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+228
-115
@@ -1,15 +1,215 @@
|
|||||||
{
|
{
|
||||||
|
"name": "alex",
|
||||||
|
"metricConfig": [
|
||||||
|
{
|
||||||
|
"name": "cpu_load",
|
||||||
|
"unit": {
|
||||||
|
"base": ""
|
||||||
|
},
|
||||||
|
"scope": "node",
|
||||||
|
"aggregation": "avg",
|
||||||
|
"footprint": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 128,
|
||||||
|
"normal": 128,
|
||||||
|
"caution": 10,
|
||||||
|
"alert": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "cpu_user",
|
||||||
|
"unit": {
|
||||||
|
"base": ""
|
||||||
|
},
|
||||||
|
"scope": "hwthread",
|
||||||
|
"aggregation": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 100,
|
||||||
|
"normal": 50,
|
||||||
|
"caution": 20,
|
||||||
|
"alert": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mem_used",
|
||||||
|
"unit": {
|
||||||
|
"base": "B",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"scope": "node",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"footprint": "max",
|
||||||
|
"lowerIsBetter": true,
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 512,
|
||||||
|
"normal": 128,
|
||||||
|
"caution": 200,
|
||||||
|
"alert": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "flops_any",
|
||||||
|
"unit": {
|
||||||
|
"base": "F/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"scope": "hwthread",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"footprint": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 9216,
|
||||||
|
"normal": 1000,
|
||||||
|
"caution": 200,
|
||||||
|
"alert": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mem_bw",
|
||||||
|
"unit": {
|
||||||
|
"base": "B/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"scope": "socket",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"footprint": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 350,
|
||||||
|
"normal": 100,
|
||||||
|
"caution": 50,
|
||||||
|
"alert": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "clock",
|
||||||
|
"unit": {
|
||||||
|
"base": "Hz",
|
||||||
|
"prefix": "M"
|
||||||
|
},
|
||||||
|
"scope": "hwthread",
|
||||||
|
"aggregation": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 3000,
|
||||||
|
"normal": 2400,
|
||||||
|
"caution": 1800,
|
||||||
|
"alert": 1200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "core_power",
|
||||||
|
"unit": {
|
||||||
|
"base": "W"
|
||||||
|
},
|
||||||
|
"scope": "hwthread",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"energy": "power",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 500,
|
||||||
|
"normal": 250,
|
||||||
|
"caution": 100,
|
||||||
|
"alert": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "cpu_power",
|
||||||
|
"unit": {
|
||||||
|
"base": "W"
|
||||||
|
},
|
||||||
|
"scope": "socket",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"energy": "power",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 500,
|
||||||
|
"normal": 250,
|
||||||
|
"caution": 100,
|
||||||
|
"alert": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ipc",
|
||||||
|
"unit": {
|
||||||
|
"base": "IPC"
|
||||||
|
},
|
||||||
|
"scope": "hwthread",
|
||||||
|
"aggregation": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 4,
|
||||||
|
"normal": 2,
|
||||||
|
"caution": 1,
|
||||||
|
"alert": 0.5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "acc_utilization",
|
||||||
|
"unit": {
|
||||||
|
"base": ""
|
||||||
|
},
|
||||||
|
"scope": "accelerator",
|
||||||
|
"aggregation": "avg",
|
||||||
|
"footprint": "avg",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 100,
|
||||||
|
"normal": 80,
|
||||||
|
"caution": 50,
|
||||||
|
"alert": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "acc_mem_used",
|
||||||
|
"unit": {
|
||||||
|
"base": "B",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"scope": "accelerator",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 48,
|
||||||
|
"normal": 24,
|
||||||
|
"caution": 10,
|
||||||
|
"alert": 5,
|
||||||
|
"subClusters": [
|
||||||
|
{
|
||||||
|
"name": "a100",
|
||||||
|
"peak": 40,
|
||||||
|
"normal": 20,
|
||||||
|
"caution": 10,
|
||||||
|
"alert": 5
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "acc_power",
|
||||||
|
"unit": {
|
||||||
|
"base": "W"
|
||||||
|
},
|
||||||
|
"scope": "accelerator",
|
||||||
|
"aggregation": "sum",
|
||||||
|
"energy": "power",
|
||||||
|
"timestep": 60,
|
||||||
|
"peak": 400,
|
||||||
|
"normal": 200,
|
||||||
|
"caution": 50,
|
||||||
|
"alert": 20
|
||||||
|
}
|
||||||
|
],
|
||||||
"subClusters": [
|
"subClusters": [
|
||||||
{
|
{
|
||||||
"name": "a40",
|
"name": "a40",
|
||||||
"numberOfNodes": 38,
|
"nodes": "a[0121-0129],a[0221-0229],a[0321-0329],a[0421-0429]",
|
||||||
"processorType": "AMD Milan",
|
"processorType": "AMD Milan",
|
||||||
"socketsPerNode": 2,
|
"socketsPerNode": 2,
|
||||||
"coresPerSocket": 64,
|
"coresPerSocket": 64,
|
||||||
"threadsPerCore": 1,
|
"threadsPerCore": 1,
|
||||||
"flopRateScalar": 432,
|
"flopRateScalar": {
|
||||||
"flopRateSimd": 9216,
|
"unit": {
|
||||||
"memoryBandwidth": 400,
|
"base": "F/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 432
|
||||||
|
},
|
||||||
|
"flopRateSimd": {
|
||||||
|
"unit": {
|
||||||
|
"base": "F/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 9216
|
||||||
|
},
|
||||||
|
"memoryBandwidth": {
|
||||||
|
"unit": {
|
||||||
|
"base": "B/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 400
|
||||||
|
},
|
||||||
"topology": {
|
"topology": {
|
||||||
"node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127],
|
"node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127],
|
||||||
"socket": [
|
"socket": [
|
||||||
@@ -19,9 +219,7 @@
|
|||||||
"memoryDomain": [
|
"memoryDomain": [
|
||||||
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127]
|
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127]
|
||||||
],
|
],
|
||||||
"core": [
|
"core": [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[72],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]],
|
||||||
[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]
|
|
||||||
],
|
|
||||||
"accelerators": [
|
"accelerators": [
|
||||||
{
|
{
|
||||||
"id": "00000000:01:00.0",
|
"id": "00000000:01:00.0",
|
||||||
@@ -68,14 +266,32 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "a100",
|
"name": "a100",
|
||||||
"numberOfNodes": 20,
|
"nodes": "a[0601-0605],a[0701-0705],a[0801-0805],a[0901-0905]",
|
||||||
"processorType": "AMD Milan",
|
"processorType": "AMD Milan",
|
||||||
"socketsPerNode": 2,
|
"socketsPerNode": 2,
|
||||||
"coresPerSocket": 64,
|
"coresPerSocket": 64,
|
||||||
"threadsPerCore": 1,
|
"threadsPerCore": 1,
|
||||||
"flopRateScalar": 432,
|
"flopRateScalar": {
|
||||||
"flopRateSimd": 9216,
|
"unit": {
|
||||||
"memoryBandwidth": 400,
|
"base": "F/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 432
|
||||||
|
},
|
||||||
|
"flopRateSimd": {
|
||||||
|
"unit": {
|
||||||
|
"base": "F/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 9216
|
||||||
|
},
|
||||||
|
"memoryBandwidth": {
|
||||||
|
"unit": {
|
||||||
|
"base": "B/s",
|
||||||
|
"prefix": "G"
|
||||||
|
},
|
||||||
|
"value": 400
|
||||||
|
},
|
||||||
"topology": {
|
"topology": {
|
||||||
"node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127],
|
"node": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127],
|
||||||
"socket": [
|
"socket": [
|
||||||
@@ -85,9 +301,7 @@
|
|||||||
"memoryDomain": [
|
"memoryDomain": [
|
||||||
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127]
|
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127]
|
||||||
],
|
],
|
||||||
"core": [
|
"core": [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[72],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]],
|
||||||
[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17],[18],[19],[20],[21],[22],[23],[24],[25],[26],[27],[28],[29],[30],[31],[32],[33],[34],[35],[36],[37],[38],[39],[40],[41],[42],[43],[44],[45],[46],[47],[48],[49],[50],[51],[52],[53],[54],[55],[56],[57],[58],[59],[60],[61],[62],[63],[64],[65],[66],[67],[68],[69],[70],[71],[73],[74],[75],[76],[77],[78],[79],[80],[81],[82],[83],[84],[85],[86],[87],[88],[89],[90],[91],[92],[93],[94],[95],[96],[97],[98],[99],[100],[101],[102],[103],[104],[105],[106],[107],[108],[109],[110],[111],[112],[113],[114],[115],[116],[117],[118],[119],[120],[121],[122],[123],[124],[125],[126],[127]
|
|
||||||
],
|
|
||||||
"accelerators": [
|
"accelerators": [
|
||||||
{
|
{
|
||||||
"id": "00000000:0E:00.0",
|
"id": "00000000:0E:00.0",
|
||||||
@@ -132,106 +346,5 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
|
||||||
"metricConfig": [
|
|
||||||
{
|
|
||||||
"name": "cpu_load",
|
|
||||||
"scope": "node",
|
|
||||||
"unit": "load 1m",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": null,
|
|
||||||
"peak": 128,
|
|
||||||
"normal": 128,
|
|
||||||
"caution": 10,
|
|
||||||
"alert": 5
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "cpu_user",
|
|
||||||
"scope": "hwthread",
|
|
||||||
"unit": "cpu user",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "avg",
|
|
||||||
"peak": 100,
|
|
||||||
"normal": 50,
|
|
||||||
"caution": 20,
|
|
||||||
"alert": 10
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "mem_used",
|
|
||||||
"scope": "node",
|
|
||||||
"unit": "GB",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": null,
|
|
||||||
"peak": 512,
|
|
||||||
"normal": 128,
|
|
||||||
"caution": 200,
|
|
||||||
"alert": 240
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "flops_any",
|
|
||||||
"scope": "hwthread",
|
|
||||||
"unit": "GF/s",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "sum",
|
|
||||||
"peak": 9216,
|
|
||||||
"normal": 1000,
|
|
||||||
"caution": 200,
|
|
||||||
"alert": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "mem_bw",
|
|
||||||
"scope": "socket",
|
|
||||||
"unit": "GB/s",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "sum",
|
|
||||||
"peak": 350,
|
|
||||||
"normal": 100,
|
|
||||||
"caution": 50,
|
|
||||||
"alert": 10
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "clock",
|
|
||||||
"scope": "hwthread",
|
|
||||||
"unit": "MHz",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "avg",
|
|
||||||
"peak": 3000,
|
|
||||||
"normal": 2400,
|
|
||||||
"caution": 1800,
|
|
||||||
"alert": 1200
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "core_power",
|
|
||||||
"scope": "hwthread",
|
|
||||||
"unit": "W",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "sum",
|
|
||||||
"peak": 500,
|
|
||||||
"normal": 250,
|
|
||||||
"caution": 100,
|
|
||||||
"alert": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "cpu_power",
|
|
||||||
"scope": "socket",
|
|
||||||
"unit": "W",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "sum",
|
|
||||||
"peak": 500,
|
|
||||||
"normal": 250,
|
|
||||||
"caution": 100,
|
|
||||||
"alert": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "ipc",
|
|
||||||
"scope": "hwthread",
|
|
||||||
"unit": "IPC",
|
|
||||||
"timestep": 60,
|
|
||||||
"aggregation": "avg",
|
|
||||||
"peak": 4,
|
|
||||||
"normal": 2,
|
|
||||||
"caution": 1,
|
|
||||||
"alert": 0.5
|
|
||||||
}
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
{
|
||||||
|
"main": {
|
||||||
|
"addr": "0.0.0.0:8088",
|
||||||
|
"short-running-jobs-duration": 300,
|
||||||
|
"emission-constant": 317,
|
||||||
|
"enable-job-taggers": true,
|
||||||
|
"db-config": {
|
||||||
|
"cache-size-mb": 16384,
|
||||||
|
"soft-heap-limit-mb": 131072,
|
||||||
|
"max-open-connections": 8,
|
||||||
|
"max-idle-connections": 8,
|
||||||
|
"max-idle-time-minutes": 30
|
||||||
|
},
|
||||||
|
"nodestate-retention": {
|
||||||
|
"policy": "move",
|
||||||
|
"age": 168,
|
||||||
|
"target-kind": "file",
|
||||||
|
"target-path": "./var/nodestate-archive"
|
||||||
|
},
|
||||||
|
"resampling": {
|
||||||
|
"default-policy": "medium",
|
||||||
|
"default-algo": "average"
|
||||||
|
},
|
||||||
|
"api-subjects": {
|
||||||
|
"subject-job-event": "cc.job.event",
|
||||||
|
"subject-node-state": "cc.node.state",
|
||||||
|
"job-concurrency": 8,
|
||||||
|
"node-concurrency": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nats": {
|
||||||
|
"address": "nats://0.0.0.0:4222",
|
||||||
|
"username": "root",
|
||||||
|
"password": "root"
|
||||||
|
},
|
||||||
|
"auth": {
|
||||||
|
"jwts": {
|
||||||
|
"max-age": "2000h",
|
||||||
|
"public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=",
|
||||||
|
"private-key": "dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q=="
|
||||||
|
},
|
||||||
|
"oidc": {
|
||||||
|
"provider": "http://localhost:8080/realms/clustercockpit",
|
||||||
|
"client-id": "cc-backend",
|
||||||
|
"client-secret": "cc-backend-dev-secret",
|
||||||
|
"sync-user-on-login": true,
|
||||||
|
"update-user-on-login": true,
|
||||||
|
"role-mapping": {
|
||||||
|
"cc-admin": "admin"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ldap": {
|
||||||
|
"url": "ldap://127.0.0.1:389",
|
||||||
|
"user-base": "ou=users,dc=example,dc=com",
|
||||||
|
"search-dn": "cn=admin,dc=example,dc=com",
|
||||||
|
"user-bind": "uid={username},ou=users,dc=example,dc=com",
|
||||||
|
"user-filter": "(objectclass=posixAccount)",
|
||||||
|
"username-attr": "cn",
|
||||||
|
"uid-attr": "uid",
|
||||||
|
"sync-password": "mashup",
|
||||||
|
"sync-interval": "2h",
|
||||||
|
"sync-user-on-login": true,
|
||||||
|
"update-user-on-login": true,
|
||||||
|
"role-filters": {
|
||||||
|
"admin": "(memberOf=cn=cc-admin,ou=groups,dc=example,dc=com)",
|
||||||
|
"support": "(memberOf=cn=cc-support,ou=groups,dc=example,dc=com)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"cron": {
|
||||||
|
"commit-job-worker": "1m",
|
||||||
|
"duration-worker": "5m",
|
||||||
|
"footprint-worker": "10m"
|
||||||
|
},
|
||||||
|
"archive": {
|
||||||
|
"kind": "file",
|
||||||
|
"path": "./var/job-archive",
|
||||||
|
"compression": 7,
|
||||||
|
"retention": {
|
||||||
|
"policy": "move",
|
||||||
|
"format": "parquet",
|
||||||
|
"age": 365,
|
||||||
|
"include-db": true,
|
||||||
|
"target-kind": "file",
|
||||||
|
"target-path": "./var/job-archive-retention"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metric-store-external": [
|
||||||
|
{
|
||||||
|
"scope": "fritz",
|
||||||
|
"url": "http://0.0.0.0:8084",
|
||||||
|
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scope": "alex",
|
||||||
|
"url": "http://0.0.0.0:8084",
|
||||||
|
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric-store": {
|
||||||
|
"retention-in-memory": "24h",
|
||||||
|
"memory-cap": 100,
|
||||||
|
"num-workers": 10,
|
||||||
|
"checkpoints": {
|
||||||
|
"file-format": "wal",
|
||||||
|
"directory": "./var/checkpoints"
|
||||||
|
},
|
||||||
|
"checkpoint-interval": "12h",
|
||||||
|
"cleanup": {
|
||||||
|
"mode": "archive",
|
||||||
|
"directory": "./var/metric-archive"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+15
-7
@@ -14,9 +14,8 @@
|
|||||||
"target-path": "./var/nodestate-archive"
|
"target-path": "./var/nodestate-archive"
|
||||||
},
|
},
|
||||||
"resampling": {
|
"resampling": {
|
||||||
"minimum-points": 600,
|
"default-policy": "medium",
|
||||||
"trigger": 180,
|
"default-algo": "average"
|
||||||
"resolutions": [240, 60]
|
|
||||||
},
|
},
|
||||||
"footer-links": {
|
"footer-links": {
|
||||||
"imprint": "/imprint",
|
"imprint": "/imprint",
|
||||||
@@ -24,7 +23,9 @@
|
|||||||
},
|
},
|
||||||
"api-subjects": {
|
"api-subjects": {
|
||||||
"subject-job-event": "cc.job.event",
|
"subject-job-event": "cc.job.event",
|
||||||
"subject-node-state": "cc.node.state"
|
"subject-node-state": "cc.node.state",
|
||||||
|
"job-concurrency": 8,
|
||||||
|
"node-concurrency": 2
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nats": {
|
"nats": {
|
||||||
@@ -52,8 +53,12 @@
|
|||||||
"secret-key": "xx",
|
"secret-key": "xx",
|
||||||
"retention": {
|
"retention": {
|
||||||
"policy": "move",
|
"policy": "move",
|
||||||
|
"format": "parquet",
|
||||||
"age": 365,
|
"age": 365,
|
||||||
"location": "./var/archive"
|
"include-db": true,
|
||||||
|
"target-kind": "file",
|
||||||
|
"target-path": "./var/job-archive-retention",
|
||||||
|
"max-file-size-mb": 512
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"metric-store-external": [
|
"metric-store-external": [
|
||||||
@@ -80,8 +85,11 @@
|
|||||||
],
|
],
|
||||||
"metric-store": {
|
"metric-store": {
|
||||||
"checkpoints": {
|
"checkpoints": {
|
||||||
"directory": "./var/checkpoints"
|
"file-format": "wal",
|
||||||
|
"directory": "./var/checkpoints",
|
||||||
|
"max-wal-size": 268435456
|
||||||
},
|
},
|
||||||
|
"checkpoint-interval": "12h",
|
||||||
"memory-cap": 100,
|
"memory-cap": 100,
|
||||||
"retention-in-memory": "24h",
|
"retention-in-memory": "24h",
|
||||||
"cleanup": {
|
"cleanup": {
|
||||||
@@ -99,5 +107,5 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"ui-file": "ui-config.json"
|
"ui-file": "uiConfig.json"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"job-list": {
|
"job-list": {
|
||||||
"use-paging": false,
|
"use-paging": false,
|
||||||
"show-footprint":false
|
"show-footprint": false
|
||||||
},
|
},
|
||||||
"job-view": {
|
"job-view": {
|
||||||
"show-polar-plot": true,
|
"show-polar-plot": true,
|
||||||
@@ -9,6 +9,9 @@
|
|||||||
"show-roofline": true,
|
"show-roofline": true,
|
||||||
"show-stat-table": true
|
"show-stat-table": true
|
||||||
},
|
},
|
||||||
|
"node-list": {
|
||||||
|
"use-paging": true
|
||||||
|
},
|
||||||
"metric-config": {
|
"metric-config": {
|
||||||
"job-list-metrics": ["mem_bw", "flops_dp"],
|
"job-list-metrics": ["mem_bw", "flops_dp"],
|
||||||
"job-view-plot-metrics": ["mem_bw", "flops_dp"],
|
"job-view-plot-metrics": ["mem_bw", "flops_dp"],
|
||||||
@@ -25,13 +28,13 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"node-list": {
|
|
||||||
"use-paging": true
|
|
||||||
},
|
|
||||||
"plot-configuration": {
|
"plot-configuration": {
|
||||||
"plots-per-row": 3,
|
"plots-per-row": 3,
|
||||||
"color-background": true,
|
"color-background": true,
|
||||||
"line-width": 3,
|
"line-width": 3,
|
||||||
|
"smoothing-window": 0,
|
||||||
|
"resample-policy": "medium",
|
||||||
|
"resample-algo": "average",
|
||||||
"color-scheme": [
|
"color-scheme": [
|
||||||
"#00bfff",
|
"#00bfff",
|
||||||
"#0000ff",
|
"#0000ff",
|
||||||
|
|||||||
@@ -8,37 +8,37 @@ tool (
|
|||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/99designs/gqlgen v0.17.90
|
github.com/99designs/gqlgen v0.17.94
|
||||||
github.com/ClusterCockpit/cc-lib/v2 v2.12.0
|
github.com/ClusterCockpit/cc-lib/v2 v2.13.0
|
||||||
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
|
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
|
||||||
github.com/Masterminds/squirrel v1.5.4
|
github.com/Masterminds/squirrel v1.5.4
|
||||||
github.com/alexedwards/scs/sqlite3store v0.0.0-20251002162104-209de6e426de
|
github.com/alexedwards/scs/sqlite3store v0.0.0-20251002162104-209de6e426de
|
||||||
github.com/alexedwards/scs/v2 v2.9.0
|
github.com/alexedwards/scs/v2 v2.9.0
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
github.com/aws/aws-sdk-go-v2 v1.44.0
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.18
|
github.com/aws/aws-sdk-go-v2/config v1.32.40
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.39
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0
|
||||||
github.com/coreos/go-oidc/v3 v3.18.0
|
github.com/coreos/go-oidc/v3 v3.20.0
|
||||||
github.com/expr-lang/expr v1.17.8
|
github.com/expr-lang/expr v1.17.8
|
||||||
github.com/go-chi/chi/v5 v5.3.0
|
github.com/go-chi/chi/v5 v5.3.2
|
||||||
github.com/go-chi/cors v1.2.2
|
github.com/go-chi/cors v1.2.2
|
||||||
github.com/go-co-op/gocron/v2 v2.21.2
|
github.com/go-co-op/gocron/v2 v2.22.0
|
||||||
github.com/go-ldap/ldap/v3 v3.4.13
|
github.com/go-ldap/ldap/v3 v3.4.14
|
||||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||||
github.com/golang-migrate/migrate/v4 v4.19.1
|
github.com/golang-migrate/migrate/v4 v4.19.1
|
||||||
github.com/google/gops v0.3.29
|
github.com/google/gops v0.3.29
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
github.com/jmoiron/sqlx v1.4.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.44
|
github.com/mattn/go-sqlite3 v1.14.50
|
||||||
github.com/parquet-go/parquet-go v0.30.1
|
github.com/parquet-go/parquet-go v0.32.0
|
||||||
github.com/qustavo/sqlhooks/v2 v2.1.0
|
github.com/qustavo/sqlhooks/v2 v2.1.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
|
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
github.com/swaggo/http-swagger v1.3.4
|
github.com/swaggo/http-swagger v1.3.4
|
||||||
github.com/swaggo/swag v1.16.6
|
github.com/swaggo/swag v1.16.6
|
||||||
github.com/vektah/gqlparser/v2 v2.5.33
|
github.com/vektah/gqlparser/v2 v2.5.36
|
||||||
golang.org/x/crypto v0.52.0
|
golang.org/x/crypto v0.55.0
|
||||||
golang.org/x/oauth2 v0.36.0
|
golang.org/x/oauth2 v0.36.0
|
||||||
golang.org/x/sync v0.20.0
|
golang.org/x/sync v0.22.0
|
||||||
golang.org/x/time v0.15.0
|
golang.org/x/time v0.15.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,76 +46,76 @@ require (
|
|||||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||||
github.com/agnivade/levenshtein v1.2.1 // indirect
|
github.com/agnivade/levenshtein v1.2.1 // indirect
|
||||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 // indirect
|
||||||
github.com/aws/smithy-go v1.26.0 // indirect
|
github.com/aws/smithy-go v1.28.1 // indirect
|
||||||
|
github.com/coder/websocket v1.8.15 // indirect
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect
|
||||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||||
github.com/go-openapi/jsonpointer v0.23.1 // indirect
|
github.com/go-openapi/jsonpointer v1.0.0 // indirect
|
||||||
github.com/go-openapi/jsonreference v0.21.5 // indirect
|
github.com/go-openapi/jsonreference v1.0.1 // indirect
|
||||||
github.com/go-openapi/spec v0.22.4 // indirect
|
github.com/go-openapi/spec v0.22.11 // indirect
|
||||||
github.com/go-openapi/swag/conv v0.26.0 // indirect
|
github.com/go-openapi/swag/conv v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/jsonname v0.26.0 // indirect
|
github.com/go-openapi/swag/jsonutils v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/jsonutils v0.26.0 // indirect
|
github.com/go-openapi/swag/loading v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/loading v0.26.0 // indirect
|
github.com/go-openapi/swag/pools v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/stringutils v0.26.0 // indirect
|
github.com/go-openapi/swag/stringutils v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/typeutils v0.26.0 // indirect
|
github.com/go-openapi/swag/typeutils v0.29.1 // indirect
|
||||||
github.com/go-openapi/swag/yamlutils v0.26.0 // indirect
|
github.com/go-openapi/swag/yamlutils v0.29.1 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/gorilla/websocket v1.5.3 // indirect
|
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||||
github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
|
github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
|
||||||
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
|
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
|
||||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||||
github.com/klauspost/compress v1.18.6 // indirect
|
github.com/klauspost/compress v1.19.2 // indirect
|
||||||
github.com/kr/pretty v0.3.1 // indirect
|
github.com/kr/pretty v0.3.1 // indirect
|
||||||
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
|
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
|
||||||
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
|
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
|
||||||
github.com/nats-io/nats.go v1.52.0 // indirect
|
github.com/nats-io/nats.go v1.53.1 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
github.com/oapi-codegen/runtime v1.4.1 // indirect
|
github.com/oapi-codegen/runtime v1.7.0 // indirect
|
||||||
github.com/parquet-go/bitpack v1.0.0 // indirect
|
github.com/parquet-go/bitpack v1.1.0 // indirect
|
||||||
github.com/parquet-go/jsonlite v1.5.2 // indirect
|
github.com/parquet-go/jsonlite v1.5.5 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.26 // indirect
|
github.com/pierrec/lz4/v4 v4.1.29 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
github.com/sosodev/duration v1.4.0 // indirect
|
github.com/sosodev/duration v1.4.0 // indirect
|
||||||
github.com/stmcginnis/gofish v0.22.0 // indirect
|
github.com/stmcginnis/gofish v0.24.0 // indirect
|
||||||
github.com/stretchr/objx v0.5.2 // indirect
|
github.com/stretchr/objx v0.5.2 // indirect
|
||||||
github.com/swaggo/files v1.0.1 // indirect
|
github.com/swaggo/files v1.0.1 // indirect
|
||||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||||
github.com/urfave/cli/v2 v2.27.7 // indirect
|
github.com/urfave/cli/v2 v2.27.7 // indirect
|
||||||
github.com/urfave/cli/v3 v3.8.0 // indirect
|
github.com/urfave/cli/v3 v3.10.1 // indirect
|
||||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
|
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/mod v0.36.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/net v0.55.0 // indirect
|
golang.org/x/net v0.58.0 // indirect
|
||||||
golang.org/x/sys v0.45.0 // indirect
|
golang.org/x/sys v0.47.0 // indirect
|
||||||
golang.org/x/text v0.37.0 // indirect
|
golang.org/x/text v0.41.0 // indirect
|
||||||
golang.org/x/tools v0.45.0 // indirect
|
golang.org/x/tools v0.49.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.12 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
github.com/99designs/gqlgen v0.17.90 h1:wSv6blm/PoplU6QoNw83EcQpNtC0HX3/+44vITJOzpk=
|
github.com/99designs/gqlgen v0.17.94 h1:+3EUDVgX/8gDyDL+7NUqCo4cy2ylylwW0GvR1dGiEsA=
|
||||||
github.com/99designs/gqlgen v0.17.90/go.mod h1:GqYrEwYsqCG8VaOsq2kJUCUKwAE1T+u2i+Nj7NtXiVI=
|
github.com/99designs/gqlgen v0.17.94/go.mod h1:o+XaAMpPA/AX4rqeiK03tZUb/5T+WCgpRDD4aujgdas=
|
||||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||||
github.com/ClusterCockpit/cc-lib/v2 v2.12.0 h1:ZbGD68nDniuvzFjJCdyYawpCBrabdSyWOg5FFSyFbjQ=
|
github.com/ClusterCockpit/cc-lib/v2 v2.13.0 h1:Gev5Bl4xuj+u15oTHNBLgpQeGER56e1FVWuTdBrfWLk=
|
||||||
github.com/ClusterCockpit/cc-lib/v2 v2.12.0/go.mod h1:ml8xtcYa5WhPM7JDQ+M9/R9ZBxITCR/5xqGJ//GxXJI=
|
github.com/ClusterCockpit/cc-lib/v2 v2.13.0/go.mod h1:F7SmuvPGDqZucdm4MdMK///4RePgzWhRP0cyWK1Cons=
|
||||||
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
|
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
|
||||||
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
|
github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
|
||||||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||||
@@ -14,10 +14,8 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc
|
|||||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||||
github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
|
github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
|
||||||
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
|
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
|
||||||
github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
|
github.com/NVIDIA/go-nvml v0.13.3-1 h1:P76U2h88OZSiMtdhRsJjSF5DXyXUqHIXKeDicVAaae0=
|
||||||
github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
|
github.com/NVIDIA/go-nvml v0.13.3-1/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ=
|
||||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
|
||||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
|
||||||
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
|
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
|
||||||
github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
|
github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
|
||||||
github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
|
github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
|
||||||
@@ -31,63 +29,59 @@ github.com/alexedwards/scs/sqlite3store v0.0.0-20251002162104-209de6e426de h1:c7
|
|||||||
github.com/alexedwards/scs/sqlite3store v0.0.0-20251002162104-209de6e426de/go.mod h1:Iyk7S76cxGaiEX/mSYmTZzYehp4KfyylcLaV3OnToss=
|
github.com/alexedwards/scs/sqlite3store v0.0.0-20251002162104-209de6e426de/go.mod h1:Iyk7S76cxGaiEX/mSYmTZzYehp4KfyylcLaV3OnToss=
|
||||||
github.com/alexedwards/scs/v2 v2.9.0 h1:xa05mVpwTBm1iLeTMNFfAWpKUm4fXAW7CeAViqBVS90=
|
github.com/alexedwards/scs/v2 v2.9.0 h1:xa05mVpwTBm1iLeTMNFfAWpKUm4fXAW7CeAViqBVS90=
|
||||||
github.com/alexedwards/scs/v2 v2.9.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8=
|
github.com/alexedwards/scs/v2 v2.9.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8=
|
||||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
|
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
|
||||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op h1:p2zFsAzvhIpFya8AIOHIbWf7NGvO34QpLGclyf7nXj8=
|
||||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
|
||||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
|
|
||||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
|
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
|
||||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
|
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
|
||||||
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
|
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
|
||||||
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
|
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
github.com/aws/aws-sdk-go-v2 v1.44.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
|
github.com/aws/aws-sdk-go-v2/config v1.32.40 h1:lAVC9gMmKusmqDRe32dPtgKl/BWvJmMJoWELKHCAObw=
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
|
github.com/aws/aws-sdk-go-v2/config v1.32.40/go.mod h1:8xOJLbe/hOj1g4PVsfJYV7O2byq+UGET1onDdUgbwqc=
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.39 h1:XOg8LC3Kgnsa3WiPQjc7Bi8k5IBN92cPYfIV9XMFss0=
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.39/go.mod h1:GonTDBQ+mTpCVNwaHjj0PagspfrYYMEqOx7FehoEP/I=
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 h1:r5aGipEVgI9aT/tAGjdrPbDQvIAKdTrS3rUPQtG4Rmo=
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40/go.mod h1:vOD3CnPxAdkL6MWZeROkZsTlskklMFfgVFkHzx/oZpY=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 h1:UIXlbijuB2XK1Kr57fo8iIxCuaSHJzwZ1uo+2tbEYIk=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40/go.mod h1:wcEsL6jscjZjVUinb0Q5qD/GXOG1yT3GNfmT9HuDwzU=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 h1:xLQVRDs2NddDmK9BEyh5KSlJ1Gpy5/GIJXrV6WcVGAE=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40/go.mod h1:XRXnpFVFGLaEVK+olDdFIM1vNa04ETW452oFGEPUxAo=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 h1:nv/ILuCY0yXACzMQwvtt/HbqDDjemZiI0AeDbxGQlnU=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41/go.mod h1:dzvOSpxaPqQ3j0xS6Lc1vyVuWW0RBj7s/QqYpzu3Q/0=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 h1:tX68nPDCoX0s2ksM7CipWP0QFw2hGDWwUdxI6+eT9ZU=
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 h1:U8/A0RRBaEspzH1uul3JHLbypXwEGUkRkvoT9f0ATcM=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0/go.mod h1:UELStX5KwtJNtQxa+UuF8dc3z4UYc40e8yHYJSozNwY=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cxZXNCdeo/lQ7isHEHzvHVM7z75qb2zW9aMjw=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 h1:Q9DIKDuJix/oJnQxFpQ26L0EwVa/YNo4k2kbktrjQjE=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41/go.mod h1:x+TuqkOIG1SZS0+yN54sExGA9ZpjhPO6vPdYnpTFX1M=
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0 h1:gfPQ6do5PZTCc5n/vZUHz/G8McrNrfERGSO+iHvVbCA=
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 h1:Yp+x5PKXEmoqHsgP/pAkBy5Tyq1UlXAzM0OInh0vxWw=
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0/go.mod h1:wO6U9egJtCtsZEHG2AAcFf1kUWDRrH0Iif6K3bVmmdE=
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0/go.mod h1:locV6DtXyp7Xzr2BG6jtsbeBi3YAWJ/CY4xUThYmIwQ=
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU=
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 h1:FxaN8/sn61DTXNI6Gt678tFJUY8iUsCchm6Y/F/RjaA=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
github.com/aws/aws-sdk-go-v2/service/sso v1.34.0/go.mod h1:vu4OY6s8LJtT8BtYG2LD6BGSZMptkYn3o5hvCPB22jc=
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 h1:crWKPeGYTBTuBxQ3p73kjfJvt4brUIsr+Fuypko8FxY=
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0/go.mod h1:HjjZVhaBz0JBR/kbWKThmNDhFKS7y6EURuk493tJk9Y=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 h1:IZ63JdogSNNjex/jsODNv7jGDcO/xJYd9FsgyfCsp1g=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
github.com/aws/aws-sdk-go-v2/service/sts v1.46.0/go.mod h1:I+rwAf3spG5dITBaAo3xXRowk8kiOhtU1kYxfvCTC44=
|
||||||
github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
|
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
|
||||||
github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=
|
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||||
github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||||
github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A=
|
github.com/coreos/go-oidc/v3 v3.20.0 h1:EtE0WIBHk03N+DqGkY4+UONzzZHk7amKt6IyNd7OsZE=
|
||||||
github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
github.com/coreos/go-oidc/v3 v3.20.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
|
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
@@ -103,45 +97,45 @@ github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+
|
|||||||
github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
|
github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
|
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
|
||||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
|
||||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||||
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||||
github.com/go-co-op/gocron/v2 v2.21.2 h1:bD8/YwkojYHgXFr3iEulL148KBdTbKVxUZzFKpXcdbY=
|
github.com/go-co-op/gocron/v2 v2.22.0 h1:uEuH2F7k7VoESb1BYSaffuuV+T0kkpzsC0aXk7/z79I=
|
||||||
github.com/go-co-op/gocron/v2 v2.21.2/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
|
github.com/go-co-op/gocron/v2 v2.22.0/go.mod h1:hiH/U9RMhTi1BBZJmef9s3KC9QwhpBF6PFrvUKaXY9M=
|
||||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||||
github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ=
|
github.com/go-ldap/ldap/v3 v3.4.14 h1:D6PYdEgsaVzsXyr6w/yDC06Ria4uUhWm+Rb+er8lfAs=
|
||||||
github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0=
|
github.com/go-ldap/ldap/v3 v3.4.14/go.mod h1:S4eJUMUNjDkE0ZJtIZdybwyb03sGGLW6gxXT1Hs8VKA=
|
||||||
github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4=
|
github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s=
|
||||||
github.com/go-openapi/jsonpointer v0.23.1/go.mod h1:iWRmZTrGn7XwYhtPt/fvdSFj1OfNBngqRT2UG3BxSqY=
|
github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
|
||||||
github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
|
github.com/go-openapi/jsonreference v1.0.1 h1:4zJ7AmYDKNmD3aSpfPnFNCFA5E80/xMHUNKgydaLh38=
|
||||||
github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
|
github.com/go-openapi/jsonreference v1.0.1/go.mod h1:dYplQXa6p5lXprLcJ8LE2iU7vNpXsAHDQ5ZAgL+Qx3A=
|
||||||
github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ=
|
github.com/go-openapi/spec v0.22.11 h1:/ZW28n4PCghbBpwjmvoMeyCeDwd6pIe715Ov8T3kAaQ=
|
||||||
github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ=
|
github.com/go-openapi/spec v0.22.11/go.mod h1:Ypu+u1KTejey7/QCr3j1ChdjwhGgXuklbH7l8+YP1J4=
|
||||||
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
||||||
github.com/go-openapi/swag/conv v0.26.0 h1:5yGGsPYI1ZCva93U0AoKi/iZrNhaJEjr324YVsiD89I=
|
github.com/go-openapi/swag/conv v0.29.1 h1:AC4Eh/5c/eUDOUCzzsRC9ghmFgOSBHeRMGIngY0ZUGA=
|
||||||
github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE=
|
github.com/go-openapi/swag/conv v0.29.1/go.mod h1:S1X7/ZrBEZOC0Wc8AGxjbcGS92l3WEjA7aPtpl+RaqM=
|
||||||
github.com/go-openapi/swag/jsonname v0.26.0 h1:gV1NFX9M8avo0YSpmWogqfQISigCmpaiNci8cGECU5w=
|
github.com/go-openapi/swag/jsonutils v0.29.1 h1:AFCxs0eQZ24/QyfhVHM2t49rMz7Vv3XCsZQI6yrNy+c=
|
||||||
github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTeIz0ARZxHkAPktJK2M=
|
github.com/go-openapi/swag/jsonutils v0.29.1/go.mod h1:u3+sCfJpttDpcmS5kpm0yxL6GK0eWgODsx8Yw8fcqNM=
|
||||||
github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA=
|
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1 h1:BiiXE31Bx9SfpsMmOQj5KYpUhTZBpLVriVhJDuLuY2o=
|
||||||
github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E=
|
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1/go.mod h1:julgTUKZ9/D0j6O7GKajmRs+812FWxQg/mMpGunWSjg=
|
||||||
github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM=
|
github.com/go-openapi/swag/loading v0.29.1 h1:FCv5fG8UhTdDJa2R7w+5O9Ekpcbw7tt0nFWvmDKGBjc=
|
||||||
github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y=
|
github.com/go-openapi/swag/loading v0.29.1/go.mod h1:N0ESuem4p2oedKal8EJhciqnJ9Q9Wmt83L1CRB3Fouw=
|
||||||
github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko=
|
github.com/go-openapi/swag/pools v0.29.1 h1:NRogYxdEW9SjRM4mkAOji9iefO4MRXq3p/ZJcoQbUKg=
|
||||||
github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg=
|
github.com/go-openapi/swag/pools v0.29.1/go.mod h1:leDcaghjkRAhCuCRv9NfJU5f0mjoU3cT/XZObhMk3pc=
|
||||||
github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg=
|
github.com/go-openapi/swag/stringutils v0.29.1 h1:1ykunK7iJQk1uOO7+oUH1ukbsK85fFCOiCFMOVSY+F0=
|
||||||
github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE=
|
github.com/go-openapi/swag/stringutils v0.29.1/go.mod h1:7fSqZ+z8Qc0tOfAAK0jVa5qFGrnIlRi6n7NeGGrr1vc=
|
||||||
github.com/go-openapi/swag/typeutils v0.26.0 h1:2kdEwdiNWy+JJdOvu5MA2IIg2SylWAFuuyQIKYybfq4=
|
github.com/go-openapi/swag/typeutils v0.29.1 h1:Nzv9nhnlLCRBPQqfOX+7lB6Guju370or8StT+lIOf6M=
|
||||||
github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4YlSKgyZmFh7AlfABNVE=
|
github.com/go-openapi/swag/typeutils v0.29.1/go.mod h1:hxpgDZJVBkBsi/d3MIUosafoFdE5exaQRmVp0zwu3YE=
|
||||||
github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ=
|
github.com/go-openapi/swag/yamlutils v0.29.1 h1:69w3tsBajm7MR/fejLy7HD/3J68Ys1SeeZMEzZ3w2sk=
|
||||||
github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU=
|
github.com/go-openapi/swag/yamlutils v0.29.1/go.mod h1:rgsp3vT/QdWzKwn43CigDwjOGIenPyTZMKnxEM8jZOA=
|
||||||
github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0=
|
github.com/go-openapi/testify/enable/yaml/v2 v2.6.1 h1:Jm+/ze2rMtbD98yen92AhATGLGREDYXG56Xr4gMjEtE=
|
||||||
github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE=
|
github.com/go-openapi/testify/enable/yaml/v2 v2.6.1/go.mod h1:YDPnwCRDu38/oJBVMBVXOUDiJ9cIeBHWvfImHaXqnv4=
|
||||||
github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4=
|
github.com/go-openapi/testify/v2 v2.6.1 h1:6CNJhTjMzgaeaH8WhshcsZNPIvRemiOcFpU7seO/y7Q=
|
||||||
github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
|
github.com/go-openapi/testify/v2 v2.6.1/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
|
||||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||||
@@ -163,8 +157,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
|||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
|
||||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
|
||||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||||
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||||
@@ -194,8 +186,8 @@ github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT
|
|||||||
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
|
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
|
||||||
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
|
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
|
||||||
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
|
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
|
||||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
@@ -210,47 +202,47 @@ github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
|||||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8=
|
github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY=
|
||||||
github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
|
github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
|
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||||
github.com/nats-io/nats-server/v2 v2.12.7 h1:prQ9cPiWHcnwfT81Wi5lU9LL8TLY+7pxDru6fQYLCQQ=
|
github.com/nats-io/nats-server/v2 v2.14.5 h1:M6yeo/Xb7khi97RSEVELof3DForDqmYza3P4tHCPFWw=
|
||||||
github.com/nats-io/nats-server/v2 v2.12.7/go.mod h1:dOnmkprKMluTmTF7/QHZioxlau3sKHUM/LBPy9AiBPw=
|
github.com/nats-io/nats-server/v2 v2.14.5/go.mod h1:1D3iocrisKvWaD1B/imqarTqmaGrWMqALMLbEDo3v7Q=
|
||||||
github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
|
github.com/nats-io/nats.go v1.53.1 h1:Otsq3uLc/kLdjmkNHkXH0jBqwUquwdKFoe3fq6/3/Xo=
|
||||||
github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
|
github.com/nats-io/nats.go v1.53.1/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
|
||||||
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
|
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs=
|
github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs=
|
||||||
github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY=
|
github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY=
|
||||||
github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo=
|
github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k=
|
||||||
github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU=
|
github.com/oapi-codegen/runtime v1.7.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU=
|
||||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
github.com/parquet-go/bitpack v1.1.0 h1:b80hfKZwIJ3Ku684xJSEj/9G0tywQPKOtmTV8si30mo=
|
||||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
github.com/parquet-go/bitpack v1.1.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||||
github.com/parquet-go/jsonlite v1.5.2 h1:8TZzYknFOHUpgjTLf80qbzc+8GdeT/3a3fdXSzhMylE=
|
github.com/parquet-go/jsonlite v1.5.5 h1:TIqJ+b+mYoTvvEt1/ICS4fBacQO0xWwAbvtcrKjEcC8=
|
||||||
github.com/parquet-go/jsonlite v1.5.2/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
|
github.com/parquet-go/jsonlite v1.5.5/go.mod h1:hN57TVzxtr04kaax5XwNgCCsh1KE7BT6dvY6X/zNH90=
|
||||||
github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
|
github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM=
|
||||||
github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
|
github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
|
||||||
github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
|
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
|
||||||
github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||||
github.com/questdb/go-questdb-client/v4 v4.2.0 h1:+d0HJwCjUWMj7zmY6qmhoqTJzTyoYKl+LSTYGN0T8T8=
|
github.com/questdb/go-questdb-client/v4 v4.2.0 h1:+d0HJwCjUWMj7zmY6qmhoqTJzTyoYKl+LSTYGN0T8T8=
|
||||||
github.com/questdb/go-questdb-client/v4 v4.2.0/go.mod h1:/2x93LK1wjM4JX/b5c6q7Yqk22htjWY1lE6p1X8iLbE=
|
github.com/questdb/go-questdb-client/v4 v4.2.0/go.mod h1:/2x93LK1wjM4JX/b5c6q7Yqk22htjWY1lE6p1X8iLbE=
|
||||||
github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
|
github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
|
||||||
@@ -264,13 +256,11 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
|
|||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
|
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
|
||||||
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
|
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
|
||||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
|
||||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
|
||||||
github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE=
|
github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE=
|
||||||
github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
|
github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
|
||||||
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
|
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
|
||||||
github.com/stmcginnis/gofish v0.22.0 h1:OahXohfrIzAXOsWuKDQ7lm/QvdZBg1P2OzFYmbKAd/0=
|
github.com/stmcginnis/gofish v0.24.0 h1:zaBBFNtdSFH/+lJju29HMDHU3suIR+YhygqoJYxW+2Q=
|
||||||
github.com/stmcginnis/gofish v0.22.0/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU=
|
github.com/stmcginnis/gofish v0.24.0/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||||
@@ -289,10 +279,10 @@ github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4
|
|||||||
github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
|
github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
|
||||||
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
|
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
|
||||||
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
|
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
|
||||||
github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI=
|
github.com/urfave/cli/v3 v3.10.1 h1:7Kx9H50hrHbRbyxgO1KP6/BcbiGRz0uYh5YyQ30JEEY=
|
||||||
github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
|
github.com/urfave/cli/v3 v3.10.1/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
|
||||||
github.com/vektah/gqlparser/v2 v2.5.33 h1:lRp8aIeNUNbimf/axZd7ETg24q06hBtPaas+TcvI/7E=
|
github.com/vektah/gqlparser/v2 v2.5.36 h1:CN9mKVHgMkc+XftdOWIhb4HEL8wKSYkFAqhf8booa7s=
|
||||||
github.com/vektah/gqlparser/v2 v2.5.33/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
|
github.com/vektah/gqlparser/v2 v2.5.36/go.mod h1:cAJ9qwVgPaUkWv6Gn8vn0mqOE0Ui5Pn56wNy5396XWo=
|
||||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
|
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
|
||||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
||||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||||
@@ -302,35 +292,35 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
|||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
@@ -338,18 +328,18 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
|
|||||||
@@ -196,8 +196,8 @@ func cleanup() {
|
|||||||
func TestRestApi(t *testing.T) {
|
func TestRestApi(t *testing.T) {
|
||||||
restapi := setup(t)
|
restapi := setup(t)
|
||||||
t.Cleanup(cleanup)
|
t.Cleanup(cleanup)
|
||||||
testData := schema.JobData{
|
testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"load_one": map[schema.MetricScope]*schema.JobMetric{
|
"load_one": {
|
||||||
schema.MetricScopeNode: {
|
schema.MetricScopeNode: {
|
||||||
Unit: schema.Unit{Base: "load"},
|
Unit: schema.Unit{Base: "load"},
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -210,9 +210,9 @@ func TestRestApi(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
|
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, resampleAlgo string) (schema.JobData, error) {
|
||||||
return testData, nil
|
return testData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -355,7 +355,7 @@ func TestRestApi(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.Run("CheckArchive", func(t *testing.T) {
|
t.Run("CheckArchive", func(t *testing.T) {
|
||||||
data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
|
data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60, "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -497,8 +497,8 @@ func TestStopJobWithReusedJobId(t *testing.T) {
|
|||||||
restapi := setup(t)
|
restapi := setup(t)
|
||||||
t.Cleanup(cleanup)
|
t.Cleanup(cleanup)
|
||||||
|
|
||||||
testData := schema.JobData{
|
testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"load_one": map[schema.MetricScope]*schema.JobMetric{
|
"load_one": {
|
||||||
schema.MetricScopeNode: {
|
schema.MetricScopeNode: {
|
||||||
Unit: schema.Unit{Base: "load"},
|
Unit: schema.Unit{Base: "load"},
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -511,9 +511,9 @@ func TestStopJobWithReusedJobId(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
|
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, resampleAlgo string) (schema.JobData, error) {
|
||||||
return testData, nil
|
return testData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -309,7 +309,7 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if r.URL.Query().Get("all-metrics") == "true" {
|
if r.URL.Query().Get("all-metrics") == "true" {
|
||||||
data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)
|
data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution, config.ResampleAlgo())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
|
cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
|
||||||
return
|
return
|
||||||
@@ -405,14 +405,14 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) {
|
|||||||
resolution = max(resolution, mc.Timestep)
|
resolution = max(resolution, mc.Timestep)
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution)
|
data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution, config.ResampleAlgo())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
|
cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
res := []*JobMetricWithName{}
|
res := []*JobMetricWithName{}
|
||||||
for name, md := range data {
|
for name, md := range data.Metrics {
|
||||||
for scope, metric := range md {
|
for scope, metric := range md {
|
||||||
res = append(res, &JobMetricWithName{
|
res = append(res, &JobMetricWithName{
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -1086,7 +1086,7 @@ func (api *RestAPI) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
resolver := graph.GetResolverInstance()
|
resolver := graph.GetResolverInstance()
|
||||||
data, err := resolver.Query().JobMetrics(r.Context(), id, metrics, scopes, nil)
|
data, err := resolver.Query().JobMetrics(r.Context(), id, metrics, scopes, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err := json.NewEncoder(rw).Encode(Response{
|
if err := json.NewEncoder(rw).Encode(Response{
|
||||||
Error: &struct {
|
Error: &struct {
|
||||||
|
|||||||
@@ -531,8 +531,8 @@ func TestNatsHandleStopJob(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
testData := schema.JobData{
|
testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"load_one": map[schema.MetricScope]*schema.JobMetric{
|
"load_one": {
|
||||||
schema.MetricScopeNode: {
|
schema.MetricScopeNode: {
|
||||||
Unit: schema.Unit{Base: "load"},
|
Unit: schema.Unit{Base: "load"},
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -545,9 +545,9 @@ func TestNatsHandleStopJob(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
|
metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, resampleAlgo string) (schema.JobData, error) {
|
||||||
return testData, nil
|
return testData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,6 @@ func (api *RestAPI) getUsers(rw http.ResponseWriter, r *http.Request) {
|
|||||||
// @security ApiKeyAuth
|
// @security ApiKeyAuth
|
||||||
// @router /api/user/{id} [post]
|
// @router /api/user/{id} [post]
|
||||||
func (api *RestAPI) updateUserByRequest(rw http.ResponseWriter, r *http.Request) {
|
func (api *RestAPI) updateUserByRequest(rw http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
|
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
|
||||||
handleError(fmt.Errorf("only admins are allowed to update a user"), http.StatusForbidden, rw)
|
handleError(fmt.Errorf("only admins are allowed to update a user"), http.StatusForbidden, rw)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -59,15 +59,15 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
|
|||||||
scopes = append(scopes, schema.MetricScopeAccelerator)
|
scopes = append(scopes, schema.MetricScopeAccelerator)
|
||||||
}
|
}
|
||||||
|
|
||||||
jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
|
jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0, "") // 0 Resulotion-Value retrieves highest res (60s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Error("Error wile loading job data for archiving")
|
cclog.Error("Error wile loading job data for archiving")
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
job.Statistics = make(map[string]schema.JobStatistics)
|
job.Statistics = schema.JobStatisticsSet{Metrics: make(map[string]schema.JobStatistics, len(jobData.Metrics))}
|
||||||
|
|
||||||
for metric, data := range jobData {
|
for metric, data := range jobData.Metrics {
|
||||||
avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
||||||
nodeData, ok := data["node"]
|
nodeData, ok := data["node"]
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -82,7 +82,7 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Round AVG Result to 2 Digits
|
// Round AVG Result to 2 Digits
|
||||||
job.Statistics[metric] = schema.JobStatistics{
|
job.Statistics.Metrics[metric] = schema.JobStatistics{
|
||||||
Unit: schema.Unit{
|
Unit: schema.Unit{
|
||||||
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
|
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
|
||||||
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
|
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
|
||||||
|
|||||||
+218
-3
@@ -9,6 +9,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -36,12 +38,25 @@ type LdapConfig struct {
|
|||||||
// Password for the LDAP admin account used for syncing (optional).
|
// Password for the LDAP admin account used for syncing (optional).
|
||||||
// Overridden by the LDAP_ADMIN_PASSWORD environment variable when set.
|
// Overridden by the LDAP_ADMIN_PASSWORD environment variable when set.
|
||||||
SyncPassword string `json:"sync-password"`
|
SyncPassword string `json:"sync-password"`
|
||||||
|
|
||||||
|
// Maps an elevated role (admin/support/api/manager) to an LDAP filter.
|
||||||
|
// An account matching the filter is granted that role; LDAP is authoritative
|
||||||
|
// for every role listed here (it is both added and removed to match group
|
||||||
|
// membership). Roles not listed here are never touched. Empty/absent means
|
||||||
|
// roles are never modified by LDAP.
|
||||||
|
RoleFilters map[string]string `json:"role-filters"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type LdapAuthenticator struct {
|
type LdapAuthenticator struct {
|
||||||
syncPassword string
|
syncPassword string
|
||||||
UserAttr string
|
UserAttr string
|
||||||
UIDAttr string
|
UIDAttr string
|
||||||
|
|
||||||
|
// roleFilters holds the validated subset of LdapConfig.RoleFilters.
|
||||||
|
roleFilters map[string]string
|
||||||
|
// managedRoles is the sorted list of roles LDAP is authoritative for
|
||||||
|
// (the keys of roleFilters). Empty when no role filters are configured.
|
||||||
|
managedRoles []string
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Authenticator = (*LdapAuthenticator)(nil)
|
var _ Authenticator = (*LdapAuthenticator)(nil)
|
||||||
@@ -64,6 +79,26 @@ func (la *LdapAuthenticator) Init() error {
|
|||||||
la.UIDAttr = "uid"
|
la.UIDAttr = "uid"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate the optional role filters. Invalid keys are dropped with a
|
||||||
|
// warning rather than failing startup. The baseline "user" role cannot be
|
||||||
|
// LDAP-managed (it is always granted), and an empty filter is meaningless.
|
||||||
|
la.roleFilters = make(map[string]string)
|
||||||
|
for role, filter := range Keys.LdapConfig.RoleFilters {
|
||||||
|
role = strings.ToLower(role)
|
||||||
|
if !schema.IsValidRole(role) || role == schema.GetRoleString(schema.RoleUser) ||
|
||||||
|
role == schema.GetRoleString(schema.RoleAnonymous) {
|
||||||
|
cclog.Warnf("LDAP: ignoring role-filter for invalid or non-assignable role '%s'", role)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(filter) == "" {
|
||||||
|
cclog.Warnf("LDAP: ignoring empty role-filter for role '%s'", role)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
la.roleFilters[role] = filter
|
||||||
|
la.managedRoles = append(la.managedRoles, role)
|
||||||
|
}
|
||||||
|
sort.Strings(la.managedRoles)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,6 +112,25 @@ func (la *LdapAuthenticator) CanLogin(
|
|||||||
|
|
||||||
if user != nil {
|
if user != nil {
|
||||||
if user.AuthSource == schema.AuthViaLDAP {
|
if user.AuthSource == schema.AuthViaLDAP {
|
||||||
|
// Refresh elevated roles from LDAP when role filters are configured
|
||||||
|
// and role updates on login are enabled. Without role filters this
|
||||||
|
// stays a fast path with no extra LDAP query.
|
||||||
|
if len(la.managedRoles) > 0 && lc.UpdateUserOnLogin {
|
||||||
|
if l, err := la.getLdapConnection(true); err != nil {
|
||||||
|
cclog.Warnf("LDAP: skipping role refresh for user '%s': connection error", user.Username)
|
||||||
|
} else {
|
||||||
|
defer l.Close()
|
||||||
|
if matched, err := la.matchRoles(l, user.Username); err == nil {
|
||||||
|
roles := mergeLdapRoles(user.Roles, la.managedRoles, matched, user.Projects)
|
||||||
|
current := append([]string{}, user.Roles...)
|
||||||
|
sort.Strings(current)
|
||||||
|
if !slices.Equal(roles, current) {
|
||||||
|
user.Roles = roles
|
||||||
|
handleLdapUser(user)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return user, true
|
return user, true
|
||||||
}
|
}
|
||||||
} else if lc.SyncUserOnLogin {
|
} else if lc.SyncUserOnLogin {
|
||||||
@@ -106,10 +160,18 @@ func (la *LdapAuthenticator) CanLogin(
|
|||||||
}
|
}
|
||||||
|
|
||||||
entry := sr.Entries[0]
|
entry := sr.Entries[0]
|
||||||
|
|
||||||
|
roles := []string{schema.GetRoleString(schema.RoleUser)}
|
||||||
|
if len(la.managedRoles) > 0 {
|
||||||
|
if matched, err := la.matchRoles(l, username); err == nil {
|
||||||
|
roles = mergeLdapRoles(nil, la.managedRoles, matched, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
user = &schema.User{
|
user = &schema.User{
|
||||||
Username: username,
|
Username: username,
|
||||||
Name: entry.GetAttributeValue(la.UserAttr),
|
Name: entry.GetAttributeValue(la.UserAttr),
|
||||||
Roles: []string{schema.GetRoleString(schema.RoleUser)},
|
Roles: roles,
|
||||||
Projects: make([]string, 0),
|
Projects: make([]string, 0),
|
||||||
AuthType: schema.AuthSession,
|
AuthType: schema.AuthSession,
|
||||||
AuthSource: schema.AuthViaLDAP,
|
AuthSource: schema.AuthViaLDAP,
|
||||||
@@ -194,6 +256,28 @@ func (la *LdapAuthenticator) Sync() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Evaluate configured role filters once over the whole base. Empty when no
|
||||||
|
// role filters are configured, in which case role handling is a no-op and
|
||||||
|
// behaviour is identical to before.
|
||||||
|
matched, err := la.matchRolesBulk(l)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Current roles/projects of users that already hold a non-default role, so
|
||||||
|
// existing users can be reconciled without a per-user lookup.
|
||||||
|
currentRoles := map[string]*schema.User{}
|
||||||
|
if len(la.managedRoles) > 0 {
|
||||||
|
specials, err := ur.ListUsers(true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, u := range specials {
|
||||||
|
currentRoles[u.Username] = u
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
userRole := schema.GetRoleString(schema.RoleUser)
|
||||||
for username, where := range users {
|
for username, where := range users {
|
||||||
if where == InDB && lc.SyncDelOldUsers {
|
if where == InDB && lc.SyncDelOldUsers {
|
||||||
if err := ur.DelUser(username); err != nil {
|
if err := ur.DelUser(username); err != nil {
|
||||||
@@ -204,25 +288,156 @@ func (la *LdapAuthenticator) Sync() error {
|
|||||||
} else if where == InLdap {
|
} else if where == InLdap {
|
||||||
name := newnames[username]
|
name := newnames[username]
|
||||||
|
|
||||||
|
roles := []string{userRole}
|
||||||
|
if len(la.managedRoles) > 0 {
|
||||||
|
roles = mergeLdapRoles(nil, la.managedRoles, matched[username], nil)
|
||||||
|
}
|
||||||
|
|
||||||
user := &schema.User{
|
user := &schema.User{
|
||||||
Username: username,
|
Username: username,
|
||||||
Name: name,
|
Name: name,
|
||||||
Roles: []string{schema.GetRoleString(schema.RoleUser)},
|
Roles: roles,
|
||||||
Projects: make([]string, 0),
|
Projects: make([]string, 0),
|
||||||
AuthSource: schema.AuthViaLDAP,
|
AuthSource: schema.AuthViaLDAP,
|
||||||
}
|
}
|
||||||
|
|
||||||
cclog.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
|
cclog.Debugf("sync: add %v (name: %v, roles: %v, ldap: true)", username, name, roles)
|
||||||
if err := ur.AddUserIfNotExists(user); err != nil {
|
if err := ur.AddUserIfNotExists(user); err != nil {
|
||||||
cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
|
cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
} else if where == InBoth && len(la.managedRoles) > 0 {
|
||||||
|
// Reconcile elevated roles for existing users: LDAP is authoritative
|
||||||
|
// for the managed roles, all other roles are preserved.
|
||||||
|
cur := []string{userRole}
|
||||||
|
var projects []string
|
||||||
|
if u, ok := currentRoles[username]; ok {
|
||||||
|
cur = u.Roles
|
||||||
|
projects = u.Projects
|
||||||
|
}
|
||||||
|
|
||||||
|
roles := mergeLdapRoles(cur, la.managedRoles, matched[username], projects)
|
||||||
|
sortedCur := append([]string{}, cur...)
|
||||||
|
sort.Strings(sortedCur)
|
||||||
|
if !slices.Equal(roles, sortedCur) {
|
||||||
|
cclog.Debugf("sync: update %v roles %v -> %v", username, cur, roles)
|
||||||
|
if err := ur.UpdateRoles(username, roles); err != nil {
|
||||||
|
cclog.Errorf("User '%s' LDAP: role update failed: %v", username, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mergeLdapRoles computes the role set for an LDAP user. Every current role that
|
||||||
|
// LDAP does not manage is preserved, the baseline "user" role is always present,
|
||||||
|
// and the matched managed roles are added. managed is the set of roles LDAP is
|
||||||
|
// authoritative for; matched is the subset of those the account currently
|
||||||
|
// qualifies for. projects is used to guard manager removal: a manager that still
|
||||||
|
// has assigned projects keeps the role even if it is no longer matched (mirrors
|
||||||
|
// UserRepository.RemoveRole). The result is deduplicated and sorted.
|
||||||
|
func mergeLdapRoles(current, managed, matched, projects []string) []string {
|
||||||
|
managedSet := make(map[string]bool, len(managed))
|
||||||
|
for _, r := range managed {
|
||||||
|
managedSet[r] = true
|
||||||
|
}
|
||||||
|
matchedSet := make(map[string]bool, len(matched))
|
||||||
|
for _, r := range matched {
|
||||||
|
matchedSet[r] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
result := map[string]bool{schema.GetRoleString(schema.RoleUser): true}
|
||||||
|
|
||||||
|
// Preserve roles LDAP does not manage (e.g. a manually granted manager).
|
||||||
|
for _, r := range current {
|
||||||
|
if !managedSet[r] {
|
||||||
|
result[r] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add managed roles the account currently qualifies for.
|
||||||
|
for _, r := range managed {
|
||||||
|
if matchedSet[r] {
|
||||||
|
result[r] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guard: do not strip a manager that still has assigned projects.
|
||||||
|
managerRole := schema.GetRoleString(schema.RoleManager)
|
||||||
|
if managedSet[managerRole] && !matchedSet[managerRole] && len(projects) > 0 &&
|
||||||
|
slices.Contains(current, managerRole) {
|
||||||
|
cclog.Warnf("LDAP: keeping role 'manager' despite no filter match: user still has assigned project(s): %v", projects)
|
||||||
|
result[managerRole] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
roles := make([]string, 0, len(result))
|
||||||
|
for r := range result {
|
||||||
|
roles = append(roles, r)
|
||||||
|
}
|
||||||
|
sort.Strings(roles)
|
||||||
|
return roles
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchRoles evaluates all configured role filters for a single user and returns
|
||||||
|
// the managed roles the account qualifies for. Used on the login path where only
|
||||||
|
// one user is inspected.
|
||||||
|
func (la *LdapAuthenticator) matchRoles(l *ldap.Conn, username string) ([]string, error) {
|
||||||
|
if len(la.managedRoles) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
matched := make([]string, 0, len(la.managedRoles))
|
||||||
|
for _, role := range la.managedRoles {
|
||||||
|
filter := fmt.Sprintf("(&(%s=%s)%s)", la.UIDAttr, ldap.EscapeFilter(username), la.roleFilters[role])
|
||||||
|
sr, err := l.Search(ldap.NewSearchRequest(
|
||||||
|
Keys.LdapConfig.UserBase,
|
||||||
|
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
|
||||||
|
filter,
|
||||||
|
[]string{la.UIDAttr}, nil))
|
||||||
|
if err != nil {
|
||||||
|
cclog.Warnf("LDAP: role filter search for role '%s' failed: %v", role, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(sr.Entries) > 0 {
|
||||||
|
matched = append(matched, role)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return matched, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchRolesBulk evaluates every configured role filter once over the whole user
|
||||||
|
// base and returns a username -> matched managed roles mapping. Used by Sync,
|
||||||
|
// it costs one LDAP search per configured role rather than one per user.
|
||||||
|
func (la *LdapAuthenticator) matchRolesBulk(l *ldap.Conn) (map[string][]string, error) {
|
||||||
|
matched := map[string][]string{}
|
||||||
|
if len(la.managedRoles) == 0 {
|
||||||
|
return matched, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
lc := Keys.LdapConfig
|
||||||
|
for _, role := range la.managedRoles {
|
||||||
|
filter := fmt.Sprintf("(&%s%s)", lc.UserFilter, la.roleFilters[role])
|
||||||
|
sr, err := l.Search(ldap.NewSearchRequest(
|
||||||
|
lc.UserBase,
|
||||||
|
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
|
||||||
|
filter,
|
||||||
|
[]string{la.UIDAttr}, nil))
|
||||||
|
if err != nil {
|
||||||
|
cclog.Warnf("LDAP: role filter search for role '%s' failed: %v", role, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, entry := range sr.Entries {
|
||||||
|
if username := entry.GetAttributeValue(la.UIDAttr); username != "" {
|
||||||
|
matched[username] = append(matched[username], role)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return matched, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) {
|
func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) {
|
||||||
lc := Keys.LdapConfig
|
lc := Keys.LdapConfig
|
||||||
conn, err := ldap.DialURL(lc.URL,
|
conn, err := ldap.DialURL(lc.URL,
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeLdapRoles(t *testing.T) {
|
||||||
|
var (
|
||||||
|
user = schema.GetRoleString(schema.RoleUser)
|
||||||
|
admin = schema.GetRoleString(schema.RoleAdmin)
|
||||||
|
support = schema.GetRoleString(schema.RoleSupport)
|
||||||
|
api = schema.GetRoleString(schema.RoleAPI)
|
||||||
|
manager = schema.GetRoleString(schema.RoleManager)
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
current []string
|
||||||
|
managed []string
|
||||||
|
matched []string
|
||||||
|
projects []string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no managed roles is a no-op keeping user baseline",
|
||||||
|
current: []string{user},
|
||||||
|
managed: nil,
|
||||||
|
matched: nil,
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "add matched elevated role to plain user",
|
||||||
|
current: []string{user},
|
||||||
|
managed: []string{admin, support},
|
||||||
|
matched: []string{admin},
|
||||||
|
want: []string{admin, user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "remove managed role when no longer matched",
|
||||||
|
current: []string{admin, user},
|
||||||
|
managed: []string{admin},
|
||||||
|
matched: nil,
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preserve non-managed roles (manager, api)",
|
||||||
|
current: []string{api, manager, user},
|
||||||
|
managed: []string{admin, support},
|
||||||
|
matched: []string{support},
|
||||||
|
want: []string{api, manager, support, user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "managed manager with projects is not removed",
|
||||||
|
current: []string{manager, user},
|
||||||
|
managed: []string{manager},
|
||||||
|
matched: nil,
|
||||||
|
projects: []string{"projA"},
|
||||||
|
want: []string{manager, user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "managed manager without projects is removed",
|
||||||
|
current: []string{manager, user},
|
||||||
|
managed: []string{manager},
|
||||||
|
matched: nil,
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "user baseline always present even if absent in current",
|
||||||
|
current: []string{admin},
|
||||||
|
managed: []string{admin},
|
||||||
|
matched: []string{admin},
|
||||||
|
want: []string{admin, user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "new user (nil current) gets matched roles plus baseline",
|
||||||
|
current: nil,
|
||||||
|
managed: []string{admin, support, api},
|
||||||
|
matched: []string{admin, api},
|
||||||
|
want: []string{admin, api, user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "result is deduplicated",
|
||||||
|
current: []string{admin, admin, user},
|
||||||
|
managed: []string{admin},
|
||||||
|
matched: []string{admin},
|
||||||
|
want: []string{admin, user},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := mergeLdapRoles(tt.current, tt.managed, tt.matched, tt.projects)
|
||||||
|
if !reflect.DeepEqual(got, tt.want) {
|
||||||
|
t.Errorf("mergeLdapRoles() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+49
-23
@@ -12,6 +12,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/ClusterCockpit/cc-backend/internal/repository"
|
"github.com/ClusterCockpit/cc-backend/internal/repository"
|
||||||
@@ -34,6 +36,12 @@ type OpenIDConfig struct {
|
|||||||
// OAuth2 client secret for the OIDC provider.
|
// OAuth2 client secret for the OIDC provider.
|
||||||
// Overridden by the OID_CLIENT_SECRET environment variable when set.
|
// Overridden by the OID_CLIENT_SECRET environment variable when set.
|
||||||
ClientSecret string `json:"client-secret"`
|
ClientSecret string `json:"client-secret"`
|
||||||
|
|
||||||
|
// Maps an OIDC role/group claim value to a CC role (admin/support/api/manager/user).
|
||||||
|
// This is the sole source of roles: a token role grants a CC role only if it is
|
||||||
|
// listed here. Unmapped token roles are ignored (no identity fallback), so literal
|
||||||
|
// CC role names must be mapped explicitly. Users without any mapped role get "user".
|
||||||
|
RoleMapping map[string]string `json:"role-mapping"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type OIDC struct {
|
type OIDC struct {
|
||||||
@@ -41,6 +49,9 @@ type OIDC struct {
|
|||||||
provider *oidc.Provider
|
provider *oidc.Provider
|
||||||
authentication *Authentication
|
authentication *Authentication
|
||||||
clientID string
|
clientID string
|
||||||
|
// roleMapping is the validated subset of OpenIDConfig.RoleMapping
|
||||||
|
// (IdP role/group name -> CC role).
|
||||||
|
roleMapping map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func randString(nByte int) (string, error) {
|
func randString(nByte int) (string, error) {
|
||||||
@@ -89,11 +100,47 @@ func NewOIDC(a *Authentication) *OIDC {
|
|||||||
Scopes: []string{oidc.ScopeOpenID, "profile", "roles"},
|
Scopes: []string{oidc.ScopeOpenID, "profile", "roles"},
|
||||||
}
|
}
|
||||||
|
|
||||||
oa := &OIDC{provider: provider, client: client, clientID: clientID, authentication: a}
|
// Validate the optional role mapping once at startup. Invalid targets are
|
||||||
|
// dropped with a warning rather than failing startup. IdP names (keys) are
|
||||||
|
// kept verbatim so they match the raw token claim values.
|
||||||
|
roleMapping := make(map[string]string)
|
||||||
|
for name, ccRole := range Keys.OpenIDConfig.RoleMapping {
|
||||||
|
role := strings.ToLower(ccRole)
|
||||||
|
if !schema.IsValidRole(role) || role == schema.GetRoleString(schema.RoleAnonymous) {
|
||||||
|
cclog.Warnf("OIDC: ignoring role-mapping '%s' -> '%s': invalid or non-assignable target role", name, ccRole)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
roleMapping[name] = role
|
||||||
|
}
|
||||||
|
|
||||||
|
oa := &OIDC{provider: provider, client: client, clientID: clientID, authentication: a, roleMapping: roleMapping}
|
||||||
|
|
||||||
return oa
|
return oa
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mapOIDCRoles translates raw OIDC role/group names into the CC role set using
|
||||||
|
// only the configured mapping (IdP name -> CC role). Unmapped names are ignored.
|
||||||
|
// Always returns at least [user]. The result is deduplicated and sorted.
|
||||||
|
func mapOIDCRoles(oidcRoles []string, mapping map[string]string) []string {
|
||||||
|
roleSet := make(map[string]bool)
|
||||||
|
for _, r := range oidcRoles {
|
||||||
|
if cc, ok := mapping[r]; ok {
|
||||||
|
roleSet[cc] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(roleSet) == 0 {
|
||||||
|
return []string{schema.GetRoleString(schema.RoleUser)}
|
||||||
|
}
|
||||||
|
|
||||||
|
roles := make([]string, 0, len(roleSet))
|
||||||
|
for role := range roleSet {
|
||||||
|
roles = append(roles, role)
|
||||||
|
}
|
||||||
|
sort.Strings(roles)
|
||||||
|
return roles
|
||||||
|
}
|
||||||
|
|
||||||
func (oa *OIDC) RegisterEndpoints(r chi.Router) {
|
func (oa *OIDC) RegisterEndpoints(r chi.Router) {
|
||||||
r.HandleFunc("/oidc-login", oa.OAuth2Login)
|
r.HandleFunc("/oidc-login", oa.OAuth2Login)
|
||||||
r.HandleFunc("/oidc-callback", oa.OAuth2Callback)
|
r.HandleFunc("/oidc-callback", oa.OAuth2Callback)
|
||||||
@@ -240,28 +287,7 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
|
|||||||
oidcRoles = append(oidcRoles, access.Roles...)
|
oidcRoles = append(oidcRoles, access.Roles...)
|
||||||
}
|
}
|
||||||
|
|
||||||
roleSet := make(map[string]bool)
|
roles := mapOIDCRoles(oidcRoles, oa.roleMapping)
|
||||||
for _, r := range oidcRoles {
|
|
||||||
switch r {
|
|
||||||
case "user":
|
|
||||||
roleSet[schema.GetRoleString(schema.RoleUser)] = true
|
|
||||||
case "admin":
|
|
||||||
roleSet[schema.GetRoleString(schema.RoleAdmin)] = true
|
|
||||||
case "manager":
|
|
||||||
roleSet[schema.GetRoleString(schema.RoleManager)] = true
|
|
||||||
case "support":
|
|
||||||
roleSet[schema.GetRoleString(schema.RoleSupport)] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var roles []string
|
|
||||||
for role := range roleSet {
|
|
||||||
roles = append(roles, role)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(roles) == 0 {
|
|
||||||
roles = append(roles, schema.GetRoleString(schema.RoleUser))
|
|
||||||
}
|
|
||||||
|
|
||||||
user := &schema.User{
|
user := &schema.User{
|
||||||
Username: username,
|
Username: username,
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMapOIDCRoles(t *testing.T) {
|
||||||
|
var (
|
||||||
|
user = schema.GetRoleString(schema.RoleUser)
|
||||||
|
admin = schema.GetRoleString(schema.RoleAdmin)
|
||||||
|
support = schema.GetRoleString(schema.RoleSupport)
|
||||||
|
api = schema.GetRoleString(schema.RoleAPI)
|
||||||
|
)
|
||||||
|
|
||||||
|
mapping := map[string]string{
|
||||||
|
"cc-admins": admin,
|
||||||
|
"cc-support": support,
|
||||||
|
"cc-api": api,
|
||||||
|
"staff": support, // second name mapping to the same role
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
oidcRoles []string
|
||||||
|
mapping map[string]string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "explicit mapping to elevated roles",
|
||||||
|
oidcRoles: []string{"cc-admins", "cc-api"},
|
||||||
|
mapping: mapping,
|
||||||
|
want: []string{admin, api},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unmapped names are ignored (no identity fallback)",
|
||||||
|
oidcRoles: []string{"admin", "support", "unknown"},
|
||||||
|
mapping: mapping,
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mix of mapped and unmapped keeps only mapped",
|
||||||
|
oidcRoles: []string{"cc-admins", "admin", "noise"},
|
||||||
|
mapping: mapping,
|
||||||
|
want: []string{admin},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty token roles default to user",
|
||||||
|
oidcRoles: nil,
|
||||||
|
mapping: mapping,
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no mapping configured defaults to user",
|
||||||
|
oidcRoles: []string{"cc-admins", "admin"},
|
||||||
|
mapping: map[string]string{},
|
||||||
|
want: []string{user},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate target roles are deduplicated and sorted",
|
||||||
|
oidcRoles: []string{"cc-support", "staff", "cc-admins"},
|
||||||
|
mapping: mapping,
|
||||||
|
want: []string{admin, support},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := mapOIDCRoles(tt.oidcRoles, tt.mapping)
|
||||||
|
if !reflect.DeepEqual(got, tt.want) {
|
||||||
|
t.Errorf("mapOIDCRoles() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
-4
@@ -5,8 +5,13 @@
|
|||||||
|
|
||||||
package auth
|
package auth
|
||||||
|
|
||||||
|
// configSchema describes the "auth" section of config.json. Every subsection is
|
||||||
|
// optional: Init() logs and continues when "jwts", "oidc" or "ldap" is absent,
|
||||||
|
// so only the fields within a configured subsection are required.
|
||||||
var configSchema = `
|
var configSchema = `
|
||||||
{
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
"jwts": {
|
"jwts": {
|
||||||
"description": "For JWT token authentication.",
|
"description": "For JWT token authentication.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -55,6 +60,7 @@ var configSchema = `
|
|||||||
"required": ["max-age"]
|
"required": ["max-age"]
|
||||||
},
|
},
|
||||||
"oidc": {
|
"oidc": {
|
||||||
|
"description": "For OpenID Connect authentication.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"provider": {
|
"provider": {
|
||||||
@@ -76,6 +82,13 @@ var configSchema = `
|
|||||||
"client-secret": {
|
"client-secret": {
|
||||||
"description": "OAuth2 client secret for the OIDC provider. Overridden by the OID_CLIENT_SECRET environment variable when set.",
|
"description": "OAuth2 client secret for the OIDC provider. Overridden by the OID_CLIENT_SECRET environment variable when set.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
},
|
||||||
|
"role-mapping": {
|
||||||
|
"description": "Maps an OIDC role/group claim value (from realm_access/resource_access) to a CC role. Valid target roles: admin, support, api, manager, user. This is the sole source of roles: only mapped roles are honored, unmapped token roles are ignored (literal CC role names must be mapped explicitly). Users without any mapped role receive the base 'user' role.",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": ["provider"]
|
"required": ["provider"]
|
||||||
@@ -131,9 +144,16 @@ var configSchema = `
|
|||||||
"sync-password": {
|
"sync-password": {
|
||||||
"description": "Password for the LDAP admin account used for syncing. Overridden by the LDAP_ADMIN_PASSWORD environment variable when set.",
|
"description": "Password for the LDAP admin account used for syncing. Overridden by the LDAP_ADMIN_PASSWORD environment variable when set.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
},
|
||||||
|
"role-filters": {
|
||||||
|
"description": "Maps an elevated role to an LDAP filter; accounts matching the filter are granted that role. LDAP is authoritative for every role listed here (roles are added and removed to match group membership), while roles not listed are preserved. Applied during sync and at login. Valid keys: admin, support, api, manager.",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": ["url", "user-base", "search-dn", "user-bind", "user-filter"]
|
"required": ["url", "user-base", "search-dn", "user-bind", "user-filter"]
|
||||||
},
|
}
|
||||||
"required": ["jwts"]
|
}
|
||||||
}`
|
}`
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// compileConfigSchema guards against the schema silently degrading into a
|
||||||
|
// document with no constraints (which is what happens when the "type"/
|
||||||
|
// "properties" wrapper is missing: every instance validates).
|
||||||
|
func compileConfigSchema(t *testing.T) *jsonschema.Schema {
|
||||||
|
t.Helper()
|
||||||
|
sch, err := jsonschema.CompileString("schema.json", configSchema)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compiling auth config schema: %v", err)
|
||||||
|
}
|
||||||
|
return sch
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAuthConfig(t *testing.T, sch *jsonschema.Schema, raw string) error {
|
||||||
|
t.Helper()
|
||||||
|
var v any
|
||||||
|
if err := json.Unmarshal([]byte(raw), &v); err != nil {
|
||||||
|
t.Fatalf("invalid test json: %v", err)
|
||||||
|
}
|
||||||
|
return sch.Validate(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigSchemaAccepts(t *testing.T) {
|
||||||
|
sch := compileConfigSchema(t)
|
||||||
|
|
||||||
|
tests := map[string]string{
|
||||||
|
"jwts only": `{"jwts":{"max-age":"2000h"}}`,
|
||||||
|
"oidc only": `{"oidc":{"provider":"http://localhost:8080/realms/cc"}}`,
|
||||||
|
"empty": `{}`,
|
||||||
|
"ldap": `{"ldap":{"url":"ldap://127.0.0.1:389","user-base":"ou=users,dc=example,dc=com","search-dn":"cn=admin,dc=example,dc=com","user-bind":"uid={username},ou=users,dc=example,dc=com","user-filter":"(objectclass=posixAccount)","role-filters":{"admin":"(memberOf=cn=cc-admin,ou=groups,dc=example,dc=com)"}}}`,
|
||||||
|
"role-mapping": `{"oidc":{"provider":"http://p","role-mapping":{"cc-admin":"admin"}}}`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, raw := range tests {
|
||||||
|
if err := validateAuthConfig(t, sch, raw); err != nil {
|
||||||
|
t.Errorf("%s: unexpected validation error: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigSchemaRejects(t *testing.T) {
|
||||||
|
sch := compileConfigSchema(t)
|
||||||
|
|
||||||
|
tests := map[string]string{
|
||||||
|
"jwts without max-age": `{"jwts":{"public-key":"abc"}}`,
|
||||||
|
"jwts max-age not a string": `{"jwts":{"max-age":2000}}`,
|
||||||
|
"oidc without provider": `{"oidc":{"client-id":"cc-backend"}}`,
|
||||||
|
"ldap without user-bind": `{"ldap":{"url":"ldap://127.0.0.1:389","user-base":"ou=users","search-dn":"cn=admin","user-filter":"(objectclass=posixAccount)"}}`,
|
||||||
|
"role-filters not strings": `{"ldap":{"url":"u","user-base":"b","search-dn":"d","user-bind":"ub","user-filter":"f","role-filters":{"admin":true}}}`,
|
||||||
|
"jwts not an object": `{"jwts":"2000h"}`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, raw := range tests {
|
||||||
|
if err := validateAuthConfig(t, sch, raw); err == nil {
|
||||||
|
t.Errorf("%s: expected validation error, got none", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConfigSchemaExampleConfigs validates the auth sections of the shipped
|
||||||
|
// example configurations.
|
||||||
|
func TestConfigSchemaExampleConfigs(t *testing.T) {
|
||||||
|
sch := compileConfigSchema(t)
|
||||||
|
|
||||||
|
for _, path := range []string{
|
||||||
|
"../../configs/config.json",
|
||||||
|
"../../configs/config-demo.json",
|
||||||
|
"../../configs/config-large.json",
|
||||||
|
} {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var sections map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, §ions); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
authCfg, ok := sections["auth"]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var v any
|
||||||
|
if err := json.Unmarshal(authCfg, &v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := sch.Validate(v); err != nil {
|
||||||
|
t.Errorf("%s: auth section does not validate: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -118,12 +118,12 @@ type NodeStateRetention struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResampleConfig struct {
|
type ResampleConfig struct {
|
||||||
// Minimum number of points to trigger resampling of data
|
// Default resample policy when no user preference is set ("low", "medium", "high")
|
||||||
MinimumPoints int `json:"minimum-points"`
|
DefaultPolicy string `json:"default-policy"`
|
||||||
// Array of resampling target resolutions, in seconds; Example: [600,300,60]
|
// Default resample algorithm when no user preference is set ("lttb", "average", "simple")
|
||||||
Resolutions []int `json:"resolutions"`
|
DefaultAlgo string `json:"default-algo"`
|
||||||
// Trigger next zoom level at less than this many visible datapoints
|
// Policy-derived target point count (set dynamically from user preference, not from config.json)
|
||||||
Trigger int `json:"trigger"`
|
TargetPoints int `json:"targetPoints,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type NATSConfig struct {
|
type NATSConfig struct {
|
||||||
@@ -171,7 +171,56 @@ func Init(mainConfig json.RawMessage) {
|
|||||||
cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
|
cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 {
|
initResampler()
|
||||||
resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints)
|
}
|
||||||
|
|
||||||
|
// initResampler aligns the resampler's MinimumRequiredPoints threshold with the
|
||||||
|
// configured policy's target point count. The resampler must be allowed to act
|
||||||
|
// exactly when a series is longer than that target; a different threshold here
|
||||||
|
// silently drops resample requests for a band of job durations.
|
||||||
|
func initResampler() {
|
||||||
|
if Keys.EnableResampling == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := Keys.EnableResampling.DefaultPolicy
|
||||||
|
if policy == "" {
|
||||||
|
policy = DefaultResamplePolicy
|
||||||
|
}
|
||||||
|
resampler.SetMinimumRequiredPoints(TargetPointsForPolicy(policy))
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultResamplePolicy is used when no resample policy is configured.
|
||||||
|
const DefaultResamplePolicy = "medium"
|
||||||
|
|
||||||
|
// DefaultResampleAlgo is used when neither the user nor the config selects a
|
||||||
|
// resample algorithm. "average" performs RRDTool-style interval averaging,
|
||||||
|
// which keeps each plotted point a true mean of its interval.
|
||||||
|
const DefaultResampleAlgo = "average"
|
||||||
|
|
||||||
|
// ResampleAlgo returns the configured default resample algorithm, falling back
|
||||||
|
// to DefaultResampleAlgo. Note that an empty string would select LTTB in
|
||||||
|
// cc-lib's resampler, so callers must not pass "" when they mean "the default".
|
||||||
|
func ResampleAlgo() string {
|
||||||
|
if Keys.EnableResampling != nil && Keys.EnableResampling.DefaultAlgo != "" {
|
||||||
|
return Keys.EnableResampling.DefaultAlgo
|
||||||
|
}
|
||||||
|
return DefaultResampleAlgo
|
||||||
|
}
|
||||||
|
|
||||||
|
// TargetPointsForPolicy returns the target number of data points for a resample
|
||||||
|
// policy. This is the single source of truth: it feeds both the requested
|
||||||
|
// resolution (via metricdispatch.ComputeResolution) and the resampler's
|
||||||
|
// MinimumRequiredPoints threshold. Unknown or empty policies return 0.
|
||||||
|
func TargetPointsForPolicy(policy string) int {
|
||||||
|
switch policy {
|
||||||
|
case "low":
|
||||||
|
return 200
|
||||||
|
case "medium":
|
||||||
|
return 500
|
||||||
|
case "high":
|
||||||
|
return 1000
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
|
ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
|
||||||
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
|
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/resampler"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInit(t *testing.T) {
|
func TestInit(t *testing.T) {
|
||||||
@@ -39,3 +40,65 @@ func TestInitMinimal(t *testing.T) {
|
|||||||
t.Errorf("wrong addr\ngot: %s \nwant: 127.0.0.1:8080", Keys.Addr)
|
t.Errorf("wrong addr\ngot: %s \nwant: 127.0.0.1:8080", Keys.Addr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// config-large.json carried the removed resampling keys (minimum-points,
|
||||||
|
// trigger, resolutions), which DisallowUnknownFields rejects.
|
||||||
|
func TestInitLarge(t *testing.T) {
|
||||||
|
fp := "../../configs/config-large.json"
|
||||||
|
ccconf.Init(fp)
|
||||||
|
if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
|
||||||
|
Init(cfg)
|
||||||
|
} else {
|
||||||
|
cclog.Abort("Main configuration must be present")
|
||||||
|
}
|
||||||
|
|
||||||
|
if Keys.EnableResampling == nil {
|
||||||
|
t.Fatal("resampling config missing")
|
||||||
|
}
|
||||||
|
if Keys.EnableResampling.DefaultAlgo != "average" {
|
||||||
|
t.Errorf("wrong default algo\ngot: %s \nwant: average", Keys.EnableResampling.DefaultAlgo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTargetPointsForPolicy(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
policy string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"low", 200},
|
||||||
|
{"medium", 500},
|
||||||
|
{"high", 1000},
|
||||||
|
{"unknown", 0},
|
||||||
|
{"", 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := TargetPointsForPolicy(tt.policy); got != tt.want {
|
||||||
|
t.Errorf("TargetPointsForPolicy(%q) = %d, want %d", tt.policy, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The resampler must be allowed to act exactly when a series exceeds the target
|
||||||
|
// point count. A mismatch here silently drops resample requests for a band of
|
||||||
|
// job durations.
|
||||||
|
func TestInitSyncsResamplerThreshold(t *testing.T) {
|
||||||
|
for _, policy := range []string{"low", "medium", "high"} {
|
||||||
|
Keys.EnableResampling = &ResampleConfig{DefaultPolicy: policy}
|
||||||
|
initResampler()
|
||||||
|
|
||||||
|
want := TargetPointsForPolicy(policy)
|
||||||
|
if resampler.MinimumRequiredPoints != want {
|
||||||
|
t.Errorf("policy %q: MinimumRequiredPoints = %d, want %d",
|
||||||
|
policy, resampler.MinimumRequiredPoints, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty policy falls back to the documented default.
|
||||||
|
Keys.EnableResampling = &ResampleConfig{}
|
||||||
|
initResampler()
|
||||||
|
if want := TargetPointsForPolicy(DefaultResamplePolicy); resampler.MinimumRequiredPoints != want {
|
||||||
|
t.Errorf("empty policy: MinimumRequiredPoints = %d, want %d",
|
||||||
|
resampler.MinimumRequiredPoints, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -92,24 +92,18 @@ var configSchema = `
|
|||||||
"description": "Enable dynamic zoom in frontend metric plots.",
|
"description": "Enable dynamic zoom in frontend metric plots.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"minimum-points": {
|
"default-policy": {
|
||||||
"description": "Minimum points to trigger resampling of time-series data.",
|
"description": "Default resample policy when no user preference is set.",
|
||||||
"type": "integer"
|
"type": "string",
|
||||||
|
"enum": ["low", "medium", "high"]
|
||||||
},
|
},
|
||||||
"trigger": {
|
"default-algo": {
|
||||||
"description": "Trigger next zoom level at less than this many visible datapoints.",
|
"description": "Default resample algorithm when no user preference is set. Default and recommended: 'average' (RRDTool-style interval averaging).",
|
||||||
"type": "integer"
|
"type": "string",
|
||||||
},
|
"enum": ["lttb", "average", "simple"]
|
||||||
"resolutions": {
|
|
||||||
"description": "Array of resampling target resolutions, in seconds.",
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": ["trigger", "resolutions"]
|
|
||||||
},
|
|
||||||
"api-subjects": {
|
"api-subjects": {
|
||||||
"description": "NATS subjects configuration for subscribing to job and node events.",
|
"description": "NATS subjects configuration for subscribing to job and node events.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -24,6 +24,6 @@ func Validate(schema string, instance json.RawMessage) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err = sch.Validate(v); err != nil {
|
if err = sch.Validate(v); err != nil {
|
||||||
cclog.Fatalf("%#v", err)
|
cclog.Fatalf("config validation failed: %s", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Package configexample guards the shipped example configuration files in
|
||||||
|
// ./configs against drift: every section must still be accepted by the code
|
||||||
|
// that parses it at startup. The test lives in its own package because it has
|
||||||
|
// to import every config-consuming package at once.
|
||||||
|
package configexample
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/config"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/taskmanager"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/web"
|
||||||
|
ccschema "github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
const configDir = "../../configs"
|
||||||
|
|
||||||
|
// knownSections mirrors the list in cmd/cc-backend/main.go.
|
||||||
|
var knownSections = map[string]bool{
|
||||||
|
"main": true, "auth": true, "nats": true, "archive": true,
|
||||||
|
"metric-store": true, "metric-store-external": true, "cron": true, "ui": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveConfig covers the union of all archive backend options plus the
|
||||||
|
// retention/compression keys consumed by the task manager.
|
||||||
|
type archiveConfig struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
DBPath string `json:"db-path"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Bucket string `json:"bucket"`
|
||||||
|
Region string `json:"region"`
|
||||||
|
UsePathStyle bool `json:"use-path-style"`
|
||||||
|
AccessKey string `json:"access-key"`
|
||||||
|
SecretKey string `json:"secret-key"`
|
||||||
|
Retention taskmanager.Retention `json:"retention"`
|
||||||
|
Compression int `json:"compression"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeStrict(t *testing.T, name string, raw json.RawMessage, target any) {
|
||||||
|
t.Helper()
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(target); err != nil {
|
||||||
|
t.Errorf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExampleConfigs(t *testing.T) {
|
||||||
|
for _, name := range []string{"config.json", "config-demo.json", "config-large.json"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(configDir, name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var sections map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, §ions); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, value := range sections {
|
||||||
|
// ccConfig resolves a "<name>-file" key by loading the
|
||||||
|
// referenced file and storing it under "<name>".
|
||||||
|
section := key
|
||||||
|
if base, ok := strings.CutSuffix(key, "-file"); ok {
|
||||||
|
section = base
|
||||||
|
var path string
|
||||||
|
if err := json.Unmarshal(value, &path); err != nil {
|
||||||
|
t.Errorf("%s: value of %q is not a file path: %v", name, key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(filepath.Join(configDir, path))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: %q references %s: %v", name, key, path, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value = json.RawMessage(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !knownSections[section] {
|
||||||
|
t.Errorf("%s: unknown top-level section %q", name, key)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch section {
|
||||||
|
case "main":
|
||||||
|
var cfg config.ProgramConfig
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
case "cron":
|
||||||
|
var cfg taskmanager.CronFrequency
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
case "archive":
|
||||||
|
var cfg archiveConfig
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
case "metric-store":
|
||||||
|
var cfg metricstore.MetricStoreConfig
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
case "metric-store-external":
|
||||||
|
var cfg []metricdispatch.CCMetricStoreConfig
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
case "ui":
|
||||||
|
var cfg web.WebConfig
|
||||||
|
decodeStrict(t, key, value, &cfg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExampleClusterConfig(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(configDir, "cluster.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ccschema.Validate(ccschema.ClusterCfg, bytes.NewReader(raw)); err != nil {
|
||||||
|
t.Errorf("cluster.json does not validate: %v", err)
|
||||||
|
}
|
||||||
|
var cluster ccschema.Cluster
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&cluster); err != nil {
|
||||||
|
t.Errorf("cluster.json: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveTestdataClusterConfigs keeps the cluster.json fixtures valid
|
||||||
|
// against the cc-lib schema. They are copied into real deployments, and an
|
||||||
|
// invalid one only fails at runtime with main.validate enabled.
|
||||||
|
func TestArchiveTestdataClusterConfigs(t *testing.T) {
|
||||||
|
for _, path := range []string{
|
||||||
|
"../../pkg/archive/testdata/archive/alex/cluster.json",
|
||||||
|
"../../pkg/archive/testdata/archive/fritz/cluster.json",
|
||||||
|
"../../pkg/archive/testdata/archive/emmy/cluster.json",
|
||||||
|
"../../internal/importer/testdata/cluster-fritz.json",
|
||||||
|
} {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ccschema.Validate(ccschema.ClusterCfg, bytes.NewReader(raw)); err != nil {
|
||||||
|
t.Errorf("%s does not validate: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -328,6 +328,63 @@ func (e Aggregate) MarshalJSON() ([]byte, error) {
|
|||||||
return buf.Bytes(), nil
|
return buf.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResampleAlgo string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ResampleAlgoLttb ResampleAlgo = "LTTB"
|
||||||
|
ResampleAlgoAverage ResampleAlgo = "AVERAGE"
|
||||||
|
ResampleAlgoSimple ResampleAlgo = "SIMPLE"
|
||||||
|
)
|
||||||
|
|
||||||
|
var AllResampleAlgo = []ResampleAlgo{
|
||||||
|
ResampleAlgoLttb,
|
||||||
|
ResampleAlgoAverage,
|
||||||
|
ResampleAlgoSimple,
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e ResampleAlgo) IsValid() bool {
|
||||||
|
switch e {
|
||||||
|
case ResampleAlgoLttb, ResampleAlgoAverage, ResampleAlgoSimple:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e ResampleAlgo) String() string {
|
||||||
|
return string(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ResampleAlgo) UnmarshalGQL(v any) error {
|
||||||
|
str, ok := v.(string)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("enums must be strings")
|
||||||
|
}
|
||||||
|
|
||||||
|
*e = ResampleAlgo(str)
|
||||||
|
if !e.IsValid() {
|
||||||
|
return fmt.Errorf("%s is not a valid ResampleAlgo", str)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e ResampleAlgo) MarshalGQL(w io.Writer) {
|
||||||
|
fmt.Fprint(w, strconv.Quote(e.String()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ResampleAlgo) UnmarshalJSON(b []byte) error {
|
||||||
|
s, err := strconv.Unquote(string(b))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return e.UnmarshalGQL(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e ResampleAlgo) MarshalJSON() ([]byte, error) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
e.MarshalGQL(&buf)
|
||||||
|
return buf.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
type SortByAggregate string
|
type SortByAggregate string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
package graph
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/config"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/repository"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/pkg/archive"
|
||||||
|
)
|
||||||
|
|
||||||
|
// resolveResolutionFromPolicy reads the user's resample policy preference and
|
||||||
|
// computes a resolution based on job duration and metric frequency. Returns nil
|
||||||
|
// if the user has no policy set.
|
||||||
|
func resolveResolutionFromPolicy(ctx context.Context, duration int64, cluster string, metrics []string) *int {
|
||||||
|
user := repository.GetUserFromContext(ctx)
|
||||||
|
if user == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
conf, err := repository.GetUserCfgRepo().GetUIConfig(user)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
policyVal, ok := conf["plotConfiguration_resamplePolicy"]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
policyStr, ok := policyVal.(string)
|
||||||
|
if !ok || policyStr == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := metricdispatch.ResamplePolicy(policyStr)
|
||||||
|
targetPoints := metricdispatch.TargetPointsForPolicy(policy)
|
||||||
|
if targetPoints == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the smallest metric frequency across the requested metrics
|
||||||
|
frequency := smallestFrequency(cluster, metrics)
|
||||||
|
if frequency <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
res := metricdispatch.ComputeResolution(duration, int64(frequency), targetPoints)
|
||||||
|
return &res
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveResampleAlgo returns the resampling algorithm name to use, checking
|
||||||
|
// the explicit GraphQL parameter first, then the user's preference.
|
||||||
|
func resolveResampleAlgo(ctx context.Context, resampleAlgo *model.ResampleAlgo) string {
|
||||||
|
if resampleAlgo != nil {
|
||||||
|
return strings.ToLower(resampleAlgo.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
user := repository.GetUserFromContext(ctx)
|
||||||
|
if user == nil {
|
||||||
|
return config.ResampleAlgo()
|
||||||
|
}
|
||||||
|
|
||||||
|
conf, err := repository.GetUserCfgRepo().GetUIConfig(user)
|
||||||
|
if err != nil {
|
||||||
|
return config.ResampleAlgo()
|
||||||
|
}
|
||||||
|
|
||||||
|
algoVal, ok := conf["plotConfiguration_resampleAlgo"]
|
||||||
|
if ok {
|
||||||
|
if algoStr, ok := algoVal.(string); ok && algoStr != "" {
|
||||||
|
return algoStr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to the global default algo
|
||||||
|
return config.ResampleAlgo()
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveResolutionFromDefaultPolicy computes a resolution using the global
|
||||||
|
// default policy from config. Returns nil if no policy is configured.
|
||||||
|
func resolveResolutionFromDefaultPolicy(duration int64, cluster string, metrics []string) *int {
|
||||||
|
cfg := config.Keys.EnableResampling
|
||||||
|
if cfg == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
policyStr := cfg.DefaultPolicy
|
||||||
|
if policyStr == "" {
|
||||||
|
policyStr = "medium"
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := metricdispatch.ResamplePolicy(policyStr)
|
||||||
|
targetPoints := metricdispatch.TargetPointsForPolicy(policy)
|
||||||
|
if targetPoints == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
frequency := smallestFrequency(cluster, metrics)
|
||||||
|
if frequency <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
res := metricdispatch.ComputeResolution(duration, int64(frequency), targetPoints)
|
||||||
|
return &res
|
||||||
|
}
|
||||||
|
|
||||||
|
// smallestFrequency returns the smallest metric timestep (in seconds) among the
|
||||||
|
// requested metrics for the given cluster. Falls back to 0 if nothing is found.
|
||||||
|
func smallestFrequency(cluster string, metrics []string) int {
|
||||||
|
cl := archive.GetCluster(cluster)
|
||||||
|
if cl == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
minFreq := 0
|
||||||
|
for _, mc := range cl.MetricConfig {
|
||||||
|
if len(metrics) > 0 {
|
||||||
|
found := false
|
||||||
|
for _, m := range metrics {
|
||||||
|
if mc.Name == m {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if minFreq == 0 || mc.Timestep < minFreq {
|
||||||
|
minFreq = mc.Timestep
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return minFreq
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ package graph
|
|||||||
// This file will be automatically regenerated based on the schema, any resolver
|
// This file will be automatically regenerated based on the schema, any resolver
|
||||||
// implementations
|
// implementations
|
||||||
// will be copied through when generating and any unknown code will be moved to the end.
|
// will be copied through when generating and any unknown code will be moved to the end.
|
||||||
// Code generated by github.com/99designs/gqlgen version v0.17.90
|
// Code generated by github.com/99designs/gqlgen version v0.17.94
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -118,7 +118,7 @@ func (r *jobResolver) EnergyFootprint(ctx context.Context, obj *schema.Job) ([]*
|
|||||||
}
|
}
|
||||||
|
|
||||||
// MetaData is the resolver for the metaData field.
|
// MetaData is the resolver for the metaData field.
|
||||||
func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (any, error) {
|
func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) {
|
||||||
return r.Repo.FetchMetadata(obj)
|
return r.Repo.FetchMetadata(obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +332,7 @@ func (r *nodeResolver) HealthState(ctx context.Context, obj *schema.Node) (strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
// MetaData is the resolver for the metaData field.
|
// MetaData is the resolver for the metaData field.
|
||||||
func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (any, error) {
|
func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (interface{}, error) {
|
||||||
if obj.MetaData != nil {
|
if obj.MetaData != nil {
|
||||||
return obj.MetaData, nil
|
return obj.MetaData, nil
|
||||||
} else {
|
} else {
|
||||||
@@ -343,7 +343,7 @@ func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (any, err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// HealthData is the resolver for the healthData field.
|
// HealthData is the resolver for the healthData field.
|
||||||
func (r *nodeResolver) HealthData(ctx context.Context, obj *schema.Node) (any, error) {
|
func (r *nodeResolver) HealthData(ctx context.Context, obj *schema.Node) (interface{}, error) {
|
||||||
if obj.HealthData != nil {
|
if obj.HealthData != nil {
|
||||||
return obj.HealthData, nil
|
return obj.HealthData, nil
|
||||||
} else {
|
} else {
|
||||||
@@ -498,31 +498,37 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// JobMetrics is the resolver for the jobMetrics field.
|
// JobMetrics is the resolver for the jobMetrics field.
|
||||||
func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) {
|
func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int, resampleAlgo *model.ResampleAlgo) ([]*model.JobMetricWithName, error) {
|
||||||
if resolution == nil { // Load from Config
|
|
||||||
if config.Keys.EnableResampling != nil {
|
|
||||||
defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
|
|
||||||
resolution = &defaultRes
|
|
||||||
} else { // Set 0 (Loads configured metric timestep)
|
|
||||||
defaultRes := 0
|
|
||||||
resolution = &defaultRes
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
job, err := r.Query().Job(ctx, id)
|
job, err := r.Query().Job(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warn("Error while querying job for metrics")
|
cclog.Warn("Error while querying job for metrics")
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution)
|
// Resolve resolution: explicit param > user policy > global config > 0
|
||||||
|
if resolution == nil {
|
||||||
|
resolution = resolveResolutionFromPolicy(ctx, int64(job.Duration), job.Cluster, metrics)
|
||||||
|
}
|
||||||
|
if resolution == nil {
|
||||||
|
if config.Keys.EnableResampling != nil {
|
||||||
|
resolution = resolveResolutionFromDefaultPolicy(int64(job.Duration), job.Cluster, metrics)
|
||||||
|
}
|
||||||
|
if resolution == nil {
|
||||||
|
defaultRes := 0
|
||||||
|
resolution = &defaultRes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
algoName := resolveResampleAlgo(ctx, resampleAlgo)
|
||||||
|
|
||||||
|
data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution, algoName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warn("Error while loading job data")
|
cclog.Warn("Error while loading job data")
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
res := []*model.JobMetricWithName{}
|
res := []*model.JobMetricWithName{}
|
||||||
for name, md := range data {
|
for name, md := range data.Metrics {
|
||||||
for scope, metric := range md {
|
for scope, metric := range md {
|
||||||
res = append(res, &model.JobMetricWithName{
|
res = append(res, &model.JobMetricWithName{
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -575,7 +581,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
|
|||||||
}
|
}
|
||||||
|
|
||||||
res := make([]*model.NamedStatsWithScope, 0)
|
res := make([]*model.NamedStatsWithScope, 0)
|
||||||
for name, scoped := range data {
|
for name, scoped := range data.Metrics {
|
||||||
for scope, stats := range scoped {
|
for scope, stats := range scoped {
|
||||||
|
|
||||||
mdlStats := make([]*model.ScopedStats, 0)
|
mdlStats := make([]*model.ScopedStats, 0)
|
||||||
@@ -877,12 +883,17 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NodeMetricsList is the resolver for the nodeMetricsList field.
|
// NodeMetricsList is the resolver for the nodeMetricsList field.
|
||||||
func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
|
func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int, resampleAlgo *model.ResampleAlgo) (*model.NodesResultList, error) {
|
||||||
if resolution == nil { // Load from Config
|
// Resolve resolution: explicit param > user policy > global config > 0
|
||||||
|
duration := int64(to.Sub(from).Seconds())
|
||||||
|
if resolution == nil {
|
||||||
|
resolution = resolveResolutionFromPolicy(ctx, duration, cluster, metrics)
|
||||||
|
}
|
||||||
|
if resolution == nil {
|
||||||
if config.Keys.EnableResampling != nil {
|
if config.Keys.EnableResampling != nil {
|
||||||
defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
|
resolution = resolveResolutionFromDefaultPolicy(duration, cluster, metrics)
|
||||||
resolution = &defaultRes
|
}
|
||||||
} else { // Set 0 (Loads configured metric timestep)
|
if resolution == nil {
|
||||||
defaultRes := 0
|
defaultRes := 0
|
||||||
resolution = &defaultRes
|
resolution = &defaultRes
|
||||||
}
|
}
|
||||||
@@ -906,8 +917,10 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
algoName := resolveResampleAlgo(ctx, resampleAlgo)
|
||||||
|
|
||||||
// data -> map hostname:jobdata
|
// data -> map hostname:jobdata
|
||||||
data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx)
|
data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx, algoName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
|
cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -926,7 +939,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
|
|||||||
cclog.Warnf("error in nodeMetrics resolver: %s", err)
|
cclog.Warnf("error in nodeMetrics resolver: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for metric, scopedMetrics := range data[hostname] {
|
for metric, scopedMetrics := range data[hostname].Metrics {
|
||||||
for scope, scopedMetric := range scopedMetrics {
|
for scope, scopedMetric := range scopedMetrics {
|
||||||
host.Metrics = append(host.Metrics, &model.JobMetricWithName{
|
host.Metrics = append(host.Metrics, &model.JobMetricWithName{
|
||||||
Name: metric,
|
Name: metric,
|
||||||
@@ -1072,10 +1085,12 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} }
|
|||||||
// SubCluster returns generated.SubClusterResolver implementation.
|
// SubCluster returns generated.SubClusterResolver implementation.
|
||||||
func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} }
|
func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} }
|
||||||
|
|
||||||
type clusterResolver struct{ *Resolver }
|
type (
|
||||||
type jobResolver struct{ *Resolver }
|
clusterResolver struct{ *Resolver }
|
||||||
type metricValueResolver struct{ *Resolver }
|
jobResolver struct{ *Resolver }
|
||||||
type mutationResolver struct{ *Resolver }
|
metricValueResolver struct{ *Resolver }
|
||||||
type nodeResolver struct{ *Resolver }
|
mutationResolver struct{ *Resolver }
|
||||||
type queryResolver struct{ *Resolver }
|
nodeResolver struct{ *Resolver }
|
||||||
type subClusterResolver struct{ *Resolver }
|
queryResolver struct{ *Resolver }
|
||||||
|
subClusterResolver struct{ *Resolver }
|
||||||
|
)
|
||||||
|
|||||||
@@ -55,13 +55,13 @@ func (r *queryResolver) rooflineHeatmap(
|
|||||||
// resolution = max(resolution, mc.Timestep)
|
// resolution = max(resolution, mc.Timestep)
|
||||||
// }
|
// }
|
||||||
|
|
||||||
jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
|
jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0, "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warnf("Error while loading roofline metrics for job %d", *job.ID)
|
cclog.Warnf("Error while loading roofline metrics for job %d", *job.ID)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
|
flops_, membw_ := jobdata.Metrics["flops_any"], jobdata.Metrics["mem_bw"]
|
||||||
if flops_ == nil && membw_ == nil {
|
if flops_ == nil && membw_ == nil {
|
||||||
cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
|
cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -290,7 +290,7 @@ func SanityChecks(job *schema.Job) error {
|
|||||||
//
|
//
|
||||||
// TODO: Either implement the metric normalization or remove this dead code.
|
// TODO: Either implement the metric normalization or remove this dead code.
|
||||||
func checkJobData(d *schema.JobData) error {
|
func checkJobData(d *schema.JobData) error {
|
||||||
for _, scopes := range *d {
|
for _, scopes := range d.Metrics {
|
||||||
// var newUnit schema.Unit
|
// var newUnit schema.Unit
|
||||||
// TODO Add node scope if missing
|
// TODO Add node scope if missing
|
||||||
for _, metric := range scopes {
|
for _, metric := range scopes {
|
||||||
|
|||||||
@@ -23,13 +23,14 @@
|
|||||||
// - Running jobs: 2 minutes (data changes frequently)
|
// - Running jobs: 2 minutes (data changes frequently)
|
||||||
// - Completed jobs: 5 hours (data is static)
|
// - Completed jobs: 5 hours (data is static)
|
||||||
//
|
//
|
||||||
// The cache key is based on job ID, state, requested metrics, scopes, and resolution.
|
// The cache key is based on job ID, state, requested metrics, scopes, resolution,
|
||||||
|
// and resample algorithm.
|
||||||
//
|
//
|
||||||
// # Usage
|
// # Usage
|
||||||
//
|
//
|
||||||
// The primary entry point is LoadData, which automatically handles both running and archived jobs:
|
// The primary entry point is LoadData, which automatically handles both running and archived jobs:
|
||||||
//
|
//
|
||||||
// jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
|
// jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution, resampleAlgo)
|
||||||
// if err != nil {
|
// if err != nil {
|
||||||
// // Handle error
|
// // Handle error
|
||||||
// }
|
// }
|
||||||
@@ -62,9 +63,10 @@ func cacheKey(
|
|||||||
metrics []string,
|
metrics []string,
|
||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
resolution int,
|
resolution int,
|
||||||
|
resampleAlgo string,
|
||||||
) string {
|
) string {
|
||||||
return fmt.Sprintf("%d(%s):[%v],[%v]-%d",
|
return fmt.Sprintf("%d(%s):[%v],[%v]-%d-%s",
|
||||||
*job.ID, job.State, metrics, scopes, resolution)
|
*job.ID, job.State, metrics, scopes, resolution, resampleAlgo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
|
// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
|
||||||
@@ -87,8 +89,9 @@ func LoadData(job *schema.Job,
|
|||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
resolution int,
|
resolution int,
|
||||||
|
resampleAlgo string,
|
||||||
) (schema.JobData, error) {
|
) (schema.JobData, error) {
|
||||||
data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {
|
data := cache.Get(cacheKey(job, metrics, scopes, resolution, resampleAlgo), func() (_ any, ttl time.Duration, size int) {
|
||||||
var jd schema.JobData
|
var jd schema.JobData
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
@@ -113,9 +116,9 @@ func LoadData(job *schema.Job,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
|
jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution, resampleAlgo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if len(jd) != 0 {
|
if len(jd.Metrics) != 0 {
|
||||||
cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
|
cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
|
||||||
job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
|
job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
|
||||||
} else {
|
} else {
|
||||||
@@ -136,13 +139,17 @@ func LoadData(job *schema.Job,
|
|||||||
|
|
||||||
jd = deepCopy(jdTemp)
|
jd = deepCopy(jdTemp)
|
||||||
|
|
||||||
// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
|
// Resample archived data to reduce data points to the requested resolution,
|
||||||
// to the requested resolution, improving transfer performance and client-side rendering.
|
// improving transfer performance and client-side rendering.
|
||||||
for _, v := range jd {
|
resampleFn, rfErr := resampler.GetResampler(resampleAlgo)
|
||||||
|
if rfErr != nil {
|
||||||
|
return rfErr, 0, 0
|
||||||
|
}
|
||||||
|
for _, v := range jd.Metrics {
|
||||||
for _, v_ := range v {
|
for _, v_ := range v {
|
||||||
timestep := int64(0)
|
timestep := int64(0)
|
||||||
for i := 0; i < len(v_.Series); i += 1 {
|
for i := 0; i < len(v_.Series); i += 1 {
|
||||||
v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
|
v_.Series[i].Data, timestep, err = resampleFn(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err, 0, 0
|
return err, 0, 0
|
||||||
}
|
}
|
||||||
@@ -154,17 +161,20 @@ func LoadData(job *schema.Job,
|
|||||||
// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
|
// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
|
||||||
if metrics != nil || scopes != nil {
|
if metrics != nil || scopes != nil {
|
||||||
if metrics == nil {
|
if metrics == nil {
|
||||||
metrics = make([]string, 0, len(jd))
|
metrics = make([]string, 0, len(jd.Metrics))
|
||||||
for k := range jd {
|
for k := range jd.Metrics {
|
||||||
metrics = append(metrics, k)
|
metrics = append(metrics, k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res := schema.JobData{}
|
res := schema.JobData{
|
||||||
|
Metrics: make(map[string]schema.ScopedMetrics, len(metrics)),
|
||||||
|
Groups: jd.Groups,
|
||||||
|
}
|
||||||
for _, metric := range metrics {
|
for _, metric := range metrics {
|
||||||
if perscope, ok := jd[metric]; ok {
|
if perscope, ok := jd.Metrics[metric]; ok {
|
||||||
if len(perscope) > 1 {
|
if len(perscope) > 1 {
|
||||||
subset := make(map[schema.MetricScope]*schema.JobMetric)
|
subset := make(schema.ScopedMetrics)
|
||||||
for _, scope := range scopes {
|
for _, scope := range scopes {
|
||||||
if jm, ok := perscope[scope]; ok {
|
if jm, ok := perscope[scope]; ok {
|
||||||
subset[scope] = jm
|
subset[scope] = jm
|
||||||
@@ -176,7 +186,7 @@ func LoadData(job *schema.Job,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res[metric] = perscope
|
res.Metrics[metric] = perscope
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
jd = res
|
jd = res
|
||||||
@@ -193,7 +203,7 @@ func LoadData(job *schema.Job,
|
|||||||
// instead of overwhelming the UI with individual node lines. Note that newly calculated
|
// instead of overwhelming the UI with individual node lines. Note that newly calculated
|
||||||
// statistics use min/median/max, while archived statistics may use min/mean/max.
|
// statistics use min/median/max, while archived statistics may use min/mean/max.
|
||||||
const maxSeriesSize int = 8
|
const maxSeriesSize int = 8
|
||||||
for _, scopes := range jd {
|
for _, scopes := range jd.Metrics {
|
||||||
for _, jm := range scopes {
|
for _, jm := range scopes {
|
||||||
if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
|
if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
|
||||||
continue
|
continue
|
||||||
@@ -223,7 +233,7 @@ func LoadData(job *schema.Job,
|
|||||||
|
|
||||||
if err, ok := data.(error); ok {
|
if err, ok := data.(error); ok {
|
||||||
cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
|
cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.(schema.JobData), nil
|
return data.(schema.JobData), nil
|
||||||
@@ -290,14 +300,14 @@ func LoadScopedJobStats(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
|
cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
|
||||||
job.Cluster, job.SubCluster, err.Error())
|
job.Cluster, job.SubCluster, err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
|
scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
|
cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
|
||||||
job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
|
job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Round Resulting Stat Values
|
// Round Resulting Stat Values
|
||||||
@@ -414,6 +424,7 @@ func LoadNodeListData(
|
|||||||
resolution int,
|
resolution int,
|
||||||
from, to time.Time,
|
from, to time.Time,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
resampleAlgo string,
|
||||||
) (map[string]schema.JobData, error) {
|
) (map[string]schema.JobData, error) {
|
||||||
if metrics == nil {
|
if metrics == nil {
|
||||||
for _, m := range archive.GetCluster(cluster).MetricConfig {
|
for _, m := range archive.GetCluster(cluster).MetricConfig {
|
||||||
@@ -428,7 +439,7 @@ func LoadNodeListData(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)
|
data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx, resampleAlgo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if len(data) != 0 {
|
if len(data) != 0 {
|
||||||
cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
|
cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
|
||||||
@@ -444,7 +455,7 @@ func LoadNodeListData(
|
|||||||
// Statistics are calculated as min/median/max.
|
// Statistics are calculated as min/median/max.
|
||||||
const maxSeriesSize int = 8
|
const maxSeriesSize int = 8
|
||||||
for _, jd := range data {
|
for _, jd := range data {
|
||||||
for _, scopes := range jd {
|
for _, scopes := range jd.Metrics {
|
||||||
for _, jm := range scopes {
|
for _, jm := range scopes {
|
||||||
if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
|
if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
|
||||||
continue
|
continue
|
||||||
@@ -465,14 +476,32 @@ func LoadNodeListData(
|
|||||||
// archived data (e.g., during resampling). This ensures the cached archive data remains
|
// archived data (e.g., during resampling). This ensures the cached archive data remains
|
||||||
// immutable while allowing per-request transformations.
|
// immutable while allowing per-request transformations.
|
||||||
func deepCopy(source schema.JobData) schema.JobData {
|
func deepCopy(source schema.JobData) schema.JobData {
|
||||||
result := make(schema.JobData, len(source))
|
result := schema.JobData{Metrics: copyScopedMetrics(source.Metrics)}
|
||||||
|
|
||||||
|
for _, group := range source.Groups {
|
||||||
|
copied := schema.MetricGroup{Key: group.Key}
|
||||||
|
for _, inst := range group.Instances {
|
||||||
|
copied.Instances = append(copied.Instances, schema.MetricGroupInstance{
|
||||||
|
Name: inst.Name,
|
||||||
|
Type: inst.Type,
|
||||||
|
Metrics: copyScopedMetrics(inst.Metrics),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
result.Groups = append(result.Groups, copied)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyScopedMetrics(source map[string]schema.ScopedMetrics) map[string]schema.ScopedMetrics {
|
||||||
|
result := make(map[string]schema.ScopedMetrics, len(source))
|
||||||
|
|
||||||
for metricName, scopeMap := range source {
|
for metricName, scopeMap := range source {
|
||||||
result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))
|
scopes := make(schema.ScopedMetrics, len(scopeMap))
|
||||||
|
|
||||||
for scope, jobMetric := range scopeMap {
|
for scope, jobMetric := range scopeMap {
|
||||||
result[metricName][scope] = copyJobMetric(jobMetric)
|
scopes[scope] = copyJobMetric(jobMetric)
|
||||||
}
|
}
|
||||||
|
result[metricName] = scopes
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
|
|
||||||
func TestDeepCopy(t *testing.T) {
|
func TestDeepCopy(t *testing.T) {
|
||||||
nodeId := "0"
|
nodeId := "0"
|
||||||
original := schema.JobData{
|
original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"cpu_load": {
|
"cpu_load": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -42,42 +42,42 @@ func TestDeepCopy(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
copied := deepCopy(original)
|
copied := deepCopy(original)
|
||||||
|
|
||||||
original["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] = 999.0
|
original.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] = 999.0
|
||||||
original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] = 888.0
|
original.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] = 888.0
|
||||||
original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] = 777.0
|
original.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] = 777.0
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] != 1.0 {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] != 1.0 {
|
||||||
t.Errorf("Series data was not deeply copied: got %v, want 1.0",
|
t.Errorf("Series data was not deeply copied: got %v, want 1.0",
|
||||||
copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0])
|
copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0])
|
||||||
}
|
}
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] != 1.0 {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] != 1.0 {
|
||||||
t.Errorf("StatisticsSeries was not deeply copied: got %v, want 1.0",
|
t.Errorf("StatisticsSeries was not deeply copied: got %v, want 1.0",
|
||||||
copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0])
|
copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0])
|
||||||
}
|
}
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] != 1.5 {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] != 1.5 {
|
||||||
t.Errorf("Percentiles was not deeply copied: got %v, want 1.5",
|
t.Errorf("Percentiles was not deeply copied: got %v, want 1.5",
|
||||||
copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0])
|
copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0])
|
||||||
}
|
}
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].Timestep != 60 {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].Timestep != 60 {
|
||||||
t.Errorf("Timestep not copied correctly: got %v, want 60",
|
t.Errorf("Timestep not copied correctly: got %v, want 60",
|
||||||
copied["cpu_load"][schema.MetricScopeNode].Timestep)
|
copied.Metrics["cpu_load"][schema.MetricScopeNode].Timestep)
|
||||||
}
|
}
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname != "node001" {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Hostname != "node001" {
|
||||||
t.Errorf("Hostname not copied correctly: got %v, want node001",
|
t.Errorf("Hostname not copied correctly: got %v, want node001",
|
||||||
copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname)
|
copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Hostname)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeepCopyNilStatisticsSeries(t *testing.T) {
|
func TestDeepCopyNilStatisticsSeries(t *testing.T) {
|
||||||
original := schema.JobData{
|
original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"mem_used": {
|
"mem_used": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -90,18 +90,18 @@ func TestDeepCopyNilStatisticsSeries(t *testing.T) {
|
|||||||
StatisticsSeries: nil,
|
StatisticsSeries: nil,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
copied := deepCopy(original)
|
copied := deepCopy(original)
|
||||||
|
|
||||||
if copied["mem_used"][schema.MetricScopeNode].StatisticsSeries != nil {
|
if copied.Metrics["mem_used"][schema.MetricScopeNode].StatisticsSeries != nil {
|
||||||
t.Errorf("StatisticsSeries should be nil, got %v",
|
t.Errorf("StatisticsSeries should be nil, got %v",
|
||||||
copied["mem_used"][schema.MetricScopeNode].StatisticsSeries)
|
copied.Metrics["mem_used"][schema.MetricScopeNode].StatisticsSeries)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeepCopyEmptyPercentiles(t *testing.T) {
|
func TestDeepCopyEmptyPercentiles(t *testing.T) {
|
||||||
original := schema.JobData{
|
original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"cpu_load": {
|
"cpu_load": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -115,11 +115,11 @@ func TestDeepCopyEmptyPercentiles(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
copied := deepCopy(original)
|
copied := deepCopy(original)
|
||||||
|
|
||||||
if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles != nil {
|
if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles != nil {
|
||||||
t.Errorf("Percentiles should be nil when source is nil/empty")
|
t.Errorf("Percentiles should be nil when source is nil/empty")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ type MetricDataRepository interface {
|
|||||||
metrics []string,
|
metrics []string,
|
||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
resolution int) (schema.JobData, error)
|
resolution int,
|
||||||
|
resampleAlgo string) (schema.JobData, error)
|
||||||
|
|
||||||
// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
|
// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
|
||||||
LoadStats(job *schema.Job,
|
LoadStats(job *schema.Job,
|
||||||
@@ -51,7 +52,8 @@ type MetricDataRepository interface {
|
|||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
resolution int,
|
resolution int,
|
||||||
from, to time.Time,
|
from, to time.Time,
|
||||||
ctx context.Context) (map[string]schema.JobData, error)
|
ctx context.Context,
|
||||||
|
resampleAlgo string) (map[string]schema.JobData, error)
|
||||||
|
|
||||||
// HealthCheck evaluates the monitoring state for a set of nodes against expected metrics.
|
// HealthCheck evaluates the monitoring state for a set of nodes against expected metrics.
|
||||||
HealthCheck(cluster string,
|
HealthCheck(cluster string,
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
package metricdispatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ResamplePolicy string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ResamplePolicyLow ResamplePolicy = "low"
|
||||||
|
ResamplePolicyMedium ResamplePolicy = "medium"
|
||||||
|
ResamplePolicyHigh ResamplePolicy = "high"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TargetPointsForPolicy returns the target number of data points for a given
|
||||||
|
// policy. The table lives in the config package so that the requested
|
||||||
|
// resolution and the resampler's MinimumRequiredPoints threshold can never
|
||||||
|
// diverge.
|
||||||
|
func TargetPointsForPolicy(policy ResamplePolicy) int {
|
||||||
|
return config.TargetPointsForPolicy(string(policy))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComputeResolution computes the resampling resolution in seconds for a given
|
||||||
|
// job duration, metric frequency, and target point count. Returns 0 if the
|
||||||
|
// total number of data points is already at or below targetPoints (no resampling needed).
|
||||||
|
func ComputeResolution(duration int64, frequency int64, targetPoints int) int {
|
||||||
|
if frequency <= 0 || targetPoints <= 0 || duration <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
totalPoints := duration / frequency
|
||||||
|
if totalPoints <= int64(targetPoints) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
targetRes := math.Ceil(float64(duration) / float64(targetPoints))
|
||||||
|
// Round up to nearest multiple of frequency
|
||||||
|
resolution := int(math.Ceil(targetRes/float64(frequency))) * int(frequency)
|
||||||
|
|
||||||
|
return resolution
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
package metricdispatch
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestTargetPointsForPolicy(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
policy ResamplePolicy
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{ResamplePolicyLow, 200},
|
||||||
|
{ResamplePolicyMedium, 500},
|
||||||
|
{ResamplePolicyHigh, 1000},
|
||||||
|
{ResamplePolicy("unknown"), 0},
|
||||||
|
{ResamplePolicy(""), 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := TargetPointsForPolicy(tt.policy); got != tt.want {
|
||||||
|
t.Errorf("TargetPointsForPolicy(%q) = %d, want %d", tt.policy, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComputeResolution(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
duration int64
|
||||||
|
frequency int64
|
||||||
|
targetPoints int
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
// 24h job, 60s frequency, 1440 total points
|
||||||
|
{"low_24h_60s", 86400, 60, 200, 480},
|
||||||
|
{"medium_24h_60s", 86400, 60, 500, 180},
|
||||||
|
{"high_24h_60s", 86400, 60, 1000, 120},
|
||||||
|
|
||||||
|
// 2h job, 60s frequency, 120 total points — no resampling needed
|
||||||
|
{"low_2h_60s", 7200, 60, 200, 0},
|
||||||
|
{"medium_2h_60s", 7200, 60, 500, 0},
|
||||||
|
{"high_2h_60s", 7200, 60, 1000, 0},
|
||||||
|
|
||||||
|
// Edge: zero/negative inputs
|
||||||
|
{"zero_duration", 0, 60, 200, 0},
|
||||||
|
{"zero_frequency", 86400, 0, 200, 0},
|
||||||
|
{"zero_target", 86400, 60, 0, 0},
|
||||||
|
{"negative_duration", -100, 60, 200, 0},
|
||||||
|
|
||||||
|
// 12h job, 30s frequency, 1440 total points
|
||||||
|
{"medium_12h_30s", 43200, 30, 500, 90},
|
||||||
|
|
||||||
|
// Exact fit: total points == target points
|
||||||
|
{"exact_fit", 12000, 60, 200, 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := ComputeResolution(tt.duration, tt.frequency, tt.targetPoints)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("ComputeResolution(%d, %d, %d) = %d, want %d",
|
||||||
|
tt.duration, tt.frequency, tt.targetPoints, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -88,6 +88,7 @@ type APIQueryRequest struct {
|
|||||||
To int64 `json:"to"` // End time (Unix timestamp)
|
To int64 `json:"to"` // End time (Unix timestamp)
|
||||||
WithStats bool `json:"with-stats"` // Include min/avg/max statistics
|
WithStats bool `json:"with-stats"` // Include min/avg/max statistics
|
||||||
WithData bool `json:"with-data"` // Include time series data points
|
WithData bool `json:"with-data"` // Include time series data points
|
||||||
|
ResampleAlgo string `json:"resample-algo,omitempty"` // Downsampling algorithm ("lttb", "average", "simple"); empty = server default
|
||||||
}
|
}
|
||||||
|
|
||||||
// APIQuery specifies a single metric query with optional scope filtering.
|
// APIQuery specifies a single metric query with optional scope filtering.
|
||||||
@@ -231,11 +232,12 @@ func (ccms *CCMetricStore) LoadData(
|
|||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
resolution int,
|
resolution int,
|
||||||
|
resampleAlgo string,
|
||||||
) (schema.JobData, error) {
|
) (schema.JobData, error) {
|
||||||
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
|
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify assignment is correct - log any inconsistencies for debugging
|
// Verify assignment is correct - log any inconsistencies for debugging
|
||||||
@@ -251,16 +253,17 @@ func (ccms *CCMetricStore) LoadData(
|
|||||||
Queries: queries,
|
Queries: queries,
|
||||||
WithStats: true,
|
WithStats: true,
|
||||||
WithData: true,
|
WithData: true,
|
||||||
|
ResampleAlgo: resampleAlgo,
|
||||||
}
|
}
|
||||||
|
|
||||||
resBody, err := ccms.doRequest(ctx, &req)
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
|
cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var errors []string
|
var errors []string
|
||||||
jobData := make(schema.JobData)
|
jobData := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
|
||||||
|
|
||||||
// Add safety check for potential index out of range errors
|
// Add safety check for potential index out of range errors
|
||||||
if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
|
if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
|
||||||
@@ -285,8 +288,8 @@ func (ccms *CCMetricStore) LoadData(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := jobData[metric]; !ok {
|
if _, ok := jobData.Metrics[metric]; !ok {
|
||||||
jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
|
jobData.Metrics[metric] = make(schema.ScopedMetrics)
|
||||||
}
|
}
|
||||||
|
|
||||||
res := mc.Timestep
|
res := mc.Timestep
|
||||||
@@ -294,14 +297,14 @@ func (ccms *CCMetricStore) LoadData(
|
|||||||
res = row[0].Resolution
|
res = row[0].Resolution
|
||||||
}
|
}
|
||||||
|
|
||||||
jobMetric, ok := jobData[metric][scope]
|
jobMetric, ok := jobData.Metrics[metric][scope]
|
||||||
if !ok {
|
if !ok {
|
||||||
jobMetric = &schema.JobMetric{
|
jobMetric = &schema.JobMetric{
|
||||||
Unit: mc.Unit,
|
Unit: mc.Unit,
|
||||||
Timestep: res,
|
Timestep: res,
|
||||||
Series: make([]schema.Series, 0),
|
Series: make([]schema.Series, 0),
|
||||||
}
|
}
|
||||||
jobData[metric][scope] = jobMetric
|
jobData.Metrics[metric][scope] = jobMetric
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
@@ -329,9 +332,9 @@ func (ccms *CCMetricStore) LoadData(
|
|||||||
|
|
||||||
// So that one can later check len(jobData):
|
// So that one can later check len(jobData):
|
||||||
if len(jobMetric.Series) == 0 {
|
if len(jobMetric.Series) == 0 {
|
||||||
delete(jobData[metric], scope)
|
delete(jobData.Metrics[metric], scope)
|
||||||
if len(jobData[metric]) == 0 {
|
if len(jobData.Metrics[metric]) == 0 {
|
||||||
delete(jobData, metric)
|
delete(jobData.Metrics, metric)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -426,7 +429,7 @@ func (ccms *CCMetricStore) LoadScopedStats(
|
|||||||
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
|
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req := APIQueryRequest{
|
req := APIQueryRequest{
|
||||||
@@ -441,23 +444,23 @@ func (ccms *CCMetricStore) LoadScopedStats(
|
|||||||
resBody, err := ccms.doRequest(ctx, &req)
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
|
cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var errors []string
|
var errors []string
|
||||||
scopedJobStats := make(schema.ScopedJobStats)
|
scopedJobStats := schema.ScopedJobStats{Metrics: make(map[string]schema.ScopedMetricStats)}
|
||||||
|
|
||||||
for i, row := range resBody.Results {
|
for i, row := range resBody.Results {
|
||||||
query := req.Queries[i]
|
query := req.Queries[i]
|
||||||
metric := query.Metric
|
metric := query.Metric
|
||||||
scope := assignedScope[i]
|
scope := assignedScope[i]
|
||||||
|
|
||||||
if _, ok := scopedJobStats[metric]; !ok {
|
if _, ok := scopedJobStats.Metrics[metric]; !ok {
|
||||||
scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
|
scopedJobStats.Metrics[metric] = make(schema.ScopedMetricStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := scopedJobStats[metric][scope]; !ok {
|
if _, ok := scopedJobStats.Metrics[metric][scope]; !ok {
|
||||||
scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
|
scopedJobStats.Metrics[metric][scope] = make([]*schema.ScopedStats, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
@@ -471,7 +474,7 @@ func (ccms *CCMetricStore) LoadScopedStats(
|
|||||||
|
|
||||||
ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
|
ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
|
||||||
|
|
||||||
scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
|
scopedJobStats.Metrics[metric][scope] = append(scopedJobStats.Metrics[metric][scope], &schema.ScopedStats{
|
||||||
Hostname: query.Hostname,
|
Hostname: query.Hostname,
|
||||||
ID: id,
|
ID: id,
|
||||||
Data: &schema.MetricStatistics{
|
Data: &schema.MetricStatistics{
|
||||||
@@ -483,10 +486,10 @@ func (ccms *CCMetricStore) LoadScopedStats(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
|
// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
|
||||||
if len(scopedJobStats[metric][scope]) == 0 {
|
if len(scopedJobStats.Metrics[metric][scope]) == 0 {
|
||||||
delete(scopedJobStats[metric], scope)
|
delete(scopedJobStats.Metrics[metric], scope)
|
||||||
if len(scopedJobStats[metric]) == 0 {
|
if len(scopedJobStats.Metrics[metric]) == 0 {
|
||||||
delete(scopedJobStats, metric)
|
delete(scopedJobStats.Metrics, metric)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -617,6 +620,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
|
|||||||
resolution int,
|
resolution int,
|
||||||
from, to time.Time,
|
from, to time.Time,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
resampleAlgo string,
|
||||||
) (map[string]schema.JobData, error) {
|
) (map[string]schema.JobData, error) {
|
||||||
queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
|
queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -637,6 +641,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
|
|||||||
To: to.Unix(),
|
To: to.Unix(),
|
||||||
WithStats: true,
|
WithStats: true,
|
||||||
WithData: true,
|
WithData: true,
|
||||||
|
ResampleAlgo: resampleAlgo,
|
||||||
}
|
}
|
||||||
|
|
||||||
resBody, err := ccms.doRequest(ctx, &req)
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
@@ -690,14 +695,14 @@ func (ccms *CCMetricStore) LoadNodeListData(
|
|||||||
// Init Nested Map Data Structures If Not Found
|
// Init Nested Map Data Structures If Not Found
|
||||||
hostData, ok := data[query.Hostname]
|
hostData, ok := data[query.Hostname]
|
||||||
if !ok {
|
if !ok {
|
||||||
hostData = make(schema.JobData)
|
hostData = schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
|
||||||
data[query.Hostname] = hostData
|
data[query.Hostname] = hostData
|
||||||
}
|
}
|
||||||
|
|
||||||
metricData, ok := hostData[metric]
|
metricData, ok := hostData.Metrics[metric]
|
||||||
if !ok {
|
if !ok {
|
||||||
metricData = make(map[schema.MetricScope]*schema.JobMetric)
|
metricData = make(schema.ScopedMetrics)
|
||||||
data[query.Hostname][metric] = metricData
|
hostData.Metrics[metric] = metricData
|
||||||
}
|
}
|
||||||
|
|
||||||
scopeData, ok := metricData[scope]
|
scopeData, ok := metricData[scope]
|
||||||
@@ -707,7 +712,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
|
|||||||
Timestep: res,
|
Timestep: res,
|
||||||
Series: make([]schema.Series, 0),
|
Series: make([]schema.Series, 0),
|
||||||
}
|
}
|
||||||
data[query.Hostname][metric][scope] = scopeData
|
metricData[scope] = scopeData
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
|
|||||||
@@ -403,7 +403,7 @@ func (r *JobRepository) JobsStats(
|
|||||||
//
|
//
|
||||||
// Returns the requested statistic value or 0.0 if not found.
|
// Returns the requested statistic value or 0.0 if not found.
|
||||||
func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
|
func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
|
||||||
if stats, ok := job.Statistics[metric]; ok {
|
if stats, ok := job.Statistics.Metrics[metric]; ok {
|
||||||
switch statType {
|
switch statType {
|
||||||
case "avg":
|
case "avg":
|
||||||
return stats.Avg
|
return stats.Avg
|
||||||
|
|||||||
Vendored
BIN
Binary file not shown.
@@ -298,6 +298,18 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpdateRoles overwrites a user's role list with the provided roles.
|
||||||
|
// Used by the LDAP sync to reconcile elevated roles; callers are responsible for
|
||||||
|
// computing the full role set (the value replaces the existing one verbatim).
|
||||||
|
func (r *UserRepository) UpdateRoles(username string, roles []string) error {
|
||||||
|
rolesJSON, _ := json.Marshal(roles)
|
||||||
|
if _, err := sq.Update("hpc_user").Set("roles", rolesJSON).Where("hpc_user.username = ?", username).RunWith(r.DB).Exec(); err != nil {
|
||||||
|
cclog.Errorf("error while updating roles of user '%s'", username)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *UserRepository) DelUser(username string) error {
|
func (r *UserRepository) DelUser(username string) error {
|
||||||
_, err := r.DB.Exec(`DELETE FROM hpc_user WHERE hpc_user.username = ?`, username)
|
_, err := r.DB.Exec(`DELETE FROM hpc_user WHERE hpc_user.username = ?`, username)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
|
|
||||||
"github.com/ClusterCockpit/cc-backend/internal/config"
|
"github.com/ClusterCockpit/cc-backend/internal/config"
|
||||||
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
|
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
|
||||||
|
"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
|
||||||
"github.com/ClusterCockpit/cc-backend/internal/repository"
|
"github.com/ClusterCockpit/cc-backend/internal/repository"
|
||||||
"github.com/ClusterCockpit/cc-backend/web"
|
"github.com/ClusterCockpit/cc-backend/web"
|
||||||
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
|
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
|
||||||
@@ -496,13 +497,15 @@ func SetupRoutes(router chi.Router, buildInfo web.Build) {
|
|||||||
// Get Roles
|
// Get Roles
|
||||||
availableRoles, _ := schema.GetValidRolesMap(user)
|
availableRoles, _ := schema.GetValidRolesMap(user)
|
||||||
|
|
||||||
|
resampling := resamplingForUser(conf)
|
||||||
|
|
||||||
page := web.Page{
|
page := web.Page{
|
||||||
Title: title,
|
Title: title,
|
||||||
User: *user,
|
User: *user,
|
||||||
Roles: availableRoles,
|
Roles: availableRoles,
|
||||||
Build: buildInfo,
|
Build: buildInfo,
|
||||||
Config: conf,
|
Config: conf,
|
||||||
Resampling: config.Keys.EnableResampling,
|
Resampling: resampling,
|
||||||
Infos: infos,
|
Infos: infos,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -589,3 +592,36 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, buildInfo web.Buil
|
|||||||
web.RenderTemplate(rw, "message.tmpl", &web.Page{Title: "Warning", MsgType: "alert-warning", Message: "Empty search", User: *user, Roles: availableRoles, Build: buildInfo})
|
web.RenderTemplate(rw, "message.tmpl", &web.Page{Title: "Warning", MsgType: "alert-warning", Message: "Empty search", User: *user, Roles: availableRoles, Build: buildInfo})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resamplingForUser returns a ResampleConfig that incorporates the user's
|
||||||
|
// resample policy preference. If the user has a policy set, it creates a
|
||||||
|
// policy-derived config with targetPoints and trigger. Otherwise falls back
|
||||||
|
// to the global config.
|
||||||
|
func resamplingForUser(conf map[string]any) *config.ResampleConfig {
|
||||||
|
globalCfg := config.Keys.EnableResampling
|
||||||
|
|
||||||
|
policyStr := ""
|
||||||
|
if policyVal, ok := conf["plotConfiguration_resamplePolicy"]; ok {
|
||||||
|
if s, ok := policyVal.(string); ok {
|
||||||
|
policyStr = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to global default policy, then to "medium"
|
||||||
|
if policyStr == "" && globalCfg != nil {
|
||||||
|
policyStr = globalCfg.DefaultPolicy
|
||||||
|
}
|
||||||
|
if policyStr == "" {
|
||||||
|
policyStr = "medium"
|
||||||
|
}
|
||||||
|
|
||||||
|
policy := metricdispatch.ResamplePolicy(policyStr)
|
||||||
|
targetPoints := metricdispatch.TargetPointsForPolicy(policy)
|
||||||
|
if targetPoints == 0 {
|
||||||
|
return globalCfg
|
||||||
|
}
|
||||||
|
|
||||||
|
return &config.ResampleConfig{
|
||||||
|
TargetPoints: targetPoints,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ type JobClassTagger struct {
|
|||||||
// repo provides access to job database operations
|
// repo provides access to job database operations
|
||||||
repo JobRepository
|
repo JobRepository
|
||||||
// getStatistics retrieves job statistics for analysis
|
// getStatistics retrieves job statistics for analysis
|
||||||
getStatistics func(job *schema.Job) (map[string]schema.JobStatistics, error)
|
getStatistics func(job *schema.Job) (schema.JobStatisticsSet, error)
|
||||||
// getMetricConfig retrieves metric configuration (limits) for a cluster
|
// getMetricConfig retrieves metric configuration (limits) for a cluster
|
||||||
getMetricConfig func(cluster, subCluster string) map[string]*schema.Metric
|
getMetricConfig func(cluster, subCluster string) map[string]*schema.Metric
|
||||||
}
|
}
|
||||||
@@ -361,7 +361,7 @@ func (t *JobClassTagger) Match(job *schema.Job) {
|
|||||||
// add metrics to env
|
// add metrics to env
|
||||||
skipRule := false
|
skipRule := false
|
||||||
for _, m := range ri.metrics {
|
for _, m := range ri.metrics {
|
||||||
stats, ok := jobStats[m]
|
stats, ok := jobStats.Metrics[m]
|
||||||
if !ok {
|
if !ok {
|
||||||
cclog.Debugf("job classification: missing metric '%s' for rule %s on job %d", m, tag, job.JobID)
|
cclog.Debugf("job classification: missing metric '%s' for rule %s on job %d", m, tag, job.JobID)
|
||||||
skipRule = true
|
skipRule = true
|
||||||
|
|||||||
@@ -64,10 +64,10 @@ func TestClassifyJobMatch(t *testing.T) {
|
|||||||
parameters: make(map[string]any),
|
parameters: make(map[string]any),
|
||||||
tagType: "jobClass",
|
tagType: "jobClass",
|
||||||
repo: mockRepo,
|
repo: mockRepo,
|
||||||
getStatistics: func(job *schema.Job) (map[string]schema.JobStatistics, error) {
|
getStatistics: func(job *schema.Job) (schema.JobStatisticsSet, error) {
|
||||||
return map[string]schema.JobStatistics{
|
return schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
|
||||||
"flops_any": {Min: 0, Max: 200, Avg: 150},
|
"flops_any": {Min: 0, Max: 200, Avg: 150},
|
||||||
}, nil
|
}}, nil
|
||||||
},
|
},
|
||||||
getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
|
getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
|
||||||
return map[string]*schema.Metric{
|
return map[string]*schema.Metric{
|
||||||
@@ -120,10 +120,10 @@ func TestMatch_NoMatch(t *testing.T) {
|
|||||||
parameters: make(map[string]any),
|
parameters: make(map[string]any),
|
||||||
tagType: "jobClass",
|
tagType: "jobClass",
|
||||||
repo: mockRepo,
|
repo: mockRepo,
|
||||||
getStatistics: func(job *schema.Job) (map[string]schema.JobStatistics, error) {
|
getStatistics: func(job *schema.Job) (schema.JobStatisticsSet, error) {
|
||||||
return map[string]schema.JobStatistics{
|
return schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
|
||||||
"flops_any": {Min: 0, Max: 50, Avg: 20}, // Avg 20 < 100
|
"flops_any": {Min: 0, Max: 50, Avg: 20}, // Avg 20 < 100
|
||||||
}, nil
|
}}, nil
|
||||||
},
|
},
|
||||||
getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
|
getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
|
||||||
return map[string]*schema.Metric{
|
return map[string]*schema.Metric{
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ func RegisterFootprintWorker() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
job.Statistics = make(map[string]schema.JobStatistics)
|
job.Statistics = schema.JobStatisticsSet{Metrics: make(map[string]schema.JobStatistics, len(allMetrics))}
|
||||||
|
|
||||||
for _, metric := range allMetrics {
|
for _, metric := range allMetrics {
|
||||||
avg, min, max := 0.0, 0.0, 0.0
|
avg, min, max := 0.0, 0.0, 0.0
|
||||||
@@ -98,7 +98,7 @@ func RegisterFootprintWorker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add values rounded to 2 digits: repo.LoadStats may return unrounded
|
// Add values rounded to 2 digits: repo.LoadStats may return unrounded
|
||||||
job.Statistics[metric] = schema.JobStatistics{
|
job.Statistics.Metrics[metric] = schema.JobStatistics{
|
||||||
Unit: schema.Unit{
|
Unit: schema.Unit{
|
||||||
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
|
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
|
||||||
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
|
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
|
||||||
|
|||||||
+12
-12
@@ -31,7 +31,7 @@
|
|||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
// For S3 backend (endpoint, region, and usePathStyle are optional):
|
// For S3 backend (endpoint, region, and use-path-style are optional):
|
||||||
//
|
//
|
||||||
// {
|
// {
|
||||||
// "archive": {
|
// "archive": {
|
||||||
@@ -39,9 +39,9 @@
|
|||||||
// "endpoint": "http://192.168.178.10",
|
// "endpoint": "http://192.168.178.10",
|
||||||
// "bucket": "my-job-archive",
|
// "bucket": "my-job-archive",
|
||||||
// "region": "us-east-1",
|
// "region": "us-east-1",
|
||||||
// "usePathStyle": true,
|
// "use-path-style": true,
|
||||||
// "accessKey": "...",
|
// "access-key": "...",
|
||||||
// "secretKey": "..."
|
// "secret-key": "..."
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
@@ -50,7 +50,7 @@
|
|||||||
// {
|
// {
|
||||||
// "archive": {
|
// "archive": {
|
||||||
// "kind": "sqlite",
|
// "kind": "sqlite",
|
||||||
// "dbPath": "/var/lib/archive.db"
|
// "db-path": "/var/lib/archive.db"
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
@@ -299,7 +299,7 @@ func LoadAveragesFromArchive(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, m := range metrics {
|
for i, m := range metrics {
|
||||||
if stat, ok := metaFile.Statistics[m]; ok {
|
if stat, ok := metaFile.Statistics.Metrics[m]; ok {
|
||||||
data[i] = append(data[i], schema.Float(stat.Avg))
|
data[i] = append(data[i], schema.Float(stat.Avg))
|
||||||
} else {
|
} else {
|
||||||
data[i] = append(data[i], schema.NaN)
|
data[i] = append(data[i], schema.NaN)
|
||||||
@@ -323,7 +323,7 @@ func LoadStatsFromArchive(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, m := range metrics {
|
for _, m := range metrics {
|
||||||
stat, ok := metaFile.Statistics[m]
|
stat, ok := metaFile.Statistics.Metrics[m]
|
||||||
if !ok {
|
if !ok {
|
||||||
data[m] = schema.MetricStatistics{Min: 0.0, Avg: 0.0, Max: 0.0}
|
data[m] = schema.MetricStatistics{Min: 0.0, Avg: 0.0, Max: 0.0}
|
||||||
continue
|
continue
|
||||||
@@ -349,19 +349,19 @@ func LoadScopedStatsFromArchive(
|
|||||||
data, err := ar.LoadJobStats(job)
|
data, err := ar.LoadJobStats(job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while loading job stats from archiveBackend: %s", err.Error())
|
cclog.Errorf("Error while loading job stats from archiveBackend: %s", err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetStatistics returns all metric statistics for a job.
|
// GetStatistics returns all metric statistics for a job, including the
|
||||||
// Returns a map of metric names to their job-level statistics.
|
// array-valued statistics groups (e.g. filesystems).
|
||||||
func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
|
func GetStatistics(job *schema.Job) (schema.JobStatisticsSet, error) {
|
||||||
metaFile, err := ar.LoadJobMeta(job)
|
metaFile, err := ar.LoadJobMeta(job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error())
|
cclog.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error())
|
||||||
return nil, err
|
return schema.JobStatisticsSet{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return metaFile.Statistics, nil
|
return metaFile.Statistics, nil
|
||||||
|
|||||||
@@ -63,8 +63,11 @@ func initClusterConfig() error {
|
|||||||
|
|
||||||
if _, ok := metricLookup[mc.Name]; !ok {
|
if _, ok := metricLookup[mc.Name]; !ok {
|
||||||
metricLookup[mc.Name] = schema.GlobalMetricListItem{
|
metricLookup[mc.Name] = schema.GlobalMetricListItem{
|
||||||
Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint,
|
Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint, Tooltip: mc.Tooltip,
|
||||||
}
|
}
|
||||||
|
} else if item := metricLookup[mc.Name]; item.Tooltip == "" && mc.Tooltip != "" {
|
||||||
|
item.Tooltip = mc.Tooltip
|
||||||
|
metricLookup[mc.Name] = item
|
||||||
}
|
}
|
||||||
|
|
||||||
availability := schema.ClusterSupport{Cluster: cluster.Name}
|
availability := schema.ClusterSupport{Cluster: cluster.Name}
|
||||||
@@ -139,8 +142,10 @@ func initClusterConfig() error {
|
|||||||
userItem, ok := userMetricLookup[mc.Name]
|
userItem, ok := userMetricLookup[mc.Name]
|
||||||
if !ok {
|
if !ok {
|
||||||
userItem = schema.GlobalMetricListItem{
|
userItem = schema.GlobalMetricListItem{
|
||||||
Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint,
|
Name: mc.Name, Scope: mc.Scope, Unit: mc.Unit, Footprint: mc.Footprint, Tooltip: mc.Tooltip,
|
||||||
}
|
}
|
||||||
|
} else if userItem.Tooltip == "" && mc.Tooltip != "" {
|
||||||
|
userItem.Tooltip = mc.Tooltip
|
||||||
}
|
}
|
||||||
userItem.Availability = append(userItem.Availability, userAvailability)
|
userItem.Availability = append(userItem.Availability, userAvailability)
|
||||||
userMetricLookup[mc.Name] = userItem
|
userMetricLookup[mc.Name] = userItem
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) {
|
|||||||
f, err := os.Open(filename)
|
f, err := os.Open(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("fsBackend LoadJobData()- %v", err)
|
cclog.Errorf("fsBackend LoadJobData()- %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
@@ -101,7 +101,7 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) {
|
|||||||
r, err := gzip.NewReader(f)
|
r, err := gzip.NewReader(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf(" %v", err)
|
cclog.Errorf(" %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
|
|||||||
f, err := os.Open(filename)
|
f, err := os.Open(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("fsBackend LoadJobStats()- %v", err)
|
cclog.Errorf("fsBackend LoadJobStats()- %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
@@ -134,13 +134,13 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
|
|||||||
r, err := gzip.NewReader(f)
|
r, err := gzip.NewReader(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf(" %v", err)
|
cclog.Errorf(" %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
|
|
||||||
if config.Keys.Validate {
|
if config.Keys.Validate {
|
||||||
if err := schema.Validate(schema.Data, r); err != nil {
|
if err := schema.Validate(schema.Data, r); err != nil {
|
||||||
return nil, fmt.Errorf("validate job data: %v", err)
|
return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,7 +148,7 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
|
|||||||
} else {
|
} else {
|
||||||
if config.Keys.Validate {
|
if config.Keys.Validate {
|
||||||
if err := schema.Validate(schema.Data, bufio.NewReader(f)); err != nil {
|
if err := schema.Validate(schema.Data, bufio.NewReader(f)); err != nil {
|
||||||
return nil, fmt.Errorf("validate job data: %v", err)
|
return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return DecodeJobStats(bufio.NewReader(f), filename)
|
return DecodeJobStats(bufio.NewReader(f), filename)
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ func TestLoadJobData(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, scopes := range data {
|
for _, scopes := range data.Metrics {
|
||||||
// fmt.Printf("Metric name: %s\n", name)
|
// fmt.Printf("Metric name: %s\n", name)
|
||||||
|
|
||||||
if _, exists := scopes[schema.MetricScopeNode]; !exists {
|
if _, exists := scopes[schema.MetricScopeNode]; !exists {
|
||||||
|
|||||||
+46
-26
@@ -27,7 +27,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) {
|
|||||||
|
|
||||||
if err, ok := data.(error); ok {
|
if err, ok := data.(error); ok {
|
||||||
cclog.Warn("Error in decoded job data set")
|
cclog.Warn("Error in decoded job data set")
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.(schema.JobData), nil
|
return data.(schema.JobData), nil
|
||||||
@@ -35,39 +35,59 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) {
|
|||||||
|
|
||||||
func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) {
|
func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) {
|
||||||
jobData, err := DecodeJobData(r, k)
|
jobData, err := DecodeJobData(r, k)
|
||||||
|
if err != nil {
|
||||||
|
return schema.ScopedJobStats{}, err
|
||||||
|
}
|
||||||
|
|
||||||
// Convert schema.JobData to schema.ScopedJobStats
|
// Convert schema.JobData to schema.ScopedJobStats
|
||||||
if jobData != nil {
|
scopedJobStats := schema.ScopedJobStats{
|
||||||
scopedJobStats := make(schema.ScopedJobStats)
|
Metrics: scopedStatsFromMetrics(jobData.Metrics),
|
||||||
for metric, metricData := range jobData {
|
|
||||||
if _, ok := scopedJobStats[metric]; !ok {
|
|
||||||
scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for scope, jobMetric := range metricData {
|
for _, group := range jobData.Groups {
|
||||||
if _, ok := scopedJobStats[metric][scope]; !ok {
|
statsGroup := schema.ScopedStatsGroup{Key: group.Key}
|
||||||
scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
|
for _, inst := range group.Instances {
|
||||||
}
|
statsGroup.Instances = append(statsGroup.Instances, schema.ScopedStatsGroupInstance{
|
||||||
|
Name: inst.Name,
|
||||||
for _, series := range jobMetric.Series {
|
Type: inst.Type,
|
||||||
scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
|
Metrics: scopedStatsFromMetrics(inst.Metrics),
|
||||||
Hostname: series.Hostname,
|
|
||||||
ID: series.ID,
|
|
||||||
Data: &series.Statistics,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
scopedJobStats.Groups = append(scopedJobStats.Groups, statsGroup)
|
||||||
|
}
|
||||||
|
|
||||||
// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
|
|
||||||
if len(scopedJobStats[metric][scope]) == 0 {
|
|
||||||
delete(scopedJobStats[metric], scope)
|
|
||||||
if len(scopedJobStats[metric]) == 0 {
|
|
||||||
delete(scopedJobStats, metric)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return scopedJobStats, nil
|
return scopedJobStats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// scopedStatsFromMetrics reduces the full time series of every metric/scope to
|
||||||
|
// the per-series statistics. Scopes without any series are dropped so that
|
||||||
|
// callers can rely on len(stats[metric][scope]) being non-zero when present.
|
||||||
|
func scopedStatsFromMetrics(metrics map[string]schema.ScopedMetrics) map[string]schema.ScopedMetricStats {
|
||||||
|
stats := make(map[string]schema.ScopedMetricStats, len(metrics))
|
||||||
|
for metric, metricData := range metrics {
|
||||||
|
scoped := make(schema.ScopedMetricStats, len(metricData))
|
||||||
|
for scope, jobMetric := range metricData {
|
||||||
|
if len(jobMetric.Series) == 0 {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
return nil, err
|
|
||||||
|
series := make([]*schema.ScopedStats, 0, len(jobMetric.Series))
|
||||||
|
for i := range jobMetric.Series {
|
||||||
|
series = append(series, &schema.ScopedStats{
|
||||||
|
Hostname: jobMetric.Series[i].Hostname,
|
||||||
|
ID: jobMetric.Series[i].ID,
|
||||||
|
Data: &jobMetric.Series[i].Statistics,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
scoped[scope] = series
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(scoped) > 0 {
|
||||||
|
stats[metric] = scoped
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats
|
||||||
}
|
}
|
||||||
|
|
||||||
func DecodeJobMeta(r io.Reader) (*schema.Job, error) {
|
func DecodeJobMeta(r io.Reader) (*schema.Job, error) {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func JobToParquetRow(meta *schema.Job, data *schema.JobData) (*ParquetJobRow, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
var statisticsJSON []byte
|
var statisticsJSON []byte
|
||||||
if meta.Statistics != nil {
|
if len(meta.Statistics.Metrics) > 0 || len(meta.Statistics.Groups) > 0 {
|
||||||
statisticsJSON, err = json.Marshal(meta.Statistics)
|
statisticsJSON, err = json.Marshal(meta.Statistics)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal statistics: %w", err)
|
return nil, fmt.Errorf("marshal statistics: %w", err)
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ func TestParquetRowToJob(t *testing.T) {
|
|||||||
{Hostname: "node001", HWThreads: []int{0, 1, 2, 3}},
|
{Hostname: "node001", HWThreads: []int{0, 1, 2, 3}},
|
||||||
{Hostname: "node002", HWThreads: []int{4, 5, 6, 7}},
|
{Hostname: "node002", HWThreads: []int{4, 5, 6, 7}},
|
||||||
},
|
},
|
||||||
Statistics: map[string]schema.JobStatistics{
|
Statistics: schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
|
||||||
"cpu_load": {Avg: 50.0, Min: 10.0, Max: 90.0},
|
"cpu_load": {Avg: 50.0, Min: 10.0, Max: 90.0},
|
||||||
},
|
}},
|
||||||
Tags: []*schema.Tag{
|
Tags: []*schema.Tag{
|
||||||
{Type: "test", Name: "tag1"},
|
{Type: "test", Name: "tag1"},
|
||||||
},
|
},
|
||||||
@@ -49,7 +49,7 @@ func TestParquetRowToJob(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
data := &schema.JobData{
|
data := &schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"cpu_load": {
|
"cpu_load": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Unit: schema.Unit{Base: ""},
|
Unit: schema.Unit{Base: ""},
|
||||||
@@ -62,7 +62,7 @@ func TestParquetRowToJob(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
// Convert to parquet row
|
// Convert to parquet row
|
||||||
row, err := JobToParquetRow(meta, data)
|
row, err := JobToParquetRow(meta, data)
|
||||||
@@ -134,10 +134,10 @@ func TestParquetRowToJob(t *testing.T) {
|
|||||||
t.Errorf("Resources[0].HWThreads len = %d, want 4", len(gotMeta.Resources[0].HWThreads))
|
t.Errorf("Resources[0].HWThreads len = %d, want 4", len(gotMeta.Resources[0].HWThreads))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(gotMeta.Statistics) != 1 {
|
if len(gotMeta.Statistics.Metrics) != 1 {
|
||||||
t.Fatalf("Statistics len = %d, want 1", len(gotMeta.Statistics))
|
t.Fatalf("Statistics len = %d, want 1", len(gotMeta.Statistics.Metrics))
|
||||||
}
|
}
|
||||||
if stat, ok := gotMeta.Statistics["cpu_load"]; !ok {
|
if stat, ok := gotMeta.Statistics.Metrics["cpu_load"]; !ok {
|
||||||
t.Error("Statistics missing cpu_load")
|
t.Error("Statistics missing cpu_load")
|
||||||
} else if stat.Avg != 50.0 {
|
} else if stat.Avg != 50.0 {
|
||||||
t.Errorf("Statistics[cpu_load].Avg = %f, want 50.0", stat.Avg)
|
t.Errorf("Statistics[cpu_load].Avg = %f, want 50.0", stat.Avg)
|
||||||
@@ -163,7 +163,7 @@ func TestParquetRowToJob(t *testing.T) {
|
|||||||
if gotData == nil {
|
if gotData == nil {
|
||||||
t.Fatal("JobData is nil")
|
t.Fatal("JobData is nil")
|
||||||
}
|
}
|
||||||
cpuLoad, ok := (*gotData)["cpu_load"]
|
cpuLoad, ok := gotData.Metrics["cpu_load"]
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatal("JobData missing cpu_load")
|
t.Fatal("JobData missing cpu_load")
|
||||||
}
|
}
|
||||||
@@ -201,7 +201,7 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
data := &schema.JobData{
|
data := &schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"cpu_load": {
|
"cpu_load": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Timestep: 60,
|
Timestep: 60,
|
||||||
@@ -210,7 +210,7 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
row, err := JobToParquetRow(meta, data)
|
row, err := JobToParquetRow(meta, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -228,8 +228,8 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
|
|||||||
if gotMeta.Tags != nil {
|
if gotMeta.Tags != nil {
|
||||||
t.Errorf("Tags should be nil, got %v", gotMeta.Tags)
|
t.Errorf("Tags should be nil, got %v", gotMeta.Tags)
|
||||||
}
|
}
|
||||||
if gotMeta.Statistics != nil {
|
if len(gotMeta.Statistics.Metrics) != 0 || len(gotMeta.Statistics.Groups) != 0 {
|
||||||
t.Errorf("Statistics should be nil, got %v", gotMeta.Statistics)
|
t.Errorf("Statistics should be empty, got %v", gotMeta.Statistics)
|
||||||
}
|
}
|
||||||
if gotMeta.MetaData != nil {
|
if gotMeta.MetaData != nil {
|
||||||
t.Errorf("MetaData should be nil, got %v", gotMeta.MetaData)
|
t.Errorf("MetaData should be nil, got %v", gotMeta.MetaData)
|
||||||
@@ -299,7 +299,7 @@ func TestRoundTripThroughParquetFile(t *testing.T) {
|
|||||||
if gotData == nil {
|
if gotData == nil {
|
||||||
t.Fatal("JobData is nil")
|
t.Fatal("JobData is nil")
|
||||||
}
|
}
|
||||||
if _, ok := (*gotData)["cpu_load"]; !ok {
|
if _, ok := gotData.Metrics["cpu_load"]; !ok {
|
||||||
t.Error("JobData missing cpu_load")
|
t.Error("JobData missing cpu_load")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ func makeTestJob(jobID int64) (*schema.Job, *schema.JobData) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
data := schema.JobData{
|
data := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
|
||||||
"cpu_load": {
|
"cpu_load": {
|
||||||
schema.MetricScopeNode: &schema.JobMetric{
|
schema.MetricScopeNode: &schema.JobMetric{
|
||||||
Unit: schema.Unit{Base: ""},
|
Unit: schema.Unit{Base: ""},
|
||||||
@@ -70,7 +70,7 @@ func makeTestJob(jobID int64) (*schema.Job, *schema.JobData) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}}
|
||||||
|
|
||||||
return meta, &data
|
return meta, &data
|
||||||
}
|
}
|
||||||
@@ -132,7 +132,7 @@ func TestJobToParquetRowConversion(t *testing.T) {
|
|||||||
if err := json.Unmarshal(decompressed, &jobData); err != nil {
|
if err := json.Unmarshal(decompressed, &jobData); err != nil {
|
||||||
t.Fatalf("unmarshal metric data: %v", err)
|
t.Fatalf("unmarshal metric data: %v", err)
|
||||||
}
|
}
|
||||||
if _, ok := jobData["cpu_load"]; !ok {
|
if _, ok := jobData.Metrics["cpu_load"]; !ok {
|
||||||
t.Error("metric data missing cpu_load key")
|
t.Error("metric data missing cpu_load key")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -330,7 +330,7 @@ func (s3a *S3Archive) LoadJobData(job *schema.Job) (schema.JobData, error) {
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("S3Archive LoadJobData() > GetObject error: %v", err)
|
cclog.Errorf("S3Archive LoadJobData() > GetObject error: %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
@@ -349,7 +349,7 @@ func (s3a *S3Archive) LoadJobData(job *schema.Job) (schema.JobData, error) {
|
|||||||
r, err := gzip.NewReader(result.Body)
|
r, err := gzip.NewReader(result.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("S3Archive LoadJobData() > gzip error: %v", err)
|
cclog.Errorf("S3Archive LoadJobData() > gzip error: %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
|
|
||||||
@@ -381,14 +381,14 @@ func (s3a *S3Archive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, erro
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("S3Archive LoadJobStats() > GetObject error: %v", err)
|
cclog.Errorf("S3Archive LoadJobStats() > GetObject error: %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
defer result.Body.Close()
|
defer result.Body.Close()
|
||||||
|
|
||||||
if config.Keys.Validate {
|
if config.Keys.Validate {
|
||||||
b, _ := io.ReadAll(result.Body)
|
b, _ := io.ReadAll(result.Body)
|
||||||
if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
|
if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
|
||||||
return nil, fmt.Errorf("validate job data: %v", err)
|
return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
|
||||||
}
|
}
|
||||||
return DecodeJobStats(bytes.NewReader(b), key)
|
return DecodeJobStats(bytes.NewReader(b), key)
|
||||||
}
|
}
|
||||||
@@ -400,14 +400,14 @@ func (s3a *S3Archive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, erro
|
|||||||
r, err := gzip.NewReader(result.Body)
|
r, err := gzip.NewReader(result.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("S3Archive LoadJobStats() > gzip error: %v", err)
|
cclog.Errorf("S3Archive LoadJobStats() > gzip error: %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
|
|
||||||
if config.Keys.Validate {
|
if config.Keys.Validate {
|
||||||
b, _ := io.ReadAll(r)
|
b, _ := io.ReadAll(r)
|
||||||
if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
|
if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
|
||||||
return nil, fmt.Errorf("validate job data: %v", err)
|
return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
|
||||||
}
|
}
|
||||||
return DecodeJobStats(bytes.NewReader(b), keyGz)
|
return DecodeJobStats(bytes.NewReader(b), keyGz)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -252,7 +252,7 @@ func (sa *SqliteArchive) LoadJobData(job *schema.Job) (schema.JobData, error) {
|
|||||||
job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
|
job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("SqliteArchive LoadJobData() > query error: %v", err)
|
cclog.Errorf("SqliteArchive LoadJobData() > query error: %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
|
key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
|
||||||
|
|
||||||
@@ -261,7 +261,7 @@ func (sa *SqliteArchive) LoadJobData(job *schema.Job) (schema.JobData, error) {
|
|||||||
gzipReader, err := gzip.NewReader(reader)
|
gzipReader, err := gzip.NewReader(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("SqliteArchive LoadJobData() > gzip error: %v", err)
|
cclog.Errorf("SqliteArchive LoadJobData() > gzip error: %v", err)
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
defer gzipReader.Close()
|
defer gzipReader.Close()
|
||||||
reader = gzipReader
|
reader = gzipReader
|
||||||
@@ -285,7 +285,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
|
|||||||
job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
|
job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("SqliteArchive LoadJobStats() > query error: %v", err)
|
cclog.Errorf("SqliteArchive LoadJobStats() > query error: %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
|
key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
|
||||||
|
|
||||||
@@ -294,7 +294,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
|
|||||||
gzipReader, err := gzip.NewReader(reader)
|
gzipReader, err := gzip.NewReader(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("SqliteArchive LoadJobStats() > gzip error: %v", err)
|
cclog.Errorf("SqliteArchive LoadJobStats() > gzip error: %v", err)
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
defer gzipReader.Close()
|
defer gzipReader.Close()
|
||||||
reader = gzipReader
|
reader = gzipReader
|
||||||
@@ -303,7 +303,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
|
|||||||
if config.Keys.Validate {
|
if config.Keys.Validate {
|
||||||
data, _ := io.ReadAll(reader)
|
data, _ := io.ReadAll(reader)
|
||||||
if err := schema.Validate(schema.Data, bytes.NewReader(data)); err != nil {
|
if err := schema.Validate(schema.Data, bytes.NewReader(data)); err != nil {
|
||||||
return nil, fmt.Errorf("validate job data: %v", err)
|
return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
|
||||||
}
|
}
|
||||||
return DecodeJobStats(bytes.NewReader(data), key)
|
return DecodeJobStats(bytes.NewReader(data), key)
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -46,7 +46,7 @@
|
|||||||
{
|
{
|
||||||
"name": "flops_any",
|
"name": "flops_any",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "Flops/s",
|
"base": "F/s",
|
||||||
"prefix": "G"
|
"prefix": "G"
|
||||||
},
|
},
|
||||||
"scope": "hwthread",
|
"scope": "hwthread",
|
||||||
|
|||||||
+5
-5
@@ -85,7 +85,7 @@
|
|||||||
{
|
{
|
||||||
"name": "flops_any",
|
"name": "flops_any",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "Flops/s",
|
"base": "F/s",
|
||||||
"prefix": "G"
|
"prefix": "G"
|
||||||
},
|
},
|
||||||
"scope": "hwthread",
|
"scope": "hwthread",
|
||||||
@@ -118,7 +118,7 @@
|
|||||||
{
|
{
|
||||||
"name": "flops_sp",
|
"name": "flops_sp",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "Flops/s",
|
"base": "F/s",
|
||||||
"prefix": "G"
|
"prefix": "G"
|
||||||
},
|
},
|
||||||
"scope": "hwthread",
|
"scope": "hwthread",
|
||||||
@@ -150,7 +150,7 @@
|
|||||||
{
|
{
|
||||||
"name": "flops_dp",
|
"name": "flops_dp",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "Flops/s",
|
"base": "F/s",
|
||||||
"prefix": "G"
|
"prefix": "G"
|
||||||
},
|
},
|
||||||
"scope": "hwthread",
|
"scope": "hwthread",
|
||||||
@@ -327,7 +327,7 @@
|
|||||||
{
|
{
|
||||||
"name": "ib_recv_pkts",
|
"name": "ib_recv_pkts",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "packets/s"
|
"base": ""
|
||||||
},
|
},
|
||||||
"scope": "node",
|
"scope": "node",
|
||||||
"aggregation": "sum",
|
"aggregation": "sum",
|
||||||
@@ -340,7 +340,7 @@
|
|||||||
{
|
{
|
||||||
"name": "ib_xmit_pkts",
|
"name": "ib_xmit_pkts",
|
||||||
"unit": {
|
"unit": {
|
||||||
"base": "packets/s"
|
"base": ""
|
||||||
},
|
},
|
||||||
"scope": "node",
|
"scope": "node",
|
||||||
"aggregation": "sum",
|
"aggregation": "sum",
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ type APIQueryRequest struct {
|
|||||||
WithStats bool `json:"with-stats"`
|
WithStats bool `json:"with-stats"`
|
||||||
WithData bool `json:"with-data"`
|
WithData bool `json:"with-data"`
|
||||||
WithPadding bool `json:"with-padding"`
|
WithPadding bool `json:"with-padding"`
|
||||||
|
ResampleAlgo string `json:"resample-algo,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// APIQueryResponse represents the response to an APIQueryRequest.
|
// APIQueryResponse represents the response to an APIQueryRequest.
|
||||||
@@ -279,7 +280,7 @@ func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
|
|||||||
for _, sel := range sels {
|
for _, sel := range sels {
|
||||||
data := APIMetricData{}
|
data := APIMetricData{}
|
||||||
|
|
||||||
data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
|
data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution, req.ResampleAlgo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Skip Error If Just Missing Host or Metric, Continue
|
// Skip Error If Just Missing Host or Metric, Continue
|
||||||
// Empty Return For Metric Handled Gracefully By Frontend
|
// Empty Return For Metric Handled Gracefully By Frontend
|
||||||
|
|||||||
@@ -94,16 +94,32 @@ var ErrNoNewArchiveData error = errors.New("all data already archived")
|
|||||||
|
|
||||||
// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
|
// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
|
||||||
// When archiving, consolidates all hosts per cluster into a single Parquet file.
|
// When archiving, consolidates all hosts per cluster into a single Parquet file.
|
||||||
|
//
|
||||||
|
// Hosts currently in use by running jobs (per the MemoryStore's NodeProvider)
|
||||||
|
// are skipped entirely; their files are handled by a later cycle once the job
|
||||||
|
// has ended. If the provider query fails, the whole cycle is aborted: deleting
|
||||||
|
// without knowing which nodes are in use risks data loss. Without a provider,
|
||||||
|
// everything older than `from` is cleaned (old behavior).
|
||||||
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
|
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
|
||||||
if deleteInstead {
|
var usedNodes map[string][]string
|
||||||
return deleteCheckpoints(checkpointsDir, from)
|
if ms := GetMemoryStore(); ms != nil && ms.nodeProvider != nil {
|
||||||
|
un, err := ms.nodeProvider.GetUsedNodes(from)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("aborting cleanup, could not determine nodes in use: %w", err)
|
||||||
|
}
|
||||||
|
usedNodes = un
|
||||||
}
|
}
|
||||||
|
|
||||||
return archiveCheckpoints(checkpointsDir, cleanupDir, from)
|
if deleteInstead {
|
||||||
|
return deleteCheckpoints(checkpointsDir, from, usedNodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
return archiveCheckpoints(checkpointsDir, cleanupDir, from, usedNodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
|
// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
|
||||||
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
|
// Hosts present in usedNodes are skipped.
|
||||||
|
func deleteCheckpoints(checkpointsDir string, from int64, usedNodes map[string][]string) (int, error) {
|
||||||
entries1, err := os.ReadDir(checkpointsDir)
|
entries1, err := os.ReadDir(checkpointsDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -157,6 +173,10 @@ func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, de2 := range entries2 {
|
for _, de2 := range entries2 {
|
||||||
|
if isNodeUsed(usedNodes, de1.Name(), de2.Name()) {
|
||||||
|
cclog.Debugf("[METRICSTORE]> skipping delete for %s/%s: node in use by running job", de1.Name(), de2.Name())
|
||||||
|
continue
|
||||||
|
}
|
||||||
work <- workItem{
|
work <- workItem{
|
||||||
dir: filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
|
dir: filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
|
||||||
cluster: de1.Name(),
|
cluster: de1.Name(),
|
||||||
@@ -182,7 +202,7 @@ func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
|
|||||||
// Workers load checkpoint files from disk and send CheckpointFile trees on a
|
// Workers load checkpoint files from disk and send CheckpointFile trees on a
|
||||||
// back-pressured channel. The main thread streams each tree directly to Parquet
|
// back-pressured channel. The main thread streams each tree directly to Parquet
|
||||||
// rows without materializing all rows in memory.
|
// rows without materializing all rows in memory.
|
||||||
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
|
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64, usedNodes map[string][]string) (int, error) {
|
||||||
cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
|
cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
@@ -252,6 +272,10 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
|
|||||||
if !hostEntry.IsDir() {
|
if !hostEntry.IsDir() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if isNodeUsed(usedNodes, cluster, hostEntry.Name()) {
|
||||||
|
cclog.Debugf("[METRICSTORE]> skipping archive for %s/%s: node in use by running job", cluster, hostEntry.Name())
|
||||||
|
continue
|
||||||
|
}
|
||||||
dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
|
dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
|
||||||
work <- struct {
|
work <- struct {
|
||||||
dir, host string
|
dir, host string
|
||||||
|
|||||||
+144
-15
@@ -42,6 +42,7 @@ package metricstore
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -176,6 +177,17 @@ type buffer struct {
|
|||||||
archived bool
|
archived bool
|
||||||
closed bool
|
closed bool
|
||||||
lastUsed int64
|
lastUsed int64
|
||||||
|
|
||||||
|
// Running statistics over the non-NaN values currently in data.
|
||||||
|
// avg is derived as statSum/statSamples, never stored.
|
||||||
|
// statsValid is false when the cached aggregate may be stale (after an
|
||||||
|
// overwrite, or for a bare-constructed buffer); a stats query recomputes
|
||||||
|
// before using the cache. Zero value (false) fails safe to "recompute".
|
||||||
|
statSum float64
|
||||||
|
statSamples int
|
||||||
|
statMin float64
|
||||||
|
statMax float64
|
||||||
|
statsValid bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBuffer(ts, freq int64) *buffer {
|
func newBuffer(ts, freq int64) *buffer {
|
||||||
@@ -187,9 +199,31 @@ func newBuffer(ts, freq int64) *buffer {
|
|||||||
b.archived = false
|
b.archived = false
|
||||||
b.closed = false
|
b.closed = false
|
||||||
b.data = b.data[:0]
|
b.data = b.data[:0]
|
||||||
|
b.statSum = 0
|
||||||
|
b.statSamples = 0
|
||||||
|
b.statMin = math.MaxFloat32
|
||||||
|
b.statMax = -math.MaxFloat32
|
||||||
|
b.statsValid = true
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// updateStats folds a single value into the buffer's running statistics.
|
||||||
|
// NaN values are ignored so they never affect min/max/sum/samples.
|
||||||
|
func (b *buffer) updateStats(value schema.Float) {
|
||||||
|
xf := float64(value)
|
||||||
|
if math.IsNaN(xf) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.statSum += xf
|
||||||
|
b.statSamples++
|
||||||
|
if xf < b.statMin {
|
||||||
|
b.statMin = xf
|
||||||
|
}
|
||||||
|
if xf > b.statMax {
|
||||||
|
b.statMax = xf
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// write appends a timestamped value to the buffer chain.
|
// write appends a timestamped value to the buffer chain.
|
||||||
//
|
//
|
||||||
// Returns the head buffer (which may be newly created if capacity was reached).
|
// Returns the head buffer (which may be newly created if capacity was reached).
|
||||||
@@ -224,6 +258,9 @@ func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
|
|||||||
// Overwriting value or writing value from past
|
// Overwriting value or writing value from past
|
||||||
if idx < len(b.data) {
|
if idx < len(b.data) {
|
||||||
b.data[idx] = value
|
b.data[idx] = value
|
||||||
|
// The replaced value may have been the running min/max; the cached
|
||||||
|
// aggregate can no longer be trusted. Force a recompute on next query.
|
||||||
|
b.statsValid = false
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -233,6 +270,7 @@ func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
b.data = append(b.data, value)
|
b.data = append(b.data, value)
|
||||||
|
b.updateStats(value)
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -265,43 +303,134 @@ func (b *buffer) firstWrite() int64 {
|
|||||||
// - error: Non-nil on failure
|
// - error: Non-nil on failure
|
||||||
//
|
//
|
||||||
// Panics if 'data' slice is too small to hold all values in [from, to).
|
// Panics if 'data' slice is too small to hold all values in [from, to).
|
||||||
func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
|
// func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
|
||||||
// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
|
// // Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
|
||||||
for from < b.firstWrite() {
|
// for from < b.firstWrite() {
|
||||||
if b.prev == nil {
|
// if b.prev == nil {
|
||||||
from = b.firstWrite()
|
// from = b.firstWrite()
|
||||||
break
|
// break
|
||||||
}
|
// }
|
||||||
|
// b = b.prev
|
||||||
|
// }
|
||||||
|
|
||||||
|
// i := 0
|
||||||
|
// t := from
|
||||||
|
// for ; t < to; t += b.frequency {
|
||||||
|
// idx := int((t - b.start) / b.frequency)
|
||||||
|
// if idx >= cap(b.data) {
|
||||||
|
// if b.next == nil {
|
||||||
|
// break
|
||||||
|
// }
|
||||||
|
// b = b.next
|
||||||
|
// // Recalculate idx in the new buffer; a gap between buffers may exist.
|
||||||
|
// idx = int((t - b.start) / b.frequency)
|
||||||
|
// }
|
||||||
|
|
||||||
|
// if idx >= len(b.data) {
|
||||||
|
// if b.next == nil || to <= b.next.start {
|
||||||
|
// break
|
||||||
|
// }
|
||||||
|
// data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
|
||||||
|
// } else if t < b.start {
|
||||||
|
// data[i] += schema.NaN // gap before this buffer's first write
|
||||||
|
// } else {
|
||||||
|
// data[i] += b.data[idx]
|
||||||
|
// }
|
||||||
|
// i++
|
||||||
|
// }
|
||||||
|
|
||||||
|
// return data[:i], from, t, nil
|
||||||
|
// }
|
||||||
|
|
||||||
|
// read retrieves time-series data from the buffer chain for the specified time range.
|
||||||
|
//
|
||||||
|
// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
|
||||||
|
// buffer's start. Missing data points are represented as NaN. Values are accumulated
|
||||||
|
// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
|
||||||
|
//
|
||||||
|
// Points with no stored data (before the oldest write, in inter-buffer gaps, or
|
||||||
|
// past the newest write) are NaN-filled while scanning. Whether those NaN pads
|
||||||
|
// are kept is controlled by 'trim':
|
||||||
|
//
|
||||||
|
// - trim == false: the returned slice always spans the full requested window
|
||||||
|
// [from, to), NaN where missing. Use this when several scopes accumulate
|
||||||
|
// into one index-aligned slice (aggregation) or when the caller wants the
|
||||||
|
// whole window for display.
|
||||||
|
// - trim == true: the returned slice is cut down to the real data extent
|
||||||
|
// [dataFrom, dataTo), i.e. leading and trailing NaN pads are dropped so that
|
||||||
|
// result[0] aligns with dataFrom. Use this when persisting only real data
|
||||||
|
// (checkpointing).
|
||||||
|
//
|
||||||
|
// The second and third return values report the actual extent of real (non-NaN)
|
||||||
|
// data found within the window: dataFrom is the timestamp of the first stored
|
||||||
|
// sample and dataTo is one frequency past the last. When the buffer fully covers
|
||||||
|
// the request these equal from/to; when a scope holds less data they shrink,
|
||||||
|
// which is how callers detect misalignment across aggregated scopes. If no real
|
||||||
|
// data exists in the window both equal 'from' (and a trimmed read returns an
|
||||||
|
// empty slice).
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - from: Start timestamp (Unix seconds)
|
||||||
|
// - to: End timestamp (Unix seconds, exclusive)
|
||||||
|
// - data: Pre-allocated slice to accumulate results (must be large enough)
|
||||||
|
// - trim: If true, cut the result to the real data extent; if false, keep the
|
||||||
|
// full NaN-padded [from, to) window.
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - []schema.Float: Full [from, to) window, or the [dataFrom, dataTo) slice if trim
|
||||||
|
// - int64: dataFrom — timestamp of the first real sample in the window
|
||||||
|
// - int64: dataTo — one frequency past the last real sample
|
||||||
|
// - error: Non-nil on failure
|
||||||
|
//
|
||||||
|
// Panics if 'data' slice is too small to hold all values in [from, to).
|
||||||
|
func (b *buffer) read(from, to int64, data []schema.Float, trim bool) ([]schema.Float, int64, int64, error) {
|
||||||
|
// Walk back toward the buffer that covers 'from'. Do NOT clamp 'from'
|
||||||
|
// forward when it predates the oldest write; those leading points are
|
||||||
|
// NaN-filled below so the full requested window is always available.
|
||||||
|
for from < b.firstWrite() && b.prev != nil {
|
||||||
b = b.prev
|
b = b.prev
|
||||||
}
|
}
|
||||||
|
|
||||||
|
freq := b.frequency // constant across the chain for this metric level
|
||||||
i := 0
|
i := 0
|
||||||
t := from
|
t := from
|
||||||
|
dataFrom, dataTo := from, from // real (non-NaN) data extent within [from, to)
|
||||||
|
haveData := false
|
||||||
for ; t < to; t += b.frequency {
|
for ; t < to; t += b.frequency {
|
||||||
idx := int((t - b.start) / b.frequency)
|
idx := int((t - b.start) / b.frequency)
|
||||||
if idx >= cap(b.data) {
|
if idx >= cap(b.data) {
|
||||||
if b.next == nil {
|
if b.next == nil {
|
||||||
break
|
data[i] += schema.NaN // past the newest buffer; rest is missing
|
||||||
|
i++
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
b = b.next
|
b = b.next
|
||||||
// Recalculate idx in the new buffer; a gap between buffers may exist.
|
// Recalculate idx in the new buffer; a gap between buffers may exist.
|
||||||
idx = int((t - b.start) / b.frequency)
|
idx = int((t - b.start) / b.frequency)
|
||||||
}
|
}
|
||||||
|
|
||||||
if idx >= len(b.data) {
|
if idx >= len(b.data) || t < b.start {
|
||||||
if b.next == nil || to <= b.next.start {
|
// Not yet written, or in the gap before this buffer's first write.
|
||||||
break
|
|
||||||
}
|
|
||||||
data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
|
data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
|
||||||
} else if t < b.start {
|
|
||||||
data[i] += schema.NaN // gap before this buffer's first write
|
|
||||||
} else {
|
} else {
|
||||||
data[i] += b.data[idx]
|
data[i] += b.data[idx]
|
||||||
|
if !haveData {
|
||||||
|
dataFrom = t
|
||||||
|
haveData = true
|
||||||
|
}
|
||||||
|
dataTo = t + b.frequency
|
||||||
}
|
}
|
||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
return data[:i], from, t, nil
|
if trim {
|
||||||
|
// Drop leading/trailing NaN pads so result[0] aligns with dataFrom.
|
||||||
|
front := int((dataFrom - from) / freq)
|
||||||
|
back := int((dataTo - from) / freq)
|
||||||
|
return data[front:back], dataFrom, dataTo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return data[:i], dataFrom, dataTo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// free removes buffers older than the specified timestamp from the chain.
|
// free removes buffers older than the specified timestamp from the chain.
|
||||||
|
|||||||
@@ -298,15 +298,14 @@ func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
|
|||||||
}
|
}
|
||||||
|
|
||||||
data := make([]schema.Float, (to-from)/b.frequency+1)
|
data := make([]schema.Float, (to-from)/b.frequency+1)
|
||||||
data, start, end, err := b.read(from, to, data)
|
|
||||||
|
// trim=true: read() returns only the real data extent, sliced so that
|
||||||
|
// Data[0] aligns with Start (the checkpoint reload contract).
|
||||||
|
data, start, _, err := b.read(from, to, data, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := int((end - start) / b.frequency); i < len(data); i++ {
|
|
||||||
data[i] = schema.NaN
|
|
||||||
}
|
|
||||||
|
|
||||||
retval.Metrics[metric] = &CheckpointMetrics{
|
retval.Metrics[metric] = &CheckpointMetrics{
|
||||||
Frequency: b.frequency,
|
Frequency: b.frequency,
|
||||||
Start: start,
|
Start: start,
|
||||||
@@ -397,9 +396,25 @@ func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
|
|||||||
|
|
||||||
// FromCheckpoint loads checkpoint files from disk into memory in parallel.
|
// FromCheckpoint loads checkpoint files from disk into memory in parallel.
|
||||||
//
|
//
|
||||||
// Uses worker pool to load cluster/host combinations. Returns number of files
|
// Uses worker pool to load cluster/host combinations. Hosts that the
|
||||||
// loaded and any errors.
|
// NodeProvider reports as in use by running jobs load ALL their checkpoint
|
||||||
|
// files (a job that started before `from` needs its full history in memory);
|
||||||
|
// all other hosts load only files satisfying the `from` cutoff. A nil
|
||||||
|
// provider or a provider error falls back to cutoff-only loading with a
|
||||||
|
// warning. Returns number of files loaded and any errors.
|
||||||
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
|
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
|
||||||
|
var usedNodes map[string][]string
|
||||||
|
if m.nodeProvider != nil {
|
||||||
|
un, err := m.nodeProvider.GetUsedNodes(from)
|
||||||
|
if err != nil {
|
||||||
|
cclog.Warnf("[METRICSTORE]> GetUsedNodes failed, loading checkpoints with retention cutoff only: %s", err.Error())
|
||||||
|
} else {
|
||||||
|
usedNodes = un
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cclog.Warnf("[METRICSTORE]> no NodeProvider set, loading checkpoints with retention cutoff only")
|
||||||
|
}
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
work := make(chan [2]string, Keys.NumWorkers*4)
|
work := make(chan [2]string, Keys.NumWorkers*4)
|
||||||
n, errs := int32(0), int32(0)
|
n, errs := int32(0), int32(0)
|
||||||
@@ -409,8 +424,14 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
|
|||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for host := range work {
|
for host := range work {
|
||||||
|
// A host with a running job that started before the cutoff
|
||||||
|
// needs its full history in memory: load all its files.
|
||||||
|
loadFrom := from
|
||||||
|
if isNodeUsed(usedNodes, host[0], host[1]) {
|
||||||
|
loadFrom = 0
|
||||||
|
}
|
||||||
lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
|
lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
|
||||||
nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
|
nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), loadFrom)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
|
cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
|
||||||
atomic.AddInt32(&errs, 1)
|
atomic.AddInt32(&errs, 1)
|
||||||
@@ -479,6 +500,9 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
|
|||||||
next: nil,
|
next: nil,
|
||||||
archived: true,
|
archived: true,
|
||||||
}
|
}
|
||||||
|
// Front-load the single scan so the first stat query hits the cache
|
||||||
|
// instead of lazily rescanning this checkpoint-loaded buffer.
|
||||||
|
b.recomputeStats()
|
||||||
|
|
||||||
minfo, ok := m.Metrics[name]
|
minfo, ok := m.Metrics[name]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
+93
-49
@@ -41,6 +41,7 @@
|
|||||||
package metricstore
|
package metricstore
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
@@ -129,45 +130,6 @@ func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
|
|||||||
return child.findLevelOrCreate(selector[1:], nMetrics)
|
return child.findLevelOrCreate(selector[1:], nMetrics)
|
||||||
}
|
}
|
||||||
|
|
||||||
// collectPaths gathers all selector paths at the specified depth in the tree.
|
|
||||||
//
|
|
||||||
// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
|
|
||||||
// Each path is a selector that can be used with findLevel() or findBuffers().
|
|
||||||
//
|
|
||||||
// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
|
|
||||||
// unintended mutations.
|
|
||||||
//
|
|
||||||
// Parameters:
|
|
||||||
// - currentDepth: Depth of current level (0 = root)
|
|
||||||
// - targetDepth: Depth to collect paths from
|
|
||||||
// - currentPath: Path accumulated so far
|
|
||||||
// - results: Output slice (appended to)
|
|
||||||
//
|
|
||||||
// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
|
|
||||||
// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
|
|
||||||
func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
|
|
||||||
l.lock.RLock()
|
|
||||||
defer l.lock.RUnlock()
|
|
||||||
|
|
||||||
for key, child := range l.children {
|
|
||||||
if child == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
|
|
||||||
newPath := make([]string, len(currentPath))
|
|
||||||
copy(newPath, currentPath)
|
|
||||||
newPath = append(newPath, key)
|
|
||||||
|
|
||||||
// Check depth, and just return if depth reached
|
|
||||||
if currentDepth+1 == targetDepth {
|
|
||||||
*results = append(*results, newPath)
|
|
||||||
} else {
|
|
||||||
child.collectPaths(currentDepth+1, targetDepth, newPath, results)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// free removes buffers older than the retention threshold from the entire subtree.
|
// free removes buffers older than the retention threshold from the entire subtree.
|
||||||
//
|
//
|
||||||
// Recursively frees buffers in this level's metrics and all child levels. Buffers
|
// Recursively frees buffers in this level's metrics and all child levels. Buffers
|
||||||
@@ -233,6 +195,75 @@ func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
|
|||||||
return n, empty, nil
|
return n, empty, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// freeExcludingUsed frees buffers older than t across the whole tree except for
|
||||||
|
// nodes listed in used (cluster name -> sorted hostnames), and deletes node
|
||||||
|
// Levels that become empty. The receiver must be the root level. Returns the
|
||||||
|
// total number of buffers freed.
|
||||||
|
//
|
||||||
|
// This is the exclusion-aware counterpart of freeAndCheckEmpty used by the
|
||||||
|
// retention path when a NodeProvider reports nodes in use by running jobs:
|
||||||
|
// used nodes are never freed, so never empty, so never pruned. Holding the root
|
||||||
|
// write lock for the full pass matches the existing root free (ms.Free(nil, t)).
|
||||||
|
//
|
||||||
|
// Holding the root lock for the entire traversal is intentional: it serializes
|
||||||
|
// this pass against findLevelOrCreate (which RLocks root first), preventing a
|
||||||
|
// writer from grabbing a node pointer that this pass then deletes as empty,
|
||||||
|
// which would silently lose data. Do not "optimize" this to per-cluster locking.
|
||||||
|
func (l *Level) freeExcludingUsed(t int64, used map[string][]string) (int, error) {
|
||||||
|
l.lock.Lock()
|
||||||
|
defer l.lock.Unlock()
|
||||||
|
|
||||||
|
total := 0
|
||||||
|
for cluster, clusterLvl := range l.children {
|
||||||
|
n, empty, err := clusterLvl.freeNodesExcludingUsed(t, used[cluster])
|
||||||
|
total += n
|
||||||
|
if err != nil {
|
||||||
|
return total, err
|
||||||
|
}
|
||||||
|
if empty {
|
||||||
|
delete(l.children, cluster)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// freeNodesExcludingUsed operates on a cluster level. For each node child whose
|
||||||
|
// name is not in usedHosts (sorted, per NodeProvider contract), it frees buffers
|
||||||
|
// older than t and deletes the node when it becomes empty. Returns the number of
|
||||||
|
// buffers freed and whether the cluster level itself is now empty.
|
||||||
|
func (l *Level) freeNodesExcludingUsed(t int64, usedHosts []string) (int, bool, error) {
|
||||||
|
l.lock.Lock()
|
||||||
|
defer l.lock.Unlock()
|
||||||
|
|
||||||
|
n := 0
|
||||||
|
for node, nodeLvl := range l.children {
|
||||||
|
if _, found := slices.BinarySearch(usedHosts, node); found {
|
||||||
|
continue // used node: preserve entirely
|
||||||
|
}
|
||||||
|
m, empty, err := nodeLvl.freeAndCheckEmpty(t)
|
||||||
|
n += m
|
||||||
|
if err != nil {
|
||||||
|
return n, false, err
|
||||||
|
}
|
||||||
|
if empty {
|
||||||
|
delete(l.children, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cluster is empty only if it has no children and no buffers of its own
|
||||||
|
// (inner levels may hold aggregated metrics).
|
||||||
|
empty := len(l.children) == 0
|
||||||
|
if empty {
|
||||||
|
for _, b := range l.metrics {
|
||||||
|
if b != nil {
|
||||||
|
empty = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n, empty, nil
|
||||||
|
}
|
||||||
|
|
||||||
// forceFree removes the oldest buffer from each metric chain in the subtree.
|
// forceFree removes the oldest buffer from each metric chain in the subtree.
|
||||||
//
|
//
|
||||||
// Unlike free(), which removes based on time threshold, this unconditionally removes
|
// Unlike free(), which removes based on time threshold, this unconditionally removes
|
||||||
@@ -350,7 +381,10 @@ func (l *Level) findLevel(selector []string) *Level {
|
|||||||
// Parameters:
|
// Parameters:
|
||||||
// - selector: Pattern to match (consumed recursively)
|
// - selector: Pattern to match (consumed recursively)
|
||||||
// - offset: Metric index in metrics slice (from MetricConfig.offset)
|
// - offset: Metric index in metrics slice (from MetricConfig.offset)
|
||||||
// - f: Callback invoked on each matching buffer
|
// - f: Callback invoked on each matching buffer; path holds the matched
|
||||||
|
// level keys (e.g. ["cluster","node001","cpu0"]) so callers can
|
||||||
|
// identify which node/component the buffer belongs to
|
||||||
|
// - path: Accumulated level keys from the root; pass nil at the top level
|
||||||
//
|
//
|
||||||
// Returns:
|
// Returns:
|
||||||
// - error: First error returned by callback, or nil if all succeeded
|
// - error: First error returned by callback, or nil if all succeeded
|
||||||
@@ -358,19 +392,19 @@ func (l *Level) findLevel(selector []string) *Level {
|
|||||||
// Example:
|
// Example:
|
||||||
//
|
//
|
||||||
// // Find all cpu0 buffers across all hosts:
|
// // Find all cpu0 buffers across all hosts:
|
||||||
// findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
|
// findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback, nil)
|
||||||
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
|
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer, path []string) error, path []string) error {
|
||||||
l.lock.RLock()
|
l.lock.RLock()
|
||||||
defer l.lock.RUnlock()
|
defer l.lock.RUnlock()
|
||||||
|
|
||||||
if len(selector) == 0 {
|
if len(selector) == 0 {
|
||||||
b := l.metrics[offset]
|
b := l.metrics[offset]
|
||||||
if b != nil {
|
if b != nil {
|
||||||
return f(b)
|
return f(b, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, lvl := range l.children {
|
for key, lvl := range l.children {
|
||||||
err := lvl.findBuffers(nil, offset, f)
|
err := lvl.findBuffers(nil, offset, f, appendPath(path, key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -382,7 +416,7 @@ func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer
|
|||||||
if len(sel.String) != 0 && l.children != nil {
|
if len(sel.String) != 0 && l.children != nil {
|
||||||
lvl, ok := l.children[sel.String]
|
lvl, ok := l.children[sel.String]
|
||||||
if ok {
|
if ok {
|
||||||
err := lvl.findBuffers(selector[1:], offset, f)
|
err := lvl.findBuffers(selector[1:], offset, f, appendPath(path, sel.String))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -394,7 +428,7 @@ func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer
|
|||||||
for _, key := range sel.Group {
|
for _, key := range sel.Group {
|
||||||
lvl, ok := l.children[key]
|
lvl, ok := l.children[key]
|
||||||
if ok {
|
if ok {
|
||||||
err := lvl.findBuffers(selector[1:], offset, f)
|
err := lvl.findBuffers(selector[1:], offset, f, appendPath(path, key))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -404,8 +438,8 @@ func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer
|
|||||||
}
|
}
|
||||||
|
|
||||||
if sel.Any && l.children != nil {
|
if sel.Any && l.children != nil {
|
||||||
for _, lvl := range l.children {
|
for key, lvl := range l.children {
|
||||||
if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
|
if err := lvl.findBuffers(selector[1:], offset, f, appendPath(path, key)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -414,3 +448,13 @@ func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// appendPath returns a new slice holding path followed by key. It copies rather
|
||||||
|
// than appending in place so sibling recursions never share/overwrite backing
|
||||||
|
// storage while the tree is walked concurrently.
|
||||||
|
func appendPath(path []string, key string) []string {
|
||||||
|
np := make([]string, len(path)+1)
|
||||||
|
copy(np, path)
|
||||||
|
np[len(path)] = key
|
||||||
|
return np
|
||||||
|
}
|
||||||
|
|||||||
+55
-137
@@ -28,6 +28,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -42,7 +43,6 @@ import (
|
|||||||
type GlobalState struct {
|
type GlobalState struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
lastRetentionTime int64
|
lastRetentionTime int64
|
||||||
selectorsExcluded bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -115,6 +115,9 @@ type MemoryStore struct {
|
|||||||
// Parameters:
|
// Parameters:
|
||||||
// - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
|
// - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
|
||||||
// - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
|
// - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
|
||||||
|
// - provider: NodeProvider consulted during the checkpoint restore (and later by
|
||||||
|
// Free/CleanupCheckpoints) to preserve data for nodes with running jobs; may be
|
||||||
|
// nil, in which case all provider-aware paths fall back to their plain behavior
|
||||||
// - wg: WaitGroup that will be incremented for each background goroutine started
|
// - wg: WaitGroup that will be incremented for each background goroutine started
|
||||||
//
|
//
|
||||||
// The function will call cclog.Fatal on critical errors during initialization.
|
// The function will call cclog.Fatal on critical errors during initialization.
|
||||||
@@ -122,7 +125,7 @@ type MemoryStore struct {
|
|||||||
//
|
//
|
||||||
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
|
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
|
||||||
// receiving termination signals to ensure checkpoint data is persisted.
|
// receiving termination signals to ensure checkpoint data is persisted.
|
||||||
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
|
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, provider NodeProvider, wg *sync.WaitGroup) {
|
||||||
startupTime := time.Now()
|
startupTime := time.Now()
|
||||||
|
|
||||||
if rawConfig != nil {
|
if rawConfig != nil {
|
||||||
@@ -144,6 +147,9 @@ func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.W
|
|||||||
InitMetrics(metrics)
|
InitMetrics(metrics)
|
||||||
|
|
||||||
ms := GetMemoryStore()
|
ms := GetMemoryStore()
|
||||||
|
if provider != nil {
|
||||||
|
ms.SetNodeProvider(provider)
|
||||||
|
}
|
||||||
|
|
||||||
d, err := time.ParseDuration(Keys.RetentionInMemory)
|
d, err := time.ParseDuration(Keys.RetentionInMemory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -251,9 +257,12 @@ func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
|
// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
|
||||||
// This must be called during initialization to provide job state information
|
// The provider supplies the set of nodes in use by running jobs, which is
|
||||||
// for selective buffer retention during Free operations.
|
// consulted by Free (selective buffer retention), FromCheckpoint (full-history
|
||||||
// If not set, the Free function will fall back to freeing all buffers.
|
// loading for used hosts), and CleanupCheckpoints (skipping used hosts).
|
||||||
|
// Server startup passes the provider to Init() directly; this setter serves
|
||||||
|
// callers that do not run Init (tests, the -cleanup-checkpoints CLI path).
|
||||||
|
// If not set, all provider-aware paths fall back to their plain behavior.
|
||||||
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
|
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
|
||||||
ms.nodeProvider = provider
|
ms.nodeProvider = provider
|
||||||
}
|
}
|
||||||
@@ -431,29 +440,7 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
|
|||||||
metricDataGB := ms.SizeInGB()
|
metricDataGB := ms.SizeInGB()
|
||||||
cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
|
cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
|
||||||
|
|
||||||
freedExcluded := 0
|
|
||||||
freedEmergency := 0
|
freedEmergency := 0
|
||||||
var err error
|
|
||||||
|
|
||||||
state.mu.RLock()
|
|
||||||
lastRetention := state.lastRetentionTime
|
|
||||||
selectorsExcluded := state.selectorsExcluded
|
|
||||||
state.mu.RUnlock()
|
|
||||||
|
|
||||||
if lastRetention != 0 && selectorsExcluded {
|
|
||||||
freedExcluded, err = ms.Free(nil, lastRetention)
|
|
||||||
if err != nil {
|
|
||||||
cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if freedExcluded > 0 {
|
|
||||||
debug.FreeOSMemory()
|
|
||||||
cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&mem)
|
|
||||||
actualMemoryGB = float64(mem.Alloc) / 1e9
|
|
||||||
|
|
||||||
if actualMemoryGB > float64(Keys.MemoryCap) {
|
if actualMemoryGB > float64(Keys.MemoryCap) {
|
||||||
cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
|
cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
|
||||||
@@ -547,101 +534,26 @@ func Free(ms *MemoryStore, t time.Time) (int, error) {
|
|||||||
// If the length of the map returned by GetUsedNodes() is 0,
|
// If the length of the map returned by GetUsedNodes() is 0,
|
||||||
// then use default Free method with nil selector
|
// then use default Free method with nil selector
|
||||||
case 0:
|
case 0:
|
||||||
state.selectorsExcluded = false
|
|
||||||
return ms.Free(nil, t.Unix())
|
return ms.Free(nil, t.Unix())
|
||||||
|
|
||||||
// Else formulate selectors, exclude those from the map
|
// Else free every cluster/node except the used ones, pruning node Levels
|
||||||
// and free the rest of the selectors
|
// that become empty in the same locked traversal.
|
||||||
default:
|
default:
|
||||||
state.selectorsExcluded = true
|
return ms.root.freeExcludingUsed(t.Unix(), excludeSelectors)
|
||||||
selectors := GetSelectors(ms, excludeSelectors)
|
|
||||||
return FreeSelected(ms, selectors, t)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FreeSelected frees buffers for specific selectors while preserving others.
|
// isNodeUsed reports whether cluster/host appears in the used-nodes map
|
||||||
//
|
// returned by NodeProvider.GetUsedNodes. Host lists are sorted per the
|
||||||
// This function is used when we want to retain some specific nodes beyond the retention time.
|
// interface contract, so lookup is a binary search. A nil map means no
|
||||||
// It iterates through the provided selectors and frees their associated buffers.
|
// node is in use.
|
||||||
//
|
func isNodeUsed(used map[string][]string, cluster, host string) bool {
|
||||||
// Parameters:
|
hosts, ok := used[cluster]
|
||||||
// - ms: The MemoryStore instance
|
if !ok {
|
||||||
// - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
|
return false
|
||||||
// - t: Time threshold for freeing buffers
|
|
||||||
//
|
|
||||||
// Returns the total number of buffers freed and any error encountered.
|
|
||||||
func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
|
|
||||||
freed := 0
|
|
||||||
|
|
||||||
for _, selector := range selectors {
|
|
||||||
|
|
||||||
freedBuffers, err := ms.Free(selector, t.Unix())
|
|
||||||
if err != nil {
|
|
||||||
cclog.Errorf("error while freeing selected buffers: %#v", err)
|
|
||||||
}
|
}
|
||||||
freed += freedBuffers
|
_, found := slices.BinarySearch(hosts, host)
|
||||||
|
return found
|
||||||
}
|
|
||||||
|
|
||||||
return freed, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
|
|
||||||
//
|
|
||||||
// This function generates a list of selectors whose buffers should be freed by excluding
|
|
||||||
// selectors that correspond to nodes currently in use by running jobs.
|
|
||||||
//
|
|
||||||
// Parameters:
|
|
||||||
// - ms: The MemoryStore instance
|
|
||||||
// - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
|
|
||||||
//
|
|
||||||
// Returns a list of selectors ([]string paths) that can be safely freed.
|
|
||||||
//
|
|
||||||
// Example:
|
|
||||||
//
|
|
||||||
// If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
|
|
||||||
// and excludeSelectors contains {"emmy": ["node001"]},
|
|
||||||
// then only [["emmy", "node002"]] is returned.
|
|
||||||
func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
|
|
||||||
allSelectors := ms.GetPaths(2)
|
|
||||||
|
|
||||||
filteredSelectors := make([][]string, 0, len(allSelectors))
|
|
||||||
|
|
||||||
for _, path := range allSelectors {
|
|
||||||
if len(path) < 2 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
key := path[0] // The "Key" (Level 1)
|
|
||||||
value := path[1] // The "Value" (Level 2)
|
|
||||||
|
|
||||||
exclude := false
|
|
||||||
|
|
||||||
// Check if the key exists in our exclusion map
|
|
||||||
if excludedValues, exists := excludeSelectors[key]; exists {
|
|
||||||
// The key exists, now check if the specific value is in the exclusion list
|
|
||||||
if slices.Contains(excludedValues, value) {
|
|
||||||
exclude = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !exclude {
|
|
||||||
filteredSelectors = append(filteredSelectors, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return filteredSelectors
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetPaths returns a list of lists (paths) to the specified depth.
|
|
||||||
func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
|
|
||||||
var results [][]string
|
|
||||||
|
|
||||||
// Start recursion. Initial path is empty.
|
|
||||||
// We treat Root as depth 0.
|
|
||||||
ms.root.collectPaths(0, targetDepth, []string{}, &results)
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write all values in `metrics` to the level specified by `selector` for time `ts`.
|
// Write all values in `metrics` to the level specified by `selector` for time `ts`.
|
||||||
@@ -701,7 +613,7 @@ func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metric
|
|||||||
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
|
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
|
||||||
// The second and third return value are the actual from/to for the data. Those can be different from
|
// The second and third return value are the actual from/to for the data. Those can be different from
|
||||||
// the range asked for if no data was available.
|
// the range asked for if no data was available.
|
||||||
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
|
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64, resampleAlgo string) ([]schema.Float, int64, int64, int64, error) {
|
||||||
if from > to {
|
if from > to {
|
||||||
return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
|
return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
|
||||||
}
|
}
|
||||||
@@ -711,38 +623,40 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
|
|||||||
return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
|
return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// data spans the full requested window; every scope's read() writes into the
|
||||||
|
// same index-aligned slice (NaN where it has no value), so aggregation never
|
||||||
|
// needs trimming. We check each buffer's extent against the original request
|
||||||
|
// and warn if it doesn't cover the full range.
|
||||||
n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
|
n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
|
||||||
|
|
||||||
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
|
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer, path []string) error {
|
||||||
cdata, cfrom, cto, err := b.read(from, to, data)
|
cdata, cfrom, cto, err := b.read(from, to, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if n == 0 {
|
// Check if buffer covers full requested range
|
||||||
from, to = cfrom, cto
|
missingfront := int((cfrom - from) / minfo.Frequency)
|
||||||
} else if from != cfrom || to != cto || len(data) != len(cdata) {
|
missingback := int((to - cto) / minfo.Frequency)
|
||||||
missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
|
if missingfront > 0 || missingback > 0 {
|
||||||
if missingfront != 0 {
|
node := strings.Join(path, "/")
|
||||||
return ErrDataDoesNotAlignMissingFront
|
switch {
|
||||||
|
case missingfront > 0 && missingback > 0:
|
||||||
|
cclog.Warnf("%v: metric=%s node=%s requested[%d,%d] actual[%d,%d] missing_front=%d pts missing_back=%d pts",
|
||||||
|
ErrDataDoesNotAlignMissingFront, metric, node, from, to, cfrom, cto, missingfront, missingback)
|
||||||
|
case missingfront > 0:
|
||||||
|
cclog.Warnf("%v: metric=%s node=%s requested[%d,%d] actual[%d,%d] missing_front=%d pts",
|
||||||
|
ErrDataDoesNotAlignMissingFront, metric, node, from, to, cfrom, cto, missingfront)
|
||||||
|
case missingback > 0:
|
||||||
|
cclog.Warnf("%v: metric=%s node=%s requested[%d,%d] actual[%d,%d] missing_back=%d pts",
|
||||||
|
ErrDataDoesNotAlignMissingBack, metric, node, from, to, cfrom, cto, missingback)
|
||||||
}
|
}
|
||||||
|
|
||||||
newlen := len(cdata) - missingback
|
|
||||||
if newlen < 1 {
|
|
||||||
return ErrDataDoesNotAlignMissingBack
|
|
||||||
}
|
|
||||||
cdata = cdata[0:newlen]
|
|
||||||
if len(cdata) != len(data) {
|
|
||||||
return ErrDataDoesNotAlignDataLenMismatch
|
|
||||||
}
|
|
||||||
|
|
||||||
from, to = cfrom, cto
|
|
||||||
}
|
}
|
||||||
|
|
||||||
data = cdata
|
data = cdata
|
||||||
n += 1
|
n += 1
|
||||||
return nil
|
return nil
|
||||||
})
|
}, nil)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, 0, 0, err
|
return nil, 0, 0, 0, err
|
||||||
@@ -759,7 +673,11 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
|
resampleFn, rfErr := resampler.GetResampler(resampleAlgo)
|
||||||
|
if rfErr != nil {
|
||||||
|
return nil, 0, 0, 0, rfErr
|
||||||
|
}
|
||||||
|
data, resolution, err = resampleFn(data, minfo.Frequency, resolution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, 0, 0, err
|
return nil, 0, 0, 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -141,23 +141,35 @@ func TestBufferWriteOverwrite(t *testing.T) {
|
|||||||
|
|
||||||
// ─── Buffer read ──────────────────────────────────────────────────────────────
|
// ─── Buffer read ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// TestBufferReadBeforeFirstWrite verifies that 'from' is clamped to firstWrite
|
// TestBufferReadBeforeFirstWrite verifies that a range starting before any data
|
||||||
// when the requested range starts before any data in the chain.
|
// in the chain returns the full requested window with the leading, unwritten
|
||||||
|
// points NaN-filled, and reports the real data extent (dataFrom) via the second
|
||||||
|
// return value (rather than clamping 'from' forward and truncating).
|
||||||
func TestBufferReadBeforeFirstWrite(t *testing.T) {
|
func TestBufferReadBeforeFirstWrite(t *testing.T) {
|
||||||
b := newBuffer(100, 10) // firstWrite = 100
|
b := newBuffer(100, 10) // firstWrite = 100
|
||||||
b.write(100, schema.Float(1.0))
|
b.write(100, schema.Float(1.0))
|
||||||
b.write(110, schema.Float(2.0))
|
b.write(110, schema.Float(2.0))
|
||||||
|
|
||||||
data := make([]schema.Float, 10)
|
data := make([]schema.Float, 10)
|
||||||
result, adjustedFrom, _, err := b.read(50, 120, data)
|
result, dataFrom, dataTo, err := b.read(50, 120, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read() error = %v", err)
|
t.Fatalf("read() error = %v", err)
|
||||||
}
|
}
|
||||||
if adjustedFrom != 100 {
|
// Real data extent, not the requested window: first sample at 100, last at 110.
|
||||||
t.Errorf("adjustedFrom = %d, want 100 (clamped to firstWrite)", adjustedFrom)
|
if dataFrom != 100 || dataTo != 120 {
|
||||||
|
t.Errorf("dataFrom/dataTo = %d/%d, want 100/120", dataFrom, dataTo)
|
||||||
}
|
}
|
||||||
if len(result) != 2 {
|
// Full window [50,120) step 10 = 7 points; first 5 predate firstWrite=100 -> NaN.
|
||||||
t.Errorf("len(result) = %d, want 2", len(result))
|
if len(result) != 7 {
|
||||||
|
t.Fatalf("len(result) = %d, want 7", len(result))
|
||||||
|
}
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if !result[i].IsNaN() {
|
||||||
|
t.Errorf("result[%d] = %v, want NaN", i, result[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result[5] != 1.0 || result[6] != 2.0 {
|
||||||
|
t.Errorf("result[5:7] = %v, want [1 2]", result[5:7])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,7 +189,7 @@ func TestBufferReadChain(t *testing.T) {
|
|||||||
b1.next = b2
|
b1.next = b2
|
||||||
|
|
||||||
data := make([]schema.Float, 6)
|
data := make([]schema.Float, 6)
|
||||||
result, from, to, err := b2.read(100, 160, data)
|
result, from, to, err := b2.read(100, 160, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read() error = %v", err)
|
t.Fatalf("read() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -219,7 +231,7 @@ func TestBufferReadIdxAfterSwitch(t *testing.T) {
|
|||||||
// from=0 triggers the walkback to b1 (from < b2.firstWrite=5).
|
// from=0 triggers the walkback to b1 (from < b2.firstWrite=5).
|
||||||
// After clamping, the loop runs t=5,15,25,35.
|
// After clamping, the loop runs t=5,15,25,35.
|
||||||
data := make([]schema.Float, 4)
|
data := make([]schema.Float, 4)
|
||||||
result, _, _, err := b2.read(0, 36, data)
|
result, _, _, err := b2.read(0, 36, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read() error = %v", err)
|
t.Fatalf("read() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -247,7 +259,7 @@ func TestBufferReadNaNValues(t *testing.T) {
|
|||||||
b.write(120, schema.Float(3.0))
|
b.write(120, schema.Float(3.0))
|
||||||
|
|
||||||
data := make([]schema.Float, 3)
|
data := make([]schema.Float, 3)
|
||||||
result, _, _, err := b.read(100, 130, data)
|
result, _, _, err := b.read(100, 130, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read() error = %v", err)
|
t.Fatalf("read() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -274,7 +286,7 @@ func TestBufferReadAccumulation(t *testing.T) {
|
|||||||
|
|
||||||
// Pre-populate data slice (simulates a second metric being summed in).
|
// Pre-populate data slice (simulates a second metric being summed in).
|
||||||
data := []schema.Float{2.0, 1.0, 0.0}
|
data := []schema.Float{2.0, 1.0, 0.0}
|
||||||
result, _, _, err := b.read(100, 120, data)
|
result, _, _, err := b.read(100, 120, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read() error = %v", err)
|
t.Fatalf("read() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -577,7 +589,7 @@ func TestBufferRead(t *testing.T) {
|
|||||||
|
|
||||||
// Read data
|
// Read data
|
||||||
data := make([]schema.Float, 3)
|
data := make([]schema.Float, 3)
|
||||||
result, from, to, err := b.read(100, 130, data)
|
result, from, to, err := b.read(100, 130, data, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("buffer.read() error = %v", err)
|
t.Errorf("buffer.read() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,264 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package metricstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeNodeProvider implements NodeProvider for tests.
|
||||||
|
type fakeNodeProvider struct {
|
||||||
|
nodes map[string][]string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeNodeProvider) GetUsedNodes(ts int64) (map[string][]string, error) {
|
||||||
|
return f.nodes, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
var errTestProvider = errors.New("provider failure")
|
||||||
|
|
||||||
|
func TestIsNodeUsed(t *testing.T) {
|
||||||
|
used := map[string][]string{"fritz": {"node001", "node003"}}
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
cluster, host string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"fritz", "node001", true},
|
||||||
|
{"fritz", "node003", true},
|
||||||
|
{"fritz", "node002", false},
|
||||||
|
{"alex", "node001", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := isNodeUsed(used, c.cluster, c.host); got != c.want {
|
||||||
|
t.Errorf("isNodeUsed(%q, %q) = %v, want %v", c.cluster, c.host, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if isNodeUsed(nil, "fritz", "node001") {
|
||||||
|
t.Error("nil map must report false")
|
||||||
|
}
|
||||||
|
if isNodeUsed(map[string][]string{}, "fritz", "node001") {
|
||||||
|
t.Error("empty map must report false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTestStore builds a minimal MemoryStore without touching the singleton.
|
||||||
|
func newTestStore() *MemoryStore {
|
||||||
|
return &MemoryStore{
|
||||||
|
Metrics: map[string]MetricConfig{
|
||||||
|
"cpu_load": {Frequency: 60, offset: 0},
|
||||||
|
},
|
||||||
|
root: Level{
|
||||||
|
metrics: make([]*buffer, 1),
|
||||||
|
children: make(map[string]*Level),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeTestCheckpoint writes a valid JSON checkpoint file <ts>.json into dir.
|
||||||
|
func writeTestCheckpoint(t *testing.T, dir string, ts int64) {
|
||||||
|
t.Helper()
|
||||||
|
cf := &CheckpointFile{
|
||||||
|
From: ts,
|
||||||
|
To: ts + 120,
|
||||||
|
Metrics: map[string]*CheckpointMetrics{
|
||||||
|
"cpu_load": {Frequency: 60, Start: ts, Data: []schema.Float{1.0, 2.0, 3.0}},
|
||||||
|
},
|
||||||
|
Children: make(map[string]*CheckpointFile),
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(cf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("%d.json", ts)), data, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeThreeCheckpointsPerHost creates ts 1000/2000/5000 for node001+node002.
|
||||||
|
func writeThreeCheckpointsPerHost(t *testing.T, dir string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, host := range []string{"node001", "node002"} {
|
||||||
|
hostDir := filepath.Join(dir, "fritz", host)
|
||||||
|
writeTestCheckpoint(t, hostDir, 1000)
|
||||||
|
writeTestCheckpoint(t, hostDir, 2000)
|
||||||
|
writeTestCheckpoint(t, hostDir, 5000)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFromCheckpointLoadsAllFilesForUsedNodes(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeThreeCheckpointsPerHost(t, dir)
|
||||||
|
|
||||||
|
ms := newTestStore()
|
||||||
|
ms.SetNodeProvider(&fakeNodeProvider{nodes: map[string][]string{"fritz": {"node001"}}})
|
||||||
|
|
||||||
|
n, err := ms.FromCheckpoint(dir, 3000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// node001 (used): all 3 files. node002: bridge 2000.json + 5000.json = 2.
|
||||||
|
if n != 5 {
|
||||||
|
t.Fatalf("expected 5 files loaded, got %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFromCheckpointWithoutProviderKeepsCutoff(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeThreeCheckpointsPerHost(t, dir)
|
||||||
|
|
||||||
|
ms := newTestStore()
|
||||||
|
|
||||||
|
n, err := ms.FromCheckpoint(dir, 3000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 2 files per host, no provider set.
|
||||||
|
if n != 4 {
|
||||||
|
t.Fatalf("expected 4 files loaded, got %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFromCheckpointProviderErrorFallsBack(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeThreeCheckpointsPerHost(t, dir)
|
||||||
|
|
||||||
|
ms := newTestStore()
|
||||||
|
ms.SetNodeProvider(&fakeNodeProvider{err: errTestProvider})
|
||||||
|
|
||||||
|
n, err := ms.FromCheckpoint(dir, 3000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 4 {
|
||||||
|
t.Fatalf("expected fallback to cutoff load (4 files), got %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteCheckpointsSkipsUsedNodes(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeTestCheckpoint(t, filepath.Join(dir, "fritz", "node001"), 1000)
|
||||||
|
writeTestCheckpoint(t, filepath.Join(dir, "fritz", "node002"), 1000)
|
||||||
|
|
||||||
|
used := map[string][]string{"fritz": {"node001"}}
|
||||||
|
n, err := deleteCheckpoints(dir, 3000, used)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("expected 1 file deleted, got %d", n)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "fritz", "node001", "1000.json")); err != nil {
|
||||||
|
t.Errorf("used node's checkpoint must survive: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "fritz", "node002", "1000.json")); !os.IsNotExist(err) {
|
||||||
|
t.Error("unused node's checkpoint must be deleted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveCheckpointsSkipsUsedNodes(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
cpDir := t.TempDir()
|
||||||
|
archiveDir := t.TempDir()
|
||||||
|
writeTestCheckpoint(t, filepath.Join(cpDir, "fritz", "node001"), 1000)
|
||||||
|
writeTestCheckpoint(t, filepath.Join(cpDir, "fritz", "node002"), 1000)
|
||||||
|
|
||||||
|
used := map[string][]string{"fritz": {"node001"}}
|
||||||
|
n, err := archiveCheckpoints(cpDir, archiveDir, 3000, used)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("expected 1 file archived, got %d", n)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(cpDir, "fritz", "node001", "1000.json")); err != nil {
|
||||||
|
t.Errorf("used node's checkpoint must survive: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(cpDir, "fritz", "node002", "1000.json")); !os.IsNotExist(err) {
|
||||||
|
t.Error("unused node's checkpoint must be removed after archiving")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(archiveDir, "fritz", "3000.parquet")); err != nil {
|
||||||
|
t.Errorf("parquet archive must exist: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanupCheckpointsAbortsOnProviderError(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
oldMS := msInstance
|
||||||
|
msInstance = newTestStore()
|
||||||
|
msInstance.SetNodeProvider(&fakeNodeProvider{err: errTestProvider})
|
||||||
|
t.Cleanup(func() { msInstance = oldMS })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeTestCheckpoint(t, filepath.Join(dir, "fritz", "node001"), 1000)
|
||||||
|
|
||||||
|
if _, err := CleanupCheckpoints(dir, "", 3000, true); err == nil {
|
||||||
|
t.Fatal("expected error when GetUsedNodes fails")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "fritz", "node001", "1000.json")); err != nil {
|
||||||
|
t.Errorf("no files may be deleted when provider errors: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanupCheckpointsUsedNodesSurvive(t *testing.T) {
|
||||||
|
oldWorkers := Keys.NumWorkers
|
||||||
|
Keys.NumWorkers = 2
|
||||||
|
t.Cleanup(func() { Keys.NumWorkers = oldWorkers })
|
||||||
|
|
||||||
|
oldMS := msInstance
|
||||||
|
msInstance = newTestStore()
|
||||||
|
msInstance.SetNodeProvider(&fakeNodeProvider{nodes: map[string][]string{"fritz": {"node001"}}})
|
||||||
|
t.Cleanup(func() { msInstance = oldMS })
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeTestCheckpoint(t, filepath.Join(dir, "fritz", "node001"), 1000)
|
||||||
|
writeTestCheckpoint(t, filepath.Join(dir, "fritz", "node002"), 1000)
|
||||||
|
|
||||||
|
n, err := CleanupCheckpoints(dir, "", 3000, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("expected 1 file deleted, got %d", n)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "fritz", "node001", "1000.json")); err != nil {
|
||||||
|
t.Errorf("used node's checkpoint must survive: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package metricstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// writeNode creates cluster/node (and optional deeper sub-levels) and fills
|
||||||
|
// every metric buffer with points from startTs to endTs (inclusive) at the
|
||||||
|
// store's metric frequency. subPath appends levels below the node (e.g.
|
||||||
|
// {"socket0"}); pass nil to write buffers directly on the node level.
|
||||||
|
func writeNode(ms *MemoryStore, cluster, node string, subPath []string, freq, startTs, endTs int64) {
|
||||||
|
selector := append([]string{cluster, node}, subPath...)
|
||||||
|
lvl := ms.root.findLevelOrCreate(selector, len(ms.Metrics))
|
||||||
|
for i := range lvl.metrics {
|
||||||
|
lvl.metrics[i] = newBuffer(startTs, freq)
|
||||||
|
for ts := startTs; ts <= endTs; ts += freq {
|
||||||
|
lvl.metrics[i].write(ts, schema.Float(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasNode reports whether [cluster, node] still exists in the tree.
|
||||||
|
func hasNode(ms *MemoryStore, cluster, node string) bool {
|
||||||
|
return slices.Contains(ms.ListChildren([]string{cluster}), node)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeExcludingUsedPrunesDeadNode(t *testing.T) {
|
||||||
|
ms := newTestStore() // one metric "cpu_load", freq 60
|
||||||
|
const freq, thr = int64(60), int64(100000)
|
||||||
|
|
||||||
|
// dead: last data ~2000, well below threshold -> emptied -> pruned
|
||||||
|
writeNode(ms, "fritz", "dead", nil, freq, 1000, 2000)
|
||||||
|
// alive: data around/after threshold -> stays
|
||||||
|
writeNode(ms, "fritz", "alive", nil, freq, thr, thr+600)
|
||||||
|
|
||||||
|
freed, err := ms.root.freeExcludingUsed(thr, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("freeExcludingUsed: %v", err)
|
||||||
|
}
|
||||||
|
if freed == 0 {
|
||||||
|
t.Fatal("expected at least one buffer freed")
|
||||||
|
}
|
||||||
|
if hasNode(ms, "fritz", "dead") {
|
||||||
|
t.Error("dead node must be pruned from the tree")
|
||||||
|
}
|
||||||
|
if !hasNode(ms, "fritz", "alive") {
|
||||||
|
t.Error("alive node must be preserved")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeExcludingUsedPreservesUsedNode(t *testing.T) {
|
||||||
|
ms := newTestStore()
|
||||||
|
const freq, thr = int64(60), int64(100000)
|
||||||
|
|
||||||
|
// both would be dead by timestamp, but "used" is excluded
|
||||||
|
writeNode(ms, "fritz", "used", nil, freq, 1000, 2000)
|
||||||
|
writeNode(ms, "fritz", "gone", nil, freq, 1000, 2000)
|
||||||
|
|
||||||
|
used := map[string][]string{"fritz": {"used"}} // sorted hostnames
|
||||||
|
|
||||||
|
if _, err := ms.root.freeExcludingUsed(thr, used); err != nil {
|
||||||
|
t.Fatalf("freeExcludingUsed: %v", err)
|
||||||
|
}
|
||||||
|
if !hasNode(ms, "fritz", "used") {
|
||||||
|
t.Error("used node must be preserved even when stale")
|
||||||
|
}
|
||||||
|
if hasNode(ms, "fritz", "gone") {
|
||||||
|
t.Error("non-used dead node must be pruned")
|
||||||
|
}
|
||||||
|
// used node still holds its buffer
|
||||||
|
lvl := ms.root.findLevel([]string{"fritz", "used"})
|
||||||
|
if lvl == nil || lvl.metrics[0] == nil {
|
||||||
|
t.Error("used node must keep its buffers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeExcludingUsedPrunesSubLevelNode(t *testing.T) {
|
||||||
|
ms := newTestStore()
|
||||||
|
const freq, thr = int64(60), int64(100000)
|
||||||
|
|
||||||
|
// node holds no direct buffers; only a socket0 child, all stale
|
||||||
|
writeNode(ms, "fritz", "deep", []string{"socket0"}, freq, 1000, 2000)
|
||||||
|
|
||||||
|
if _, err := ms.root.freeExcludingUsed(thr, nil); err != nil {
|
||||||
|
t.Fatalf("freeExcludingUsed: %v", err)
|
||||||
|
}
|
||||||
|
if hasNode(ms, "fritz", "deep") {
|
||||||
|
t.Error("node must be pruned once all descendant buffers are freed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeViaProviderPrunesDeadNodes(t *testing.T) {
|
||||||
|
ms := newTestStore()
|
||||||
|
const freq int64 = 60
|
||||||
|
thr := time.Unix(100000, 0)
|
||||||
|
|
||||||
|
writeNode(ms, "fritz", "dead", nil, freq, 1000, 2000) // stale, not used -> pruned
|
||||||
|
writeNode(ms, "fritz", "busy", nil, freq, 1000, 2000) // stale, but used -> kept
|
||||||
|
writeNode(ms, "fritz", "live", nil, freq, 100000, 100600) // fresh -> kept
|
||||||
|
|
||||||
|
ms.SetNodeProvider(&fakeNodeProvider{nodes: map[string][]string{"fritz": {"busy"}}})
|
||||||
|
|
||||||
|
if _, err := Free(ms, thr); err != nil {
|
||||||
|
t.Fatalf("Free: %v", err)
|
||||||
|
}
|
||||||
|
if hasNode(ms, "fritz", "dead") {
|
||||||
|
t.Error("stale non-used node must be pruned")
|
||||||
|
}
|
||||||
|
if !hasNode(ms, "fritz", "busy") {
|
||||||
|
t.Error("used node must survive")
|
||||||
|
}
|
||||||
|
if !hasNode(ms, "fritz", "live") {
|
||||||
|
t.Error("fresh node must survive")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFreeExcludingUsedConcurrentReadNoRace(t *testing.T) {
|
||||||
|
ms := newTestStore()
|
||||||
|
const freq, thr = int64(60), int64(100000)
|
||||||
|
for _, n := range []string{"a", "b", "c", "d"} {
|
||||||
|
writeNode(ms, "fritz", n, nil, freq, 1000, 2000)
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
stop := make(chan struct{})
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
_ = ms.ListChildren([]string{"fritz"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := ms.root.freeExcludingUsed(thr, nil); err != nil {
|
||||||
|
t.Fatalf("freeExcludingUsed: %v", err)
|
||||||
|
}
|
||||||
|
close(stop)
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
+35
-31
@@ -50,7 +50,7 @@ func (ccms *InternalMetricStore) HealthCheck(cluster string,
|
|||||||
|
|
||||||
// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
|
// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
|
||||||
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
|
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
|
||||||
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
|
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, resampleAlgo string) (schema.JobData, error)
|
||||||
|
|
||||||
// LoadData loads metric data for a specific job with automatic scope transformation.
|
// LoadData loads metric data for a specific job with automatic scope transformation.
|
||||||
//
|
//
|
||||||
@@ -81,15 +81,16 @@ func (ccms *InternalMetricStore) LoadData(
|
|||||||
scopes []schema.MetricScope,
|
scopes []schema.MetricScope,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
resolution int,
|
resolution int,
|
||||||
|
resampleAlgo string,
|
||||||
) (schema.JobData, error) {
|
) (schema.JobData, error) {
|
||||||
if TestLoadDataCallback != nil {
|
if TestLoadDataCallback != nil {
|
||||||
return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
|
return TestLoadDataCallback(job, metrics, scopes, ctx, resolution, resampleAlgo)
|
||||||
}
|
}
|
||||||
|
|
||||||
queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
|
queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify assignment is correct - log any inconsistencies for debugging
|
// Verify assignment is correct - log any inconsistencies for debugging
|
||||||
@@ -105,16 +106,17 @@ func (ccms *InternalMetricStore) LoadData(
|
|||||||
Queries: queries,
|
Queries: queries,
|
||||||
WithStats: true,
|
WithStats: true,
|
||||||
WithData: true,
|
WithData: true,
|
||||||
|
ResampleAlgo: resampleAlgo,
|
||||||
}
|
}
|
||||||
|
|
||||||
resBody, err := FetchData(req)
|
resBody, err := FetchData(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while fetching data : %s", err.Error())
|
cclog.Errorf("Error while fetching data : %s", err.Error())
|
||||||
return nil, err
|
return schema.JobData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var errors []string
|
var errors []string
|
||||||
jobData := make(schema.JobData)
|
jobData := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
|
||||||
|
|
||||||
// Add safety check for potential index out of range errors
|
// Add safety check for potential index out of range errors
|
||||||
if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
|
if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
|
||||||
@@ -139,8 +141,8 @@ func (ccms *InternalMetricStore) LoadData(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := jobData[metric]; !ok {
|
if _, ok := jobData.Metrics[metric]; !ok {
|
||||||
jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
|
jobData.Metrics[metric] = make(schema.ScopedMetrics)
|
||||||
}
|
}
|
||||||
|
|
||||||
res := mc.Timestep
|
res := mc.Timestep
|
||||||
@@ -148,14 +150,14 @@ func (ccms *InternalMetricStore) LoadData(
|
|||||||
res = int(row[0].Resolution)
|
res = int(row[0].Resolution)
|
||||||
}
|
}
|
||||||
|
|
||||||
jobMetric, ok := jobData[metric][scope]
|
jobMetric, ok := jobData.Metrics[metric][scope]
|
||||||
if !ok {
|
if !ok {
|
||||||
jobMetric = &schema.JobMetric{
|
jobMetric = &schema.JobMetric{
|
||||||
Unit: mc.Unit,
|
Unit: mc.Unit,
|
||||||
Timestep: res,
|
Timestep: res,
|
||||||
Series: make([]schema.Series, 0),
|
Series: make([]schema.Series, 0),
|
||||||
}
|
}
|
||||||
jobData[metric][scope] = jobMetric
|
jobData.Metrics[metric][scope] = jobMetric
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
@@ -181,11 +183,11 @@ func (ccms *InternalMetricStore) LoadData(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// So that one can later check len(jobData):
|
// So that one can later check len(jobData.Metrics):
|
||||||
if len(jobMetric.Series) == 0 {
|
if len(jobMetric.Series) == 0 {
|
||||||
delete(jobData[metric], scope)
|
delete(jobData.Metrics[metric], scope)
|
||||||
if len(jobData[metric]) == 0 {
|
if len(jobData.Metrics[metric]) == 0 {
|
||||||
delete(jobData, metric)
|
delete(jobData.Metrics, metric)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -407,7 +409,7 @@ func (ccms *InternalMetricStore) LoadScopedStats(
|
|||||||
queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
|
queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req := APIQueryRequest{
|
req := APIQueryRequest{
|
||||||
@@ -422,11 +424,11 @@ func (ccms *InternalMetricStore) LoadScopedStats(
|
|||||||
resBody, err := FetchData(req)
|
resBody, err := FetchData(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cclog.Errorf("Error while fetching data : %s", err.Error())
|
cclog.Errorf("Error while fetching data : %s", err.Error())
|
||||||
return nil, err
|
return schema.ScopedJobStats{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var errors []string
|
var errors []string
|
||||||
scopedJobStats := make(schema.ScopedJobStats)
|
scopedJobStats := schema.ScopedJobStats{Metrics: make(map[string]schema.ScopedMetricStats)}
|
||||||
|
|
||||||
for i, row := range resBody.Results {
|
for i, row := range resBody.Results {
|
||||||
if len(row) == 0 {
|
if len(row) == 0 {
|
||||||
@@ -437,12 +439,12 @@ func (ccms *InternalMetricStore) LoadScopedStats(
|
|||||||
metric := query.Metric
|
metric := query.Metric
|
||||||
scope := assignedScope[i]
|
scope := assignedScope[i]
|
||||||
|
|
||||||
if _, ok := scopedJobStats[metric]; !ok {
|
if _, ok := scopedJobStats.Metrics[metric]; !ok {
|
||||||
scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
|
scopedJobStats.Metrics[metric] = make(schema.ScopedMetricStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := scopedJobStats[metric][scope]; !ok {
|
if _, ok := scopedJobStats.Metrics[metric][scope]; !ok {
|
||||||
scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
|
scopedJobStats.Metrics[metric][scope] = make([]*schema.ScopedStats, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
@@ -456,7 +458,7 @@ func (ccms *InternalMetricStore) LoadScopedStats(
|
|||||||
|
|
||||||
SanitizeStats(&res.Avg, &res.Min, &res.Max)
|
SanitizeStats(&res.Avg, &res.Min, &res.Max)
|
||||||
|
|
||||||
scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
|
scopedJobStats.Metrics[metric][scope] = append(scopedJobStats.Metrics[metric][scope], &schema.ScopedStats{
|
||||||
Hostname: query.Hostname,
|
Hostname: query.Hostname,
|
||||||
ID: id,
|
ID: id,
|
||||||
Data: &schema.MetricStatistics{
|
Data: &schema.MetricStatistics{
|
||||||
@@ -467,11 +469,11 @@ func (ccms *InternalMetricStore) LoadScopedStats(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
|
// So that one can later check len(scopedJobStats.Metrics[metric][scope]): Remove from map if empty
|
||||||
if len(scopedJobStats[metric][scope]) == 0 {
|
if len(scopedJobStats.Metrics[metric][scope]) == 0 {
|
||||||
delete(scopedJobStats[metric], scope)
|
delete(scopedJobStats.Metrics[metric], scope)
|
||||||
if len(scopedJobStats[metric]) == 0 {
|
if len(scopedJobStats.Metrics[metric]) == 0 {
|
||||||
delete(scopedJobStats, metric)
|
delete(scopedJobStats.Metrics, metric)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -621,6 +623,7 @@ func (ccms *InternalMetricStore) LoadNodeListData(
|
|||||||
resolution int,
|
resolution int,
|
||||||
from, to time.Time,
|
from, to time.Time,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
resampleAlgo string,
|
||||||
) (map[string]schema.JobData, error) {
|
) (map[string]schema.JobData, error) {
|
||||||
// Note: Order of node data is not guaranteed after this point
|
// Note: Order of node data is not guaranteed after this point
|
||||||
queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
|
queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
|
||||||
@@ -642,6 +645,7 @@ func (ccms *InternalMetricStore) LoadNodeListData(
|
|||||||
To: to.Unix(),
|
To: to.Unix(),
|
||||||
WithStats: true,
|
WithStats: true,
|
||||||
WithData: true,
|
WithData: true,
|
||||||
|
ResampleAlgo: resampleAlgo,
|
||||||
}
|
}
|
||||||
|
|
||||||
resBody, err := FetchData(req)
|
resBody, err := FetchData(req)
|
||||||
@@ -695,14 +699,14 @@ func (ccms *InternalMetricStore) LoadNodeListData(
|
|||||||
// Init Nested Map Data Structures If Not Found
|
// Init Nested Map Data Structures If Not Found
|
||||||
hostData, ok := data[query.Hostname]
|
hostData, ok := data[query.Hostname]
|
||||||
if !ok {
|
if !ok {
|
||||||
hostData = make(schema.JobData)
|
hostData = schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
|
||||||
data[query.Hostname] = hostData
|
data[query.Hostname] = hostData
|
||||||
}
|
}
|
||||||
|
|
||||||
metricData, ok := hostData[metric]
|
metricData, ok := hostData.Metrics[metric]
|
||||||
if !ok {
|
if !ok {
|
||||||
metricData = make(map[schema.MetricScope]*schema.JobMetric)
|
metricData = make(schema.ScopedMetrics)
|
||||||
data[query.Hostname][metric] = metricData
|
hostData.Metrics[metric] = metricData
|
||||||
}
|
}
|
||||||
|
|
||||||
scopeData, ok := metricData[scope]
|
scopeData, ok := metricData[scope]
|
||||||
@@ -712,7 +716,7 @@ func (ccms *InternalMetricStore) LoadNodeListData(
|
|||||||
Timestep: res,
|
Timestep: res,
|
||||||
Series: make([]schema.Series, 0),
|
Series: make([]schema.Series, 0),
|
||||||
}
|
}
|
||||||
data[query.Hostname][metric][scope] = scopeData
|
metricData[scope] = scopeData
|
||||||
}
|
}
|
||||||
|
|
||||||
for ndx, res := range row {
|
for ndx, res := range row {
|
||||||
|
|||||||
+64
-10
@@ -7,7 +7,9 @@ package metricstore
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
"github.com/ClusterCockpit/cc-lib/v2/util"
|
"github.com/ClusterCockpit/cc-lib/v2/util"
|
||||||
@@ -20,6 +22,33 @@ type Stats struct {
|
|||||||
Max schema.Float
|
Max schema.Float
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recomputeStats rebuilds the buffer's running statistics from a single full
|
||||||
|
// scan of its data and marks them valid. Used after an overwrite invalidated
|
||||||
|
// the incremental aggregate, and at checkpoint load time.
|
||||||
|
func (b *buffer) recomputeStats() {
|
||||||
|
sum, samples := 0.0, 0
|
||||||
|
min, max := math.MaxFloat32, -math.MaxFloat32
|
||||||
|
for _, v := range b.data {
|
||||||
|
xf := float64(v)
|
||||||
|
if math.IsNaN(xf) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
samples++
|
||||||
|
sum += xf
|
||||||
|
if xf < min {
|
||||||
|
min = xf
|
||||||
|
}
|
||||||
|
if xf > max {
|
||||||
|
max = xf
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.statSum = sum
|
||||||
|
b.statSamples = samples
|
||||||
|
b.statMin = min
|
||||||
|
b.statMax = max
|
||||||
|
b.statsValid = true
|
||||||
|
}
|
||||||
|
|
||||||
func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
||||||
if from < b.start {
|
if from < b.start {
|
||||||
if b.prev != nil {
|
if b.prev != nil {
|
||||||
@@ -28,9 +57,6 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
|||||||
from = b.start
|
from = b.start
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Check if b.closed and if so and the full buffer is queried,
|
|
||||||
// use b.statistics instead of iterating over the buffer.
|
|
||||||
|
|
||||||
samples := 0
|
samples := 0
|
||||||
sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
||||||
|
|
||||||
@@ -45,6 +71,28 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
|||||||
idx = int((t - b.start) / b.frequency)
|
idx = int((t - b.start) / b.frequency)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fast path: standing at this buffer's first data point (idx 0) with the
|
||||||
|
// whole buffer inside [from, to). Fold in the cached aggregate and jump
|
||||||
|
// past the buffer's real data instead of scanning each point. Any slots
|
||||||
|
// between len(data) and cap are handled as gaps by the normal loop after
|
||||||
|
// t advances, so the returned `to` matches the scan semantics.
|
||||||
|
if len(b.data) > 0 && b.statsValid && idx <= 0 && t <= b.firstWrite() && b.end() <= to {
|
||||||
|
if b.statSamples > 0 {
|
||||||
|
sum += b.statSum
|
||||||
|
samples += b.statSamples
|
||||||
|
if b.statMin < min {
|
||||||
|
min = b.statMin
|
||||||
|
}
|
||||||
|
if b.statMax > max {
|
||||||
|
max = b.statMax
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Position t at the buffer's last real data point; the loop's
|
||||||
|
// t += frequency then advances into the trailing gap / next buffer.
|
||||||
|
t = b.end() - b.frequency
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if t < b.start || idx >= len(b.data) {
|
if t < b.start || idx >= len(b.data) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -54,10 +102,14 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
samples += 1
|
samples++
|
||||||
sum += xf
|
sum += xf
|
||||||
min = math.Min(min, xf)
|
if xf < min {
|
||||||
max = math.Max(max, xf)
|
min = xf
|
||||||
|
}
|
||||||
|
if xf > max {
|
||||||
|
max = xf
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return Stats{
|
return Stats{
|
||||||
@@ -83,7 +135,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
|
|||||||
|
|
||||||
n, samples := 0, 0
|
n, samples := 0, 0
|
||||||
avg, min, max := schema.Float(0), math.MaxFloat32, -math.MaxFloat32
|
avg, min, max := schema.Float(0), math.MaxFloat32, -math.MaxFloat32
|
||||||
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
|
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer, path []string) error {
|
||||||
stats, cfrom, cto, err := b.stats(from, to)
|
stats, cfrom, cto, err := b.stats(from, to)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -92,9 +144,11 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
|
|||||||
if n == 0 {
|
if n == 0 {
|
||||||
from, to = cfrom, cto
|
from, to = cfrom, cto
|
||||||
} else if from != cfrom {
|
} else if from != cfrom {
|
||||||
return ErrDataDoesNotAlignMissingFront
|
return fmt.Errorf("stats: %w: metric=%s node=%s buf#%d expected[%d,%d] actual[%d,%d]",
|
||||||
|
ErrDataDoesNotAlignMissingFront, metric, strings.Join(path, "/"), n, from, to, cfrom, cto)
|
||||||
} else if to != cto {
|
} else if to != cto {
|
||||||
return ErrDataDoesNotAlignMissingBack
|
return fmt.Errorf("stats: %w: metric=%s node=%s buf#%d expected[%d,%d] actual[%d,%d]",
|
||||||
|
ErrDataDoesNotAlignMissingBack, metric, strings.Join(path, "/"), n, from, to, cfrom, cto)
|
||||||
}
|
}
|
||||||
|
|
||||||
samples += stats.Samples
|
samples += stats.Samples
|
||||||
@@ -103,7 +157,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
|
|||||||
max = math.Max(max, float64(stats.Max))
|
max = math.Max(max, float64(stats.Max))
|
||||||
n += 1
|
n += 1
|
||||||
return nil
|
return nil
|
||||||
})
|
}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,412 @@
|
|||||||
|
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
// All rights reserved. This file is part of cc-backend.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package metricstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ClusterCockpit/cc-lib/v2/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewBufferStatsInitialized(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
if b.statSamples != 0 {
|
||||||
|
t.Errorf("statSamples = %d, want 0", b.statSamples)
|
||||||
|
}
|
||||||
|
if b.statSum != 0 {
|
||||||
|
t.Errorf("statSum = %v, want 0", b.statSum)
|
||||||
|
}
|
||||||
|
if b.statMin != math.MaxFloat32 {
|
||||||
|
t.Errorf("statMin = %v, want MaxFloat32 sentinel", b.statMin)
|
||||||
|
}
|
||||||
|
if b.statMax != -math.MaxFloat32 {
|
||||||
|
t.Errorf("statMax = %v, want -MaxFloat32 sentinel", b.statMax)
|
||||||
|
}
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Error("statsValid = false, want true for a fresh empty buffer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteUpdatesRunningStats(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(3.0))
|
||||||
|
b.write(110, schema.Float(1.0))
|
||||||
|
b.write(120, schema.Float(5.0))
|
||||||
|
|
||||||
|
if b.statSamples != 3 {
|
||||||
|
t.Errorf("statSamples = %d, want 3", b.statSamples)
|
||||||
|
}
|
||||||
|
if b.statSum != 9.0 {
|
||||||
|
t.Errorf("statSum = %v, want 9.0", b.statSum)
|
||||||
|
}
|
||||||
|
if b.statMin != 1.0 {
|
||||||
|
t.Errorf("statMin = %v, want 1.0", b.statMin)
|
||||||
|
}
|
||||||
|
if b.statMax != 5.0 {
|
||||||
|
t.Errorf("statMax = %v, want 5.0", b.statMax)
|
||||||
|
}
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Error("statsValid = false, want true after appends only")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteNaNExcludedFromStats(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(2.0))
|
||||||
|
b.write(110, schema.NaN) // explicit NaN
|
||||||
|
b.write(130, schema.Float(4.0)) // leaves a NaN-filled gap at t=120
|
||||||
|
|
||||||
|
if b.statSamples != 2 {
|
||||||
|
t.Errorf("statSamples = %d, want 2 (NaN and gap excluded)", b.statSamples)
|
||||||
|
}
|
||||||
|
if b.statSum != 6.0 {
|
||||||
|
t.Errorf("statSum = %v, want 6.0", b.statSum)
|
||||||
|
}
|
||||||
|
if b.statMin != 2.0 || b.statMax != 4.0 {
|
||||||
|
t.Errorf("statMin/statMax = %v/%v, want 2.0/4.0", b.statMin, b.statMax)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteOverwriteInvalidatesStats(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(1.0))
|
||||||
|
b.write(110, schema.Float(2.0))
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Fatal("precondition: statsValid should be true after appends")
|
||||||
|
}
|
||||||
|
|
||||||
|
b.write(100, schema.Float(99.0)) // overwrite existing index
|
||||||
|
if b.statsValid {
|
||||||
|
t.Error("statsValid = true, want false after overwrite")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecomputeStatsRebuildsAfterOverwrite(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(1.0))
|
||||||
|
b.write(110, schema.Float(2.0))
|
||||||
|
b.write(100, schema.Float(99.0)) // overwrite -> statsValid false, min stale
|
||||||
|
|
||||||
|
b.recomputeStats()
|
||||||
|
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Error("statsValid = false, want true after recompute")
|
||||||
|
}
|
||||||
|
if b.statSamples != 2 {
|
||||||
|
t.Errorf("statSamples = %d, want 2", b.statSamples)
|
||||||
|
}
|
||||||
|
if b.statSum != 101.0 {
|
||||||
|
t.Errorf("statSum = %v, want 101.0", b.statSum)
|
||||||
|
}
|
||||||
|
if b.statMin != 2.0 {
|
||||||
|
t.Errorf("statMin = %v, want 2.0 (1.0 was overwritten)", b.statMin)
|
||||||
|
}
|
||||||
|
if b.statMax != 99.0 {
|
||||||
|
t.Errorf("statMax = %v, want 99.0", b.statMax)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecomputeStatsEmptyBuffer(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.recomputeStats()
|
||||||
|
if b.statSamples != 0 || b.statSum != 0 {
|
||||||
|
t.Errorf("empty buffer stats = samples %d sum %v, want 0/0", b.statSamples, b.statSum)
|
||||||
|
}
|
||||||
|
if b.statMin != math.MaxFloat32 || b.statMax != -math.MaxFloat32 {
|
||||||
|
t.Errorf("empty buffer min/max = %v/%v, want sentinels", b.statMin, b.statMax)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsFullBufferMatchesScan(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(3.0))
|
||||||
|
b.write(110, schema.Float(1.0))
|
||||||
|
b.write(120, schema.Float(5.0))
|
||||||
|
// Range fully covers the buffer (from before firstWrite, to at/after end).
|
||||||
|
s, _, _, err := b.stats(100, 130)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Samples != 3 {
|
||||||
|
t.Errorf("Samples = %d, want 3", s.Samples)
|
||||||
|
}
|
||||||
|
if s.Min != 1.0 || s.Max != 5.0 {
|
||||||
|
t.Errorf("Min/Max = %v/%v, want 1.0/5.0", s.Min, s.Max)
|
||||||
|
}
|
||||||
|
if s.Avg != 3.0 {
|
||||||
|
t.Errorf("Avg = %v, want 3.0", s.Avg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsPartialRangeScans(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(3.0)) // t=100
|
||||||
|
b.write(110, schema.Float(1.0)) // t=110
|
||||||
|
b.write(120, schema.Float(5.0)) // t=120
|
||||||
|
// Partial range: only t=110 and t=120 (from excludes t=100).
|
||||||
|
s, _, _, err := b.stats(110, 130)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Samples != 2 {
|
||||||
|
t.Errorf("Samples = %d, want 2", s.Samples)
|
||||||
|
}
|
||||||
|
if s.Min != 1.0 || s.Max != 5.0 {
|
||||||
|
t.Errorf("Min/Max = %v/%v, want 1.0/5.0", s.Min, s.Max)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsOverwriteThenFullQuery(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
b.write(100, schema.Float(1.0))
|
||||||
|
b.write(110, schema.Float(2.0))
|
||||||
|
b.write(120, schema.Float(3.0))
|
||||||
|
b.write(100, schema.Float(99.0)) // overwrite the running min -> statsValid false
|
||||||
|
|
||||||
|
s, _, _, err := b.stats(100, 130)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Min != 2.0 {
|
||||||
|
t.Errorf("Min = %v, want 2.0 (scanned after overwrite)", s.Min)
|
||||||
|
}
|
||||||
|
if s.Max != 99.0 {
|
||||||
|
t.Errorf("Max = %v, want 99.0", s.Max)
|
||||||
|
}
|
||||||
|
if s.Samples != 3 {
|
||||||
|
t.Errorf("Samples = %d, want 3", s.Samples)
|
||||||
|
}
|
||||||
|
if b.statsValid {
|
||||||
|
t.Error("statsValid should still be false: the read path must not mutate buffer state")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsMultiBufferChain(t *testing.T) {
|
||||||
|
// Two full buffers of cap=3 (interior buffers are always full).
|
||||||
|
// b1: t=100,110,120 = 1,2,3 ; b2: t=130,140,150 = 4,5,6.
|
||||||
|
b1 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 95}
|
||||||
|
b1.data = append(b1.data, 1.0, 2.0, 3.0)
|
||||||
|
b2 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 125}
|
||||||
|
b2.data = append(b2.data, 4.0, 5.0, 6.0)
|
||||||
|
b2.prev = b1
|
||||||
|
b1.next = b2
|
||||||
|
// b1/b2 built bare: statsValid is false (zero value). Recompute here to mirror
|
||||||
|
// checkpoint-loaded valid buffers, so the fast-path cache fold is exercised.
|
||||||
|
b1.recomputeStats()
|
||||||
|
b2.recomputeStats()
|
||||||
|
|
||||||
|
s, _, _, err := b2.stats(100, 160)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Samples != 6 {
|
||||||
|
t.Errorf("Samples = %d, want 6", s.Samples)
|
||||||
|
}
|
||||||
|
if s.Min != 1.0 || s.Max != 6.0 {
|
||||||
|
t.Errorf("Min/Max = %v/%v, want 1.0/6.0", s.Min, s.Max)
|
||||||
|
}
|
||||||
|
if s.Avg != 3.5 {
|
||||||
|
t.Errorf("Avg = %v, want 3.5", s.Avg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsFastPathThenPartialTail(t *testing.T) {
|
||||||
|
// Regression test: three buffers (all frequency 10) where the query fast-paths
|
||||||
|
// earlier fully-covered buffers, then only PARTIALLY covers the last buffer.
|
||||||
|
// firstWrite = start + frequency/2, so with start=95 the first value is at t=100.
|
||||||
|
// b1: 1,2,3 at t=100,110,120 (end=130, fully in [100,170) -> fast path folds all 3)
|
||||||
|
// b2: 4,5,6 at t=130,140,150 (end=160, fully in [100,170) -> fast path folds all 3)
|
||||||
|
// b3: 7,8,9 at t=160,170,180 (end=190; not fully covered -> scans; only t=160 < 170)
|
||||||
|
// Query [100, 170) includes t=100,110,120,130,140,150,160 = values 1..7 (7 points).
|
||||||
|
// t=170 is excluded since 170 >= to. Expected: Samples=7, Sum=28, Min=1, Max=7, Avg=4.0.
|
||||||
|
// Buffers built bare: statsValid is false (zero value). Recompute on b1/b2 to mirror
|
||||||
|
// checkpoint-loaded valid buffers so the fast path fires for them; b3 is only
|
||||||
|
// partially covered by the query so it still scans regardless of validity,
|
||||||
|
// giving intended mixed fast-path-then-scan coverage.
|
||||||
|
|
||||||
|
b1 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 95}
|
||||||
|
b1.data = append(b1.data, 1.0, 2.0, 3.0)
|
||||||
|
|
||||||
|
b2 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 125}
|
||||||
|
b2.data = append(b2.data, 4.0, 5.0, 6.0)
|
||||||
|
|
||||||
|
b3 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 155}
|
||||||
|
b3.data = append(b3.data, 7.0, 8.0, 9.0)
|
||||||
|
|
||||||
|
b1.next = b2
|
||||||
|
b2.prev = b1
|
||||||
|
b2.next = b3
|
||||||
|
b3.prev = b2
|
||||||
|
|
||||||
|
b1.recomputeStats()
|
||||||
|
b2.recomputeStats()
|
||||||
|
b3.recomputeStats()
|
||||||
|
|
||||||
|
s, _, _, err := b3.stats(100, 170)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Samples != 7 {
|
||||||
|
t.Errorf("Samples = %d, want 7", s.Samples)
|
||||||
|
}
|
||||||
|
if s.Min != 1.0 || s.Max != 7.0 {
|
||||||
|
t.Errorf("Min/Max = %v/%v, want 1.0/7.0", s.Min, s.Max)
|
||||||
|
}
|
||||||
|
if s.Avg != 4.0 {
|
||||||
|
t.Errorf("Avg = %v, want 4.0", s.Avg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckpointLoadedBufferStatsEager(t *testing.T) {
|
||||||
|
// Mirror how loadFile constructs a buffer: bare struct, data assigned
|
||||||
|
// directly with cap==len, then stats computed eagerly.
|
||||||
|
data := []schema.Float{2.0, 4.0, 6.0}
|
||||||
|
n := len(data)
|
||||||
|
b := &buffer{
|
||||||
|
frequency: 10,
|
||||||
|
start: 95,
|
||||||
|
data: data[0:n:n],
|
||||||
|
archived: true,
|
||||||
|
}
|
||||||
|
b.recomputeStats() // the eager call loadFile will perform
|
||||||
|
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Error("statsValid = false, want true after eager recompute at load")
|
||||||
|
}
|
||||||
|
if b.statSamples != 3 {
|
||||||
|
t.Errorf("statSamples = %d, want 3", b.statSamples)
|
||||||
|
}
|
||||||
|
if b.statMin != 2.0 || b.statMax != 6.0 {
|
||||||
|
t.Errorf("statMin/statMax = %v/%v, want 2.0/6.0", b.statMin, b.statMax)
|
||||||
|
}
|
||||||
|
if b.statSum != 12.0 {
|
||||||
|
t.Errorf("statSum = %v, want 12.0", b.statSum)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsConcurrentQueriesNoRace guards against the data race fixed by making
|
||||||
|
// the stats() read path non-mutating: previously an invalid buffer's stats()
|
||||||
|
// call would run recomputeStats() (a write) under only the caller's shared
|
||||||
|
// RLock (see MemoryStore.Stats -> Level.findBuffers), so concurrent readers on
|
||||||
|
// the same buffer could race on statSum/statSamples/statMin/statMax/statsValid.
|
||||||
|
// Run with -race; both subtests must be race-clean.
|
||||||
|
func TestStatsConcurrentQueriesNoRace(t *testing.T) {
|
||||||
|
const goroutines = 50
|
||||||
|
|
||||||
|
t.Run("valid buffer", func(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
vals := []schema.Float{3.0, 1.0, 5.0, 2.0, 4.0}
|
||||||
|
for i, v := range vals {
|
||||||
|
ts := int64(100 + i*10)
|
||||||
|
if _, err := b.write(ts, v); err != nil {
|
||||||
|
t.Fatalf("write(%d) error = %v", ts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !b.statsValid {
|
||||||
|
t.Fatal("precondition: statsValid should be true for an append-only buffer")
|
||||||
|
}
|
||||||
|
|
||||||
|
from, to := int64(100), int64(150)
|
||||||
|
expected, _, _, err := b.stats(from, to)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < goroutines; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
s, _, _, err := b.stats(from, to)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("stats() error = %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s != expected {
|
||||||
|
t.Errorf("stats() = %+v, want %+v", s, expected)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid buffer after overwrite", func(t *testing.T) {
|
||||||
|
b := newBuffer(100, 10)
|
||||||
|
if _, err := b.write(100, schema.Float(1.0)); err != nil {
|
||||||
|
t.Fatalf("write error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := b.write(110, schema.Float(2.0)); err != nil {
|
||||||
|
t.Fatalf("write error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := b.write(120, schema.Float(3.0)); err != nil {
|
||||||
|
t.Fatalf("write error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := b.write(100, schema.Float(99.0)); err != nil { // overwrite -> statsValid false
|
||||||
|
t.Fatalf("write error = %v", err)
|
||||||
|
}
|
||||||
|
if b.statsValid {
|
||||||
|
t.Fatal("precondition: statsValid should be false after overwrite")
|
||||||
|
}
|
||||||
|
|
||||||
|
from, to := int64(100), int64(130)
|
||||||
|
expected, _, _, err := b.stats(from, to)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < goroutines; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
s, _, _, err := b.stats(from, to)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("stats() error = %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s != expected {
|
||||||
|
t.Errorf("stats() = %+v, want %+v", s, expected)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatsGappedChain covers a multi-buffer chain with a real time gap between
|
||||||
|
// buffers: b1's last real point is at t=120, and b2's first real point is at
|
||||||
|
// t=170, so t=130,140,150,160 have no data anywhere (unlike an in-buffer,
|
||||||
|
// NaN-filled gap). The query spans the gap; only the real points on either
|
||||||
|
// side should be counted.
|
||||||
|
func TestStatsGappedChain(t *testing.T) {
|
||||||
|
b1 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 95}
|
||||||
|
b1.data = append(b1.data, 1.0, 2.0, 3.0) // t=100,110,120 (end=130)
|
||||||
|
|
||||||
|
b2 := &buffer{data: make([]schema.Float, 0, 3), frequency: 10, start: 165}
|
||||||
|
b2.data = append(b2.data, 4.0, 5.0, 6.0) // t=170,180,190 (end=200)
|
||||||
|
|
||||||
|
b1.next = b2
|
||||||
|
b2.prev = b1
|
||||||
|
|
||||||
|
b1.recomputeStats()
|
||||||
|
b2.recomputeStats()
|
||||||
|
|
||||||
|
s, _, _, err := b2.stats(100, 200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stats() error = %v", err)
|
||||||
|
}
|
||||||
|
if s.Samples != 6 {
|
||||||
|
t.Errorf("Samples = %d, want 6 (gap contributes no samples)", s.Samples)
|
||||||
|
}
|
||||||
|
if s.Min != 1.0 || s.Max != 6.0 {
|
||||||
|
t.Errorf("Min/Max = %v/%v, want 1.0/6.0", s.Min, s.Max)
|
||||||
|
}
|
||||||
|
if s.Avg != 3.5 {
|
||||||
|
t.Errorf("Avg = %v, want 3.5", s.Avg)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -231,9 +231,9 @@ func TestImportDataIntegrity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify metric data exists
|
// Verify metric data exists
|
||||||
if len(srcData) != len(dstData) {
|
if len(srcData.Metrics) != len(dstData.Metrics) {
|
||||||
t.Errorf("Metric count mismatch for job %d: expected %d, got %d",
|
t.Errorf("Metric count mismatch for job %d: expected %d, got %d",
|
||||||
srcJob.Meta.JobID, len(srcData), len(dstData))
|
srcJob.Meta.JobID, len(srcData.Metrics), len(dstData.Metrics))
|
||||||
}
|
}
|
||||||
|
|
||||||
verifiedJobs++
|
verifiedJobs++
|
||||||
|
|||||||
@@ -177,6 +177,21 @@ const configSchema = `{
|
|||||||
"description": "Initial thickness of rendered plotlines. Applies to metric plot, job compare plot and roofline.",
|
"description": "Initial thickness of rendered plotlines. Applies to metric plot, job compare plot and roofline.",
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"smoothing-window": {
|
||||||
|
"description": "Initial display smoothing window in data points, applied after downsampling. 0 disables smoothing. Display-only: reported statistics and footprints are unaffected.",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"resample-policy": {
|
||||||
|
"description": "Initial resample policy for new users, controlling how many data points metric plots request. Empty string falls back to main.resampling.default-policy.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["", "low", "medium", "high"]
|
||||||
|
},
|
||||||
|
"resample-algo": {
|
||||||
|
"description": "Initial resample algorithm for new users. Empty string falls back to main.resampling.default-algo.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["", "lttb", "average", "simple"]
|
||||||
|
},
|
||||||
"color-scheme": {
|
"color-scheme": {
|
||||||
"description": "Initial colorScheme to be used for metric plots.",
|
"description": "Initial colorScheme to be used for metric plots.",
|
||||||
"type": "array",
|
"type": "array",
|
||||||
|
|||||||
@@ -70,3 +70,13 @@ footer {
|
|||||||
margin: 0rem 0.8rem;
|
margin: 0rem 0.8rem;
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Fix: prevent Sveltestrap collapseIn from leaving the navbar invisible
|
||||||
|
when prefers-reduced-motion causes transition-duration = 0ms and Svelte
|
||||||
|
skips calling tick(), leaving the node stuck in .collapsing (height:0). */
|
||||||
|
@media (prefers-reduced-motion: reduce) {
|
||||||
|
.navbar-collapse.collapsing {
|
||||||
|
height: auto !important;
|
||||||
|
overflow: visible !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -60,6 +60,8 @@
|
|||||||
let presetProject = $derived(filterPresets?.project ? filterPresets.project : "");
|
let presetProject = $derived(filterPresets?.project ? filterPresets.project : "");
|
||||||
let selectedCluster = $derived(filterPresets?.cluster ? filterPresets.cluster : null);
|
let selectedCluster = $derived(filterPresets?.cluster ? filterPresets.cluster : null);
|
||||||
let selectedSubCluster = $derived(filterPresets?.partition ? filterPresets.partition : null);
|
let selectedSubCluster = $derived(filterPresets?.partition ? filterPresets.partition : null);
|
||||||
|
const maxClusters = $derived($initq?.data?.clusters?.length || 0);
|
||||||
|
const maxSubClusters = $derived($initq?.data?.clusters?.find((c) => c.name == selectedCluster)?.subClusters?.length || 0);
|
||||||
let metrics = $derived.by(() => {
|
let metrics = $derived.by(() => {
|
||||||
if (thisInit && ccconfig) {
|
if (thisInit && ccconfig) {
|
||||||
if (selectedCluster) {
|
if (selectedCluster) {
|
||||||
@@ -243,6 +245,8 @@
|
|||||||
presetMetrics={metrics}
|
presetMetrics={metrics}
|
||||||
cluster={selectedCluster}
|
cluster={selectedCluster}
|
||||||
subCluster={selectedSubCluster}
|
subCluster={selectedSubCluster}
|
||||||
|
{maxClusters}
|
||||||
|
{maxSubClusters}
|
||||||
configName="metricConfig_jobListMetrics"
|
configName="metricConfig_jobListMetrics"
|
||||||
footprintSelect
|
footprintSelect
|
||||||
{globalMetrics}
|
{globalMetrics}
|
||||||
|
|||||||
@@ -63,15 +63,14 @@
|
|||||||
let pendingHostnameFilter = $state("");
|
let pendingHostnameFilter = $state("");
|
||||||
let isMetricsSelectionOpen = $state(false);
|
let isMetricsSelectionOpen = $state(false);
|
||||||
|
|
||||||
/* Derived Init Return */
|
/* Derived Init Returns */
|
||||||
const thisInit = $derived($initq?.data ? true : false);
|
const thisInit = $derived($initq?.data ? true : false);
|
||||||
|
const maxSubClusters = $derived($initq?.data?.clusters?.find((c) => c.name == cluster)?.subClusters?.length || 0);
|
||||||
|
|
||||||
/* Derived States */
|
/* Derived States */
|
||||||
const ccconfig = $derived(thisInit ? getContext("cc-config") : null);
|
const ccconfig = $derived(thisInit ? getContext("cc-config") : null);
|
||||||
const globalMetrics = $derived(thisInit ? getContext("globalMetrics") : null);
|
const globalMetrics = $derived(thisInit ? getContext("globalMetrics") : null);
|
||||||
const resampleConfig = $derived(thisInit ? getContext("resampling") : null);
|
const resampleConfig = $derived(thisInit ? getContext("resampling") : null);
|
||||||
const resampleResolutions = $derived(resampleConfig ? [...resampleConfig.resolutions] : []);
|
|
||||||
const resampleDefault = $derived(resampleConfig ? Math.max(...resampleConfig.resolutions) : 0);
|
|
||||||
const displayNodeOverview = $derived((displayType === 'OVERVIEW'));
|
const displayNodeOverview = $derived((displayType === 'OVERVIEW'));
|
||||||
|
|
||||||
const systemMetrics = $derived(globalMetrics ? [...globalMetrics.filter((gm) => gm?.availability.find((av) => av.cluster == cluster))] : []);
|
const systemMetrics = $derived(globalMetrics ? [...globalMetrics.filter((gm) => gm?.availability.find((av) => av.cluster == cluster))] : []);
|
||||||
@@ -85,7 +84,8 @@
|
|||||||
return {...pendingUnits};
|
return {...pendingUnits};
|
||||||
});
|
});
|
||||||
|
|
||||||
let selectedResolution = $derived(resampleDefault);
|
// null lets the backend resolve the resolution from the configured resample policy.
|
||||||
|
let selectedResolution = $state(null);
|
||||||
let to = $derived(presetTo ? presetTo : new Date(Date.now()));
|
let to = $derived(presetTo ? presetTo : new Date(Date.now()));
|
||||||
let from = $derived(presetFrom ? presetFrom : new Date(nowDate.setHours(nowDate.getHours() - 4)));
|
let from = $derived(presetFrom ? presetFrom : new Date(nowDate.setHours(nowDate.getHours() - 4)));
|
||||||
|
|
||||||
@@ -159,7 +159,7 @@
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<!-- ROW1: Tools-->
|
<!-- ROW1: Tools-->
|
||||||
<Row cols={{ xs: 2, lg: !displayNodeOverview ? (resampleConfig ? 6 : 5) : 5 }} class="mb-3">
|
<Row cols={{ xs: 2, lg: 5 }} class="mb-3">
|
||||||
{#if thisInit}
|
{#if thisInit}
|
||||||
<!-- List Metric Select Col-->
|
<!-- List Metric Select Col-->
|
||||||
{#if !displayNodeOverview}
|
{#if !displayNodeOverview}
|
||||||
@@ -176,21 +176,6 @@
|
|||||||
</Button>
|
</Button>
|
||||||
</InputGroup>
|
</InputGroup>
|
||||||
</Col>
|
</Col>
|
||||||
{#if resampleConfig}
|
|
||||||
<Col>
|
|
||||||
<InputGroup>
|
|
||||||
<InputGroupText><Icon name="plus-slash-minus" /></InputGroupText>
|
|
||||||
<InputGroupText>Resolution</InputGroupText>
|
|
||||||
<Input type="select" bind:value={selectedResolution}>
|
|
||||||
{#each resampleResolutions as res}
|
|
||||||
<option value={res}
|
|
||||||
>{res} sec</option
|
|
||||||
>
|
|
||||||
{/each}
|
|
||||||
</Input>
|
|
||||||
</InputGroup>
|
|
||||||
</Col>
|
|
||||||
{/if}
|
|
||||||
{/if}
|
{/if}
|
||||||
<!-- Node Col-->
|
<!-- Node Col-->
|
||||||
<Col class="mt-2 mt-lg-0">
|
<Col class="mt-2 mt-lg-0">
|
||||||
@@ -284,6 +269,7 @@
|
|||||||
{cluster}
|
{cluster}
|
||||||
{subCluster}
|
{subCluster}
|
||||||
{globalMetrics}
|
{globalMetrics}
|
||||||
|
maxSubClusters={subCluster? null: maxSubClusters}
|
||||||
configName="nodeList_selectedMetrics"
|
configName="nodeList_selectedMetrics"
|
||||||
applyMetrics={(newMetrics) =>
|
applyMetrics={(newMetrics) =>
|
||||||
selectedMetrics = [...newMetrics]
|
selectedMetrics = [...newMetrics]
|
||||||
|
|||||||
@@ -90,6 +90,8 @@
|
|||||||
const shortDuration = $derived(ccconfig?.jobList_hideShortRunningJobs);
|
const shortDuration = $derived(ccconfig?.jobList_hideShortRunningJobs);
|
||||||
let selectedCluster = $derived(filterPresets?.cluster ? filterPresets.cluster : null);
|
let selectedCluster = $derived(filterPresets?.cluster ? filterPresets.cluster : null);
|
||||||
let selectedSubCluster = $derived(filterPresets?.partition ? filterPresets.partition : null);
|
let selectedSubCluster = $derived(filterPresets?.partition ? filterPresets.partition : null);
|
||||||
|
const maxClusters = $derived($initq?.data?.clusters?.length || 0);
|
||||||
|
const maxSubClusters = $derived($initq?.data?.clusters?.find((c) => c.name == selectedCluster)?.subClusters?.length || 0);
|
||||||
let metrics = $derived.by(() => {
|
let metrics = $derived.by(() => {
|
||||||
if (thisInit && ccconfig) {
|
if (thisInit && ccconfig) {
|
||||||
if (selectedCluster) {
|
if (selectedCluster) {
|
||||||
@@ -531,6 +533,8 @@
|
|||||||
presetMetrics={metrics}
|
presetMetrics={metrics}
|
||||||
cluster={selectedCluster}
|
cluster={selectedCluster}
|
||||||
subCluster={selectedSubCluster}
|
subCluster={selectedSubCluster}
|
||||||
|
{maxClusters}
|
||||||
|
{maxSubClusters}
|
||||||
configName="metricConfig_jobListMetrics"
|
configName="metricConfig_jobListMetrics"
|
||||||
footprintSelect
|
footprintSelect
|
||||||
{globalMetrics}
|
{globalMetrics}
|
||||||
|
|||||||
@@ -77,8 +77,8 @@
|
|||||||
<Card class="h-100">
|
<Card class="h-100">
|
||||||
<CardBody>
|
<CardBody>
|
||||||
<CardTitle class="mb-3">Metric Plot Resampling Info</CardTitle>
|
<CardTitle class="mb-3">Metric Plot Resampling Info</CardTitle>
|
||||||
<p>Triggered at {resampleConfig.trigger} datapoints.</p>
|
<p>Resampling is enabled.</p>
|
||||||
<p>Configured resolutions: {resampleConfig.resolutions}</p>
|
<p>Target data points per plot: {resampleConfig.targetPoints}</p>
|
||||||
</CardBody>
|
</CardBody>
|
||||||
</Card>
|
</Card>
|
||||||
</Col>
|
</Col>
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
Card,
|
Card,
|
||||||
CardTitle,
|
CardTitle,
|
||||||
} from "@sveltestrap/sveltestrap";
|
} from "@sveltestrap/sveltestrap";
|
||||||
|
import { getContext } from "svelte";
|
||||||
import { fade } from "svelte/transition";
|
import { fade } from "svelte/transition";
|
||||||
|
|
||||||
/* Svelte 5 Props */
|
/* Svelte 5 Props */
|
||||||
@@ -25,6 +26,8 @@
|
|||||||
displayMessage = $bindable(),
|
displayMessage = $bindable(),
|
||||||
updateSetting
|
updateSetting
|
||||||
} = $props();
|
} = $props();
|
||||||
|
|
||||||
|
const resampleConfig = getContext("resampling");
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<Row cols={3} class="p-2 g-2">
|
<Row cols={3} class="p-2 g-2">
|
||||||
@@ -64,7 +67,7 @@
|
|||||||
id="lwvalue"
|
id="lwvalue"
|
||||||
name="value"
|
name="value"
|
||||||
aria-describedby="lineWidthHelp"
|
aria-describedby="lineWidthHelp"
|
||||||
value={config.plotConfiguration_lineWidth}
|
value={config?.plotConfiguration_lineWidth}
|
||||||
min="1"
|
min="1"
|
||||||
/>
|
/>
|
||||||
<div id="lineWidthHelp" class="form-text">
|
<div id="lineWidthHelp" class="form-text">
|
||||||
@@ -111,7 +114,7 @@
|
|||||||
id="pprvalue"
|
id="pprvalue"
|
||||||
name="value"
|
name="value"
|
||||||
aria-describedby="plotsperrowHelp"
|
aria-describedby="plotsperrowHelp"
|
||||||
value={config.plotConfiguration_plotsPerRow}
|
value={config?.plotConfiguration_plotsPerRow}
|
||||||
min="1"
|
min="1"
|
||||||
/>
|
/>
|
||||||
<div id="plotsperrowHelp" class="form-text">
|
<div id="plotsperrowHelp" class="form-text">
|
||||||
@@ -153,7 +156,7 @@
|
|||||||
<input type="hidden" name="key" value="plotConfiguration_colorBackground" />
|
<input type="hidden" name="key" value="plotConfiguration_colorBackground" />
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<div>
|
<div>
|
||||||
{#if config.plotConfiguration_colorBackground}
|
{#if config?.plotConfiguration_colorBackground}
|
||||||
<input type="radio" id="colb-true-checked" name="value" value="true" checked />
|
<input type="radio" id="colb-true-checked" name="value" value="true" checked />
|
||||||
{:else}
|
{:else}
|
||||||
<input type="radio" id="colb-true" name="value" value="true" />
|
<input type="radio" id="colb-true" name="value" value="true" />
|
||||||
@@ -161,7 +164,7 @@
|
|||||||
<label for="true">Yes</label>
|
<label for="true">Yes</label>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
{#if config.plotConfiguration_colorBackground}
|
{#if config?.plotConfiguration_colorBackground}
|
||||||
<input type="radio" id="colb-false" name="value" value="false" />
|
<input type="radio" id="colb-false" name="value" value="false" />
|
||||||
{:else}
|
{:else}
|
||||||
<input type="radio" id="colb-false-checked" name="value" value="false" checked />
|
<input type="radio" id="colb-false-checked" name="value" value="false" checked />
|
||||||
@@ -219,4 +222,144 @@
|
|||||||
</form>
|
</form>
|
||||||
</Card>
|
</Card>
|
||||||
</Col>
|
</Col>
|
||||||
|
|
||||||
|
<!-- RESAMPLE POLICY -->
|
||||||
|
<Col>
|
||||||
|
<Card class="h-100">
|
||||||
|
<form
|
||||||
|
id="resample-policy-form"
|
||||||
|
method="post"
|
||||||
|
action="/frontend/configuration/"
|
||||||
|
class="card-body"
|
||||||
|
onsubmit={(e) => updateSetting(e, {
|
||||||
|
selector: "#resample-policy-form",
|
||||||
|
target: "rsp",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<CardTitle
|
||||||
|
style="margin-bottom: 1em; display: flex; align-items: center;"
|
||||||
|
>
|
||||||
|
<div>Resample Policy</div>
|
||||||
|
{#if displayMessage && message.target == "rsp"}
|
||||||
|
<div style="margin-left: auto; font-size: 0.9em;">
|
||||||
|
<code style="color: {message.color};" out:fade>
|
||||||
|
Update: {message.msg}
|
||||||
|
</code>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</CardTitle>
|
||||||
|
<input type="hidden" name="key" value="plotConfiguration_resamplePolicy" />
|
||||||
|
<div class="mb-3">
|
||||||
|
{#each [["", "Default"], ["low", "Low"], ["medium", "Medium"], ["high", "High"]] as [val, label]}
|
||||||
|
<div>
|
||||||
|
<input type="radio" id="rsp-{val || 'default'}" name="value" value={JSON.stringify(val)}
|
||||||
|
checked={(!config?.plotConfiguration_resamplePolicy && val === "") || config?.plotConfiguration_resamplePolicy === val} />
|
||||||
|
<label for="rsp-{val || 'default'}">{label}</label>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
<div id="resamplePolicyHelp" class="form-text">
|
||||||
|
Controls how many data points are shown in metric plots. Low = fast overview (~200 points), Medium = balanced (~500), High = maximum detail (~1000).
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Button color="primary" type="submit">Submit</Button>
|
||||||
|
</form>
|
||||||
|
</Card>
|
||||||
|
</Col>
|
||||||
|
|
||||||
|
<!-- RESAMPLE ALGORITHM -->
|
||||||
|
<Col>
|
||||||
|
<Card class="h-100">
|
||||||
|
<form
|
||||||
|
id="resample-algo-form"
|
||||||
|
method="post"
|
||||||
|
action="/frontend/configuration/"
|
||||||
|
class="card-body"
|
||||||
|
onsubmit={(e) => updateSetting(e, {
|
||||||
|
selector: "#resample-algo-form",
|
||||||
|
target: "rsa",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<CardTitle
|
||||||
|
style="margin-bottom: 1em; display: flex; align-items: center;"
|
||||||
|
>
|
||||||
|
<div>Resample Algorithm</div>
|
||||||
|
{#if displayMessage && message.target == "rsa"}
|
||||||
|
<div style="margin-left: auto; font-size: 0.9em;">
|
||||||
|
<code style="color: {message.color};" out:fade>
|
||||||
|
Update: {message.msg}
|
||||||
|
</code>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</CardTitle>
|
||||||
|
<input type="hidden" name="key" value="plotConfiguration_resampleAlgo" />
|
||||||
|
<div class="mb-3">
|
||||||
|
{#each [["", "Default"], ["lttb", "LTTB"], ["average", "Average"], ["simple", "Simple"]] as [val, label]}
|
||||||
|
<div>
|
||||||
|
<input type="radio" id="rsa-{val || 'default'}" name="value" value={JSON.stringify(val)}
|
||||||
|
checked={(!config?.plotConfiguration_resampleAlgo && val === "") || config?.plotConfiguration_resampleAlgo === val} />
|
||||||
|
<label for="rsa-{val || 'default'}">{label}</label>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
<div id="resampleAlgoHelp" class="form-text">
|
||||||
|
Algorithm used when downsampling time-series data. Average (the
|
||||||
|
default) makes every plotted point the true mean of its interval,
|
||||||
|
LTTB preserves visual shape by keeping extremes, Simple picks every
|
||||||
|
Nth point.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Button color="primary" type="submit">Submit</Button>
|
||||||
|
</form>
|
||||||
|
</Card>
|
||||||
|
</Col>
|
||||||
|
|
||||||
|
<!-- SMOOTHING WINDOW -->
|
||||||
|
<Col>
|
||||||
|
<Card class="h-100">
|
||||||
|
<form
|
||||||
|
id="smoothing-window-form"
|
||||||
|
method="post"
|
||||||
|
action="/frontend/configuration/"
|
||||||
|
class="card-body"
|
||||||
|
onsubmit={(e) => updateSetting(e, {
|
||||||
|
selector: "#smoothing-window-form",
|
||||||
|
target: "sw",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<CardTitle
|
||||||
|
style="margin-bottom: 1em; display: flex; align-items: center;"
|
||||||
|
>
|
||||||
|
<div>Smoothing Window</div>
|
||||||
|
{#if displayMessage && message.target == "sw"}
|
||||||
|
<div style="margin-left: auto; font-size: 0.9em;">
|
||||||
|
<code style="color: {message.color};" out:fade>
|
||||||
|
Update: {message.msg}
|
||||||
|
</code>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</CardTitle>
|
||||||
|
<input type="hidden" name="key" value="plotConfiguration_smoothingWindow" />
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="value" class="form-label">Smoothing Window</label>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
class="form-control"
|
||||||
|
id="swvalue"
|
||||||
|
name="value"
|
||||||
|
aria-describedby="smoothingWindowHelp"
|
||||||
|
value={config?.plotConfiguration_smoothingWindow}
|
||||||
|
min="0"
|
||||||
|
/>
|
||||||
|
<div id="smoothingWindowHelp" class="form-text">
|
||||||
|
Width of the moving average applied to plotted lines, in data
|
||||||
|
points. 0 disables it, the default is 3. It is applied after
|
||||||
|
downsampling and is display-only: the reported min/avg/max
|
||||||
|
statistics and the job footprint are unaffected. Combined with the
|
||||||
|
Average algorithm it is a second, milder smoothing pass on top of
|
||||||
|
the interval means.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Button color="primary" type="submit">Submit</Button>
|
||||||
|
</form>
|
||||||
|
</Card>
|
||||||
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
@@ -84,7 +84,6 @@
|
|||||||
let thresholdStates = $state({});
|
let thresholdStates = $state({});
|
||||||
|
|
||||||
/* Derived */
|
/* Derived */
|
||||||
const resampleDefault = $derived(resampleConfig ? Math.max(...resampleConfig.resolutions) : 0);
|
|
||||||
const jobId = $derived(job.id);
|
const jobId = $derived(job.id);
|
||||||
const scopes = $derived.by(() => {
|
const scopes = $derived.by(() => {
|
||||||
if (job.numNodes == 1) {
|
if (job.numNodes == 1) {
|
||||||
@@ -95,7 +94,9 @@
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
let selectedResolution = $derived(resampleDefault);
|
// null lets the backend resolve the resolution from the configured resample
|
||||||
|
// policy; zoom interactions override it with an explicit value.
|
||||||
|
let selectedResolution = $state(null);
|
||||||
let isSelected = $derived(previousSelect);
|
let isSelected = $derived(previousSelect);
|
||||||
let metricsQuery = $derived(queryStore({
|
let metricsQuery = $derived(queryStore({
|
||||||
client: client,
|
client: client,
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
- `numhwthreads Number?`: Number of job HWThreads [Default: 0]
|
- `numhwthreads Number?`: Number of job HWThreads [Default: 0]
|
||||||
- `numaccs Number?`: Number of job Accelerators [Default: 0]
|
- `numaccs Number?`: Number of job Accelerators [Default: 0]
|
||||||
- `zoomState Object?`: The last zoom state to preserve on user zoom [Default: null]
|
- `zoomState Object?`: The last zoom state to preserve on user zoom [Default: null]
|
||||||
|
- `smoothingWindow Number?`: Display smoothing window in data points; overrides the user setting. 0 or 1 disables [Default: null]
|
||||||
- `thersholdState Object?`: The last threshold state to preserve on user zoom [Default: null]
|
- `thersholdState Object?`: The last threshold state to preserve on user zoom [Default: null]
|
||||||
- `extendedLegendData Object?`: Additional information to be rendered in an extended legend [Default: null]
|
- `extendedLegendData Object?`: Additional information to be rendered in an extended legend [Default: null]
|
||||||
- `onZoom Func`: Callback function to handle zoom-in event
|
- `onZoom Func`: Callback function to handle zoom-in event
|
||||||
@@ -25,6 +26,7 @@
|
|||||||
<script>
|
<script>
|
||||||
import uPlot from "uplot";
|
import uPlot from "uplot";
|
||||||
import { formatNumber, formatDurationTime } from "../units.js";
|
import { formatNumber, formatDurationTime } from "../units.js";
|
||||||
|
import { movingAverage } from "./smoothing.js";
|
||||||
import { getContext, onMount, onDestroy } from "svelte";
|
import { getContext, onMount, onDestroy } from "svelte";
|
||||||
import { Card, CardBody, CardHeader } from "@sveltestrap/sveltestrap";
|
import { Card, CardBody, CardHeader } from "@sveltestrap/sveltestrap";
|
||||||
|
|
||||||
@@ -43,6 +45,7 @@
|
|||||||
forNode = false,
|
forNode = false,
|
||||||
zoomState = null,
|
zoomState = null,
|
||||||
thresholdState = null,
|
thresholdState = null,
|
||||||
|
smoothingWindow = null,
|
||||||
extendedLegendData = null,
|
extendedLegendData = null,
|
||||||
plotSync = null,
|
plotSync = null,
|
||||||
enableFlip = false,
|
enableFlip = false,
|
||||||
@@ -73,9 +76,16 @@
|
|||||||
const subClusterTopology = $derived(getContext("getHardwareTopology")(cluster, subCluster));
|
const subClusterTopology = $derived(getContext("getHardwareTopology")(cluster, subCluster));
|
||||||
const metricConfig = $derived(getContext("getMetricConfig")(cluster, subCluster, metric));
|
const metricConfig = $derived(getContext("getMetricConfig")(cluster, subCluster, metric));
|
||||||
const usesMeanStatsSeries = $derived((statisticsSeries?.mean && statisticsSeries.mean.length != 0));
|
const usesMeanStatsSeries = $derived((statisticsSeries?.mean && statisticsSeries.mean.length != 0));
|
||||||
const resampleTrigger = $derived(resampleConfig?.trigger ? Number(resampleConfig.trigger) : null);
|
const nativeTimestep = $derived(metricConfig?.timestep || timestep);
|
||||||
const resampleResolutions = $derived(resampleConfig?.resolutions ? [...resampleConfig.resolutions] : null);
|
// Display-only smoothing, applied after backend downsampling. Does not affect
|
||||||
const resampleMinimum = $derived(resampleConfig?.resolutions ? Math.min(...resampleConfig.resolutions) : null);
|
// the reported statistics or the job footprint.
|
||||||
|
const smoothing = $derived(
|
||||||
|
Number(smoothingWindow ?? clusterCockpitConfig?.plotConfiguration_smoothingWindow ?? 0)
|
||||||
|
);
|
||||||
|
const resampleTargetPoints = $derived(resampleConfig?.targetPoints ? Number(resampleConfig.targetPoints) : null);
|
||||||
|
// Zoom in far enough that fewer than a quarter of the target point count is
|
||||||
|
// visible, and a finer resolution is requested from the backend.
|
||||||
|
const resampleTrigger = $derived(resampleTargetPoints ? Math.floor(resampleTargetPoints / 4) : null);
|
||||||
const useStatsSeries = $derived(!!statisticsSeries); // Display Stats Series By Default if Exists
|
const useStatsSeries = $derived(!!statisticsSeries); // Display Stats Series By Default if Exists
|
||||||
const thresholds = $derived(findJobAggregationThresholds(
|
const thresholds = $derived(findJobAggregationThresholds(
|
||||||
subClusterTopology,
|
subClusterTopology,
|
||||||
@@ -135,18 +145,20 @@
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
// Y
|
// Y
|
||||||
|
// Smooth every series of a stats plot: smoothing only some of them would
|
||||||
|
// break the min <= mid <= max invariant the plot bands rely on.
|
||||||
if (useStatsSeries) {
|
if (useStatsSeries) {
|
||||||
pendingData.push(statisticsSeries.min);
|
pendingData.push(movingAverage(statisticsSeries.min, smoothing));
|
||||||
pendingData.push(statisticsSeries.max);
|
pendingData.push(movingAverage(statisticsSeries.max, smoothing));
|
||||||
if (usesMeanStatsSeries) {
|
if (usesMeanStatsSeries) {
|
||||||
pendingData.push(statisticsSeries.mean);
|
pendingData.push(movingAverage(statisticsSeries.mean, smoothing));
|
||||||
} else {
|
} else {
|
||||||
pendingData.push(statisticsSeries.median);
|
pendingData.push(movingAverage(statisticsSeries.median, smoothing));
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
for (let i = 0; i < series.length; i++) {
|
for (let i = 0; i < series.length; i++) {
|
||||||
pendingData.push(series[i].data);
|
pendingData.push(movingAverage(series[i].data, smoothing));
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
return pendingData;
|
return pendingData;
|
||||||
@@ -514,25 +526,20 @@
|
|||||||
(u, key) => { // If ZoomResample is Configured && Not System/Node View
|
(u, key) => { // If ZoomResample is Configured && Not System/Node View
|
||||||
if (resampleConfig && !forNode && key === 'x') {
|
if (resampleConfig && !forNode && key === 'x') {
|
||||||
const numX = (u.series[0].idxs[1] - u.series[0].idxs[0])
|
const numX = (u.series[0].idxs[1] - u.series[0].idxs[0])
|
||||||
if (numX <= resampleTrigger && timestep !== resampleMinimum) {
|
if (resampleTargetPoints && numX <= resampleTrigger) {
|
||||||
/* Get closest zoom level; prevents multiple iterative zoom requests for big zoom-steps (e.g. 600 -> 300 -> 120 -> 60) */
|
// Compute the resolution for the visible window
|
||||||
// Which resolution to theoretically request to achieve 30 or more visible data points:
|
const visibleDuration = (u.scales.x.max - u.scales.x.min);
|
||||||
const target = (numX * timestep) / resampleTrigger
|
let newRes = Math.ceil(visibleDuration / resampleTargetPoints / nativeTimestep) * nativeTimestep;
|
||||||
// Which configured resolution actually matches the closest to theoretical target:
|
if (newRes < nativeTimestep) newRes = nativeTimestep;
|
||||||
const closest = resampleResolutions.reduce(function(prev, curr) {
|
|
||||||
return (Math.abs(curr - target) < Math.abs(prev - target) ? curr : prev);
|
|
||||||
});
|
|
||||||
// Prevents non-required dispatches
|
// Prevents non-required dispatches
|
||||||
if (timestep !== closest) {
|
if (newRes && timestep !== newRes) {
|
||||||
// console.log('Dispatch: Zoom with Res from / to', timestep, closest)
|
|
||||||
onZoom({
|
onZoom({
|
||||||
newRes: closest,
|
newRes: newRes,
|
||||||
lastZoomState: u?.scales,
|
lastZoomState: u?.scales,
|
||||||
lastThreshold: thresholds?.normal
|
lastThreshold: thresholds?.normal
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// console.log('Dispatch: Zoom Update States')
|
|
||||||
onZoom({
|
onZoom({
|
||||||
lastZoomState: u?.scales,
|
lastZoomState: u?.scales,
|
||||||
lastThreshold: thresholds?.normal
|
lastThreshold: thresholds?.normal
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
/*
|
||||||
|
* Display-only smoothing for metric plots.
|
||||||
|
*
|
||||||
|
* This is deliberately separate from the backend resampling in cc-lib: those
|
||||||
|
* algorithms decimate (they reduce the point count and their output length is
|
||||||
|
* always shorter than their input), which makes them unusable as a filter.
|
||||||
|
* A moving average has to keep every point in place, so it lives here and runs
|
||||||
|
* after whatever downsampling the backend applied.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Centered, NaN-aware moving average. Preserves length and index alignment, so
|
||||||
|
* the caller's X array and all derived index math stay valid.
|
||||||
|
*
|
||||||
|
* Windows are given in data points, not seconds. Values that are null or NaN
|
||||||
|
* are skipped; a window containing nothing else yields NaN so uPlot keeps
|
||||||
|
* rendering the gap. At the series edges the window shrinks instead of
|
||||||
|
* producing NaN, which avoids introducing new gaps at the start and end.
|
||||||
|
*
|
||||||
|
* @param {Array<number|null>} data Input samples
|
||||||
|
* @param {number} window Window width in data points; <= 1 disables smoothing
|
||||||
|
* @returns {Array<number|null>} Smoothed copy, or `data` itself if disabled
|
||||||
|
*/
|
||||||
|
export function movingAverage(data, window) {
|
||||||
|
if (!data || data.length === 0) return data;
|
||||||
|
|
||||||
|
let w = Math.floor(Number(window));
|
||||||
|
if (!Number.isFinite(w) || w <= 1) return data;
|
||||||
|
if (w > data.length) w = data.length;
|
||||||
|
// Force odd width so the window stays centered on the sample.
|
||||||
|
if (w % 2 === 0) w -= 1;
|
||||||
|
if (w <= 1) return data;
|
||||||
|
|
||||||
|
const n = data.length;
|
||||||
|
const h = (w - 1) / 2;
|
||||||
|
const out = new Array(n);
|
||||||
|
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
const start = i - h < 0 ? 0 : i - h;
|
||||||
|
const end = i + h > n - 1 ? n - 1 : i + h;
|
||||||
|
let sum = 0;
|
||||||
|
let count = 0;
|
||||||
|
for (let j = start; j <= end; j++) {
|
||||||
|
const v = data[j];
|
||||||
|
if (v == null || Number.isNaN(v)) continue;
|
||||||
|
sum += v;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
out[i] = count === 0 ? NaN : sum / count;
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@
|
|||||||
ModalFooter,
|
ModalFooter,
|
||||||
Button,
|
Button,
|
||||||
ListGroup,
|
ListGroup,
|
||||||
|
Icon,
|
||||||
|
Tooltip
|
||||||
} from "@sveltestrap/sveltestrap";
|
} from "@sveltestrap/sveltestrap";
|
||||||
import { gql, getContextClient, mutationStore } from "@urql/svelte";
|
import { gql, getContextClient, mutationStore } from "@urql/svelte";
|
||||||
|
|
||||||
@@ -33,6 +35,8 @@
|
|||||||
presetMetrics = [],
|
presetMetrics = [],
|
||||||
cluster = null,
|
cluster = null,
|
||||||
subCluster = null,
|
subCluster = null,
|
||||||
|
maxClusters = null,
|
||||||
|
maxSubClusters = null,
|
||||||
footprintSelect = false,
|
footprintSelect = false,
|
||||||
configName,
|
configName,
|
||||||
globalMetrics,
|
globalMetrics,
|
||||||
@@ -86,20 +90,45 @@
|
|||||||
return availableMetrics;
|
return availableMetrics;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function printAvailabilityCount(metric, cluster) {
|
||||||
|
const avail = globalMetrics.find((gm) => gm.name === metric)?.availability
|
||||||
|
if (avail) {
|
||||||
|
if (!cluster) {
|
||||||
|
return `${avail.length} / ${maxClusters} Cluster`
|
||||||
|
} else {
|
||||||
|
const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters
|
||||||
|
if (subAvail) {
|
||||||
|
return `${subAvail.length} / ${maxSubClusters} SubCluster`
|
||||||
|
} else {
|
||||||
|
return `0 / ${maxSubClusters} SubCluster`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return `0 / ${maxClusters} Cluster`
|
||||||
|
}
|
||||||
|
|
||||||
function printAvailability(metric, cluster) {
|
function printAvailability(metric, cluster) {
|
||||||
const avail = globalMetrics.find((gm) => gm.name === metric)?.availability
|
const avail = globalMetrics.find((gm) => gm.name === metric)?.availability
|
||||||
if (avail) {
|
if (avail) {
|
||||||
if (!cluster) {
|
if (!cluster) {
|
||||||
return avail.map((av) => av.cluster).join(', ')
|
return avail.map((av) => av.cluster)
|
||||||
} else {
|
} else {
|
||||||
const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters
|
const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters
|
||||||
if (subAvail) {
|
if (subAvail) {
|
||||||
return subAvail.join(', ')
|
return subAvail
|
||||||
} else {
|
} else {
|
||||||
return `Not available for ${cluster}`
|
return [`Not available for ${cluster}`]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return [`Not available for ${cluster}`]
|
||||||
|
}
|
||||||
|
|
||||||
|
function printTooltip(metric) {
|
||||||
|
const toolt = globalMetrics.find((gm) => gm.name === metric)?.tooltip
|
||||||
|
if (toolt) {
|
||||||
|
return toolt
|
||||||
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,7 +201,7 @@
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<Modal {isOpen} toggle={() => (isOpen = !isOpen)}>
|
<Modal {isOpen} toggle={() => (isOpen = !isOpen)}>
|
||||||
<ModalHeader>Configure columns (Metric availability shown)</ModalHeader>
|
<ModalHeader>Configure columns</ModalHeader>
|
||||||
<ModalBody>
|
<ModalBody>
|
||||||
<ListGroup>
|
<ListGroup>
|
||||||
{#if footprintSelect}
|
{#if footprintSelect}
|
||||||
@@ -213,9 +242,34 @@
|
|||||||
/>
|
/>
|
||||||
{/if}
|
{/if}
|
||||||
{metric}
|
{metric}
|
||||||
<span style="float: right; text-align: justify;">
|
{#if maxClusters !== null || maxSubClusters !== null}
|
||||||
{printAvailability(metric, cluster)}
|
<span style="float: right;" class="ms-1">
|
||||||
|
<Button id={`${metric}-avail-info`} outline color="secondary" size="sm" class="ml-2">
|
||||||
|
<b>{ printAvailabilityCount(metric, cluster) }</b>
|
||||||
|
</Button>
|
||||||
|
<Tooltip target={`${metric}-avail-info`} placement="right">
|
||||||
|
<b>Availability</b>
|
||||||
|
<ul style="text-align: left; padding-left: 1.0rem; margin-bottom: 0.25rem;">
|
||||||
|
{#each printAvailability(metric, cluster) as avail}
|
||||||
|
<li>{avail}</li>
|
||||||
|
{/each}
|
||||||
|
</ul>
|
||||||
|
</Tooltip>
|
||||||
</span>
|
</span>
|
||||||
|
{/if}
|
||||||
|
{#if printTooltip(metric) !== ""}
|
||||||
|
<span style="float: right;">
|
||||||
|
<Button id={`${metric}-kind-info`} outline color="secondary" size="sm" class="ml-2">
|
||||||
|
<Icon name="info-square" />
|
||||||
|
</Button>
|
||||||
|
<Tooltip target={`${metric}-kind-info`} placement="right">
|
||||||
|
<b>Information</b>
|
||||||
|
<p style="text-align: left; margin-bottom: 0.25rem;">
|
||||||
|
{ printTooltip(metric) }
|
||||||
|
</p>
|
||||||
|
</Tooltip>
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
</li>
|
</li>
|
||||||
{/each}
|
{/each}
|
||||||
</ListGroup>
|
</ListGroup>
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ export function init(extraInitQuery = "") {
|
|||||||
name
|
name
|
||||||
scope
|
scope
|
||||||
footprint
|
footprint
|
||||||
|
tooltip
|
||||||
unit { base, prefix }
|
unit { base, prefix }
|
||||||
availability { cluster, subClusters }
|
availability { cluster, subClusters }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@
|
|||||||
<DropdownToggle nav caret class="dropdown-item py-1 px-2">
|
<DropdownToggle nav caret class="dropdown-item py-1 px-2">
|
||||||
{cn}
|
{cn}
|
||||||
</DropdownToggle>
|
</DropdownToggle>
|
||||||
<DropdownMenu>
|
<DropdownMenu style="max-height:75vh; overflow-y: auto;">
|
||||||
<DropdownItem class="py-1 px-2"
|
<DropdownItem class="py-1 px-2"
|
||||||
href={item.href + cn}
|
href={item.href + cn}
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -44,7 +44,6 @@
|
|||||||
const client = getContextClient();
|
const client = getContextClient();
|
||||||
const statsPattern = /(.*)-stat$/;
|
const statsPattern = /(.*)-stat$/;
|
||||||
const resampleConfig = getContext("resampling") || null;
|
const resampleConfig = getContext("resampling") || null;
|
||||||
const resampleDefault = resampleConfig ? Math.max(...resampleConfig.resolutions) : 0;
|
|
||||||
const subQuery = gql`
|
const subQuery = gql`
|
||||||
query ($dbid: ID!, $selectedMetrics: [String!]!, $selectedScopes: [MetricScope!]!, $selectedResolution: Int) {
|
query ($dbid: ID!, $selectedMetrics: [String!]!, $selectedScopes: [MetricScope!]!, $selectedResolution: Int) {
|
||||||
singleUpdate: jobMetrics(id: $dbid, metrics: $selectedMetrics, scopes: $selectedScopes, resolution: $selectedResolution) {
|
singleUpdate: jobMetrics(id: $dbid, metrics: $selectedMetrics, scopes: $selectedScopes, resolution: $selectedResolution) {
|
||||||
@@ -78,7 +77,9 @@
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
/* State Init */
|
/* State Init */
|
||||||
let selectedResolution = $state(resampleDefault);
|
// null lets the backend resolve the resolution from the configured resample
|
||||||
|
// policy; zoom interactions override it with an explicit value.
|
||||||
|
let selectedResolution = $state(null);
|
||||||
let selectedHost = $state(null);
|
let selectedHost = $state(null);
|
||||||
let zoomState = $state(null);
|
let zoomState = $state(null);
|
||||||
let thresholdState = $state(null);
|
let thresholdState = $state(null);
|
||||||
|
|||||||
+8
-1
@@ -72,6 +72,9 @@ type PlotConfiguration struct {
|
|||||||
PlotsPerRow int `json:"plots-per-row"`
|
PlotsPerRow int `json:"plots-per-row"`
|
||||||
LineWidth int `json:"line-width"`
|
LineWidth int `json:"line-width"`
|
||||||
ColorScheme []string `json:"color-scheme"`
|
ColorScheme []string `json:"color-scheme"`
|
||||||
|
ResampleAlgo string `json:"resample-algo"`
|
||||||
|
ResamplePolicy string `json:"resample-policy"`
|
||||||
|
SmoothingWindow int `json:"smoothing-window"`
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -124,6 +127,7 @@ var UIDefaults = WebConfig{
|
|||||||
PlotsPerRow: 3,
|
PlotsPerRow: 3,
|
||||||
LineWidth: 3,
|
LineWidth: 3,
|
||||||
ColorScheme: []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
|
ColorScheme: []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
|
||||||
|
SmoothingWindow: 3,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,6 +174,9 @@ func Init(rawConfig json.RawMessage) error {
|
|||||||
UIDefaultsMap["plotConfiguration_plotsPerRow"] = UIDefaults.PlotConfiguration.PlotsPerRow
|
UIDefaultsMap["plotConfiguration_plotsPerRow"] = UIDefaults.PlotConfiguration.PlotsPerRow
|
||||||
UIDefaultsMap["plotConfiguration_lineWidth"] = UIDefaults.PlotConfiguration.LineWidth
|
UIDefaultsMap["plotConfiguration_lineWidth"] = UIDefaults.PlotConfiguration.LineWidth
|
||||||
UIDefaultsMap["plotConfiguration_colorScheme"] = UIDefaults.PlotConfiguration.ColorScheme
|
UIDefaultsMap["plotConfiguration_colorScheme"] = UIDefaults.PlotConfiguration.ColorScheme
|
||||||
|
UIDefaultsMap["plotConfiguration_resampleAlgo"] = UIDefaults.PlotConfiguration.ResampleAlgo
|
||||||
|
UIDefaultsMap["plotConfiguration_resamplePolicy"] = UIDefaults.PlotConfiguration.ResamplePolicy
|
||||||
|
UIDefaultsMap["plotConfiguration_smoothingWindow"] = UIDefaults.PlotConfiguration.SmoothingWindow
|
||||||
|
|
||||||
for _, c := range UIDefaults.MetricConfig.Clusters {
|
for _, c := range UIDefaults.MetricConfig.Clusters {
|
||||||
if c.JobListMetrics != nil {
|
if c.JobListMetrics != nil {
|
||||||
@@ -290,7 +297,7 @@ type Page struct {
|
|||||||
FilterPresets map[string]any // For pages with the Filter component, this can be used to set initial filters.
|
FilterPresets map[string]any // For pages with the Filter component, this can be used to set initial filters.
|
||||||
Infos map[string]any // For generic use (e.g. username for /monitoring/user/<id>, job id for /monitoring/job/<id>)
|
Infos map[string]any // For generic use (e.g. username for /monitoring/user/<id>, job id for /monitoring/job/<id>)
|
||||||
Config map[string]any // UI settings for the currently logged in user (e.g. line width, ...)
|
Config map[string]any // UI settings for the currently logged in user (e.g. line width, ...)
|
||||||
Resampling *config.ResampleConfig // If not nil, defines resampling trigger and resolutions
|
Resampling *config.ResampleConfig // If not nil, defines the target point count for zoom resampling
|
||||||
Redirect string // The originally requested URL, for intermediate login handling
|
Redirect string // The originally requested URL, for intermediate login handling
|
||||||
FooterLinks FooterLinks // Resolved legal links for the site footer
|
FooterLinks FooterLinks // Resolved legal links for the site footer
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user