diff --git a/.gitignore b/.gitignore index c09976d..f2ce56d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ aditya.creds test.creds -./config.json +/config.json migrateTimestamps.pl test_ccms_api.sh diff --git a/configs/config-nats.json b/configs/config-nats.json new file mode 100644 index 0000000..f38bc3d --- /dev/null +++ b/configs/config-nats.json @@ -0,0 +1,196 @@ +{ + "metrics": { + "debug_metric": { + "frequency": 60, + "aggregation": "avg" + }, + "clock": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_idle": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_iowait": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_irq": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_system": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_user": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_mem_util": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_temp": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_sm_clock": { + "frequency": 60, + "aggregation": "avg" + }, + "acc_utilization": { + "frequency": 60, + "aggregation": "avg" + }, + "acc_mem_used": { + "frequency": 60, + "aggregation": "sum" + }, + "acc_power": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_any": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_dp": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_sp": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_recv": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_xmit": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_recv_pkts": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_xmit_pkts": { + "frequency": 60, + "aggregation": "sum" + }, + "cpu_power": { + "frequency": 60, + "aggregation": "sum" + }, + "core_power": { + "frequency": 60, + "aggregation": "sum" + }, + "mem_power": { + "frequency": 60, + "aggregation": "sum" + }, + "ipc": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_load": { + "frequency": 60, + "aggregation": null + }, + "lustre_close": { + "frequency": 60, + "aggregation": null + }, + "lustre_open": { + "frequency": 60, + "aggregation": null + }, + "lustre_statfs": { + "frequency": 60, + "aggregation": null + }, + "lustre_read_bytes": { + "frequency": 60, + "aggregation": null + }, + "lustre_write_bytes": { + "frequency": 60, + "aggregation": null + }, + "net_bw": { + "frequency": 60, + "aggregation": null + }, + "file_bw": { + "frequency": 60, + "aggregation": null + }, + "mem_bw": { + "frequency": 60, + "aggregation": "sum" + }, + "mem_cached": { + "frequency": 60, + "aggregation": null + }, + "mem_used": { + "frequency": 60, + "aggregation": null + }, + "net_bytes_in": { + "frequency": 60, + "aggregation": null + }, + "net_bytes_out": { + "frequency": 60, + "aggregation": null + }, + "nfs4_read": { + "frequency": 60, + "aggregation": null + }, + "nfs4_total": { + "frequency": 60, + "aggregation": null + }, + "nfs4_write": { + "frequency": 60, + "aggregation": null + }, + "vectorization_ratio": { + "frequency": 60, + "aggregation": "avg" + } + }, + "checkpoints": { + "interval": "12h", + "directory": "./var/checkpoints", + "restore": "48h" + }, + "archive": { + "interval": "50h", + "directory": "./var/archive" + }, + "http-api": { + "address": "localhost:8082", + "https-cert-file": null, + "https-key-file": null + }, + "retention-in-memory": "48h", + "nats": [ + { + "address": "nats://localhost:4222", + "creds-file-path": "test.creds", + "subscriptions": [ + { + "subscribe-to": "ee-hpc-nats", + "cluster-tag": "fritz2" + } + ] + } + ], + "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=" +} \ No newline at end of file diff --git a/configs/config.json b/configs/config.json index f38bc3d..29d4d28 100644 --- a/configs/config.json +++ b/configs/config.json @@ -180,17 +180,6 @@ "https-key-file": null }, "retention-in-memory": "48h", - "nats": [ - { - "address": "nats://localhost:4222", - "creds-file-path": "test.creds", - "subscriptions": [ - { - "subscribe-to": "ee-hpc-nats", - "cluster-tag": "fritz2" - } - ] - } - ], + "nats": null, "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=" } \ No newline at end of file diff --git a/pkg/resampler/resampler.go b/pkg/resampler/resampler.go index 59288c5..c641670 100644 --- a/pkg/resampler/resampler.go +++ b/pkg/resampler/resampler.go @@ -8,20 +8,20 @@ import ( "github.com/ClusterCockpit/cc-metric-store/internal/util" ) -func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, error) { - if old_frequency == 0 || new_frequency == 0 { - return nil, errors.New("either old or new frequency is set to 0") +func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) { + if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { + return data, old_frequency, nil } if new_frequency%old_frequency != 0 { - return nil, errors.New("new sampling frequency should be multiple of the old frequency") + return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency") } var step int = int(new_frequency / old_frequency) var new_data_length = len(data) / step if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) { - return data, nil + return data, old_frequency, nil } new_data := make([]util.Float, new_data_length) @@ -30,14 +30,14 @@ func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64 new_data[i] = data[i*step] } - return new_data, nil + return new_data, new_frequency, nil } // Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf // Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go func LargestTriangleThreeBucket(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) { - if old_frequency == 0 || new_frequency == 0 { + if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { return data, old_frequency, nil }