From 6aca448c18788e4def276a35324dc5fedfd07032 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 26 Jan 2026 09:59:29 +0100 Subject: [PATCH] Port to newest cclib. Use metricstore as library. --- cmd/cc-metric-store/cli.go | 25 + cmd/cc-metric-store/main.go | 250 ++++---- cmd/cc-metric-store/server.go | 142 +++++ configs/config.json | 384 ++++++------ go.mod | 78 ++- go.sum | 207 +++++-- internal/api/authentication.go | 5 + internal/api/docs.go | 5 + internal/api/lineprotocol.go | 350 ----------- internal/api/{api.go => metricstore.go} | 324 +++++----- internal/api/server.go | 35 +- internal/avro/avroCheckpoint.go | 474 --------------- internal/avro/avroHelper.go | 80 --- internal/avro/avroStruct.go | 161 ----- internal/config/config.go | 92 +-- internal/config/metricSchema.go | 135 +++++ internal/config/schema.go | 135 +++++ internal/config/validate.go | 29 + internal/memorystore/archive.go | 185 ------ internal/memorystore/buffer.go | 233 ------- internal/memorystore/checkpoint.go | 767 ------------------------ internal/memorystore/debug.go | 107 ---- internal/memorystore/healthcheck.go | 88 --- internal/memorystore/level.go | 187 ------ internal/memorystore/memorystore.go | 373 ------------ internal/memorystore/stats.go | 120 ---- internal/runtimeEnv/setup.go | 140 ----- internal/util/float.go | 76 --- internal/util/selector.go | 51 -- pkg/resampler/resampler.go | 122 ---- pkg/resampler/util.go | 35 -- tools.go | 8 - 32 files changed, 1215 insertions(+), 4188 deletions(-) create mode 100644 cmd/cc-metric-store/cli.go create mode 100644 cmd/cc-metric-store/server.go delete mode 100644 internal/api/lineprotocol.go rename internal/api/{api.go => metricstore.go} (74%) delete mode 100644 internal/avro/avroCheckpoint.go delete mode 100644 internal/avro/avroHelper.go delete mode 100644 internal/avro/avroStruct.go create mode 100644 internal/config/metricSchema.go create mode 100644 internal/config/schema.go create mode 100644 internal/config/validate.go delete mode 100644 internal/memorystore/archive.go delete mode 100644 internal/memorystore/buffer.go delete mode 100644 internal/memorystore/checkpoint.go delete mode 100644 internal/memorystore/debug.go delete mode 100644 internal/memorystore/healthcheck.go delete mode 100644 internal/memorystore/level.go delete mode 100644 internal/memorystore/memorystore.go delete mode 100644 internal/memorystore/stats.go delete mode 100644 internal/runtimeEnv/setup.go delete mode 100644 internal/util/float.go delete mode 100644 internal/util/selector.go delete mode 100644 pkg/resampler/resampler.go delete mode 100644 pkg/resampler/util.go delete mode 100644 tools.go diff --git a/cmd/cc-metric-store/cli.go b/cmd/cc-metric-store/cli.go new file mode 100644 index 0000000..7e2f205 --- /dev/null +++ b/cmd/cc-metric-store/cli.go @@ -0,0 +1,25 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package main provides the entry point for the ClusterCockpit metric store server. +// This file defines all command-line flags and their default values. +package main + +import "flag" + +var ( + flagGops, flagVersion, flagDev, flagLogDateTime bool + flagConfigFile, flagLogLevel string +) + +func cliInit() { + flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") + flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI") + flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") + flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") + flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`") + flag.Parse() +} diff --git a/cmd/cc-metric-store/main.go b/cmd/cc-metric-store/main.go index 7fa7f2b..9a22ba1 100644 --- a/cmd/cc-metric-store/main.go +++ b/cmd/cc-metric-store/main.go @@ -1,32 +1,25 @@ // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. +// All rights reserved. This file is part of cc-metric-store. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package main import ( "context" - "crypto/tls" - "flag" "fmt" - "log" - "net" - "net/http" "os" "os/signal" - "runtime" - "runtime/debug" "sync" "syscall" - "time" - "github.com/ClusterCockpit/cc-metric-store/internal/api" - "github.com/ClusterCockpit/cc-metric-store/internal/avro" + "github.com/ClusterCockpit/cc-backend/pkg/metricstore" + ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/nats" + "github.com/ClusterCockpit/cc-lib/v2/runtime" "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" - "github.com/ClusterCockpit/cc-metric-store/internal/runtimeEnv" "github.com/google/gops/agent" - httpSwagger "github.com/swaggo/http-swagger" ) var ( @@ -35,147 +28,148 @@ var ( version string ) -func main() { - var configFile string - var enableGopsAgent, flagVersion, flagDev bool - flag.StringVar(&configFile, "config", "./config.json", "configuration file") - flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent") - flag.BoolVar(&flagDev, "dev", false, "Enable development Swagger UI component") - flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") - flag.Parse() +func printVersion() { + fmt.Printf("Version:\t%s\n", version) + fmt.Printf("Git hash:\t%s\n", commit) + fmt.Printf("Build time:\t%s\n", date) +} - if flagVersion { - fmt.Printf("Version:\t%s\n", version) - fmt.Printf("Git hash:\t%s\n", commit) - fmt.Printf("Build time:\t%s\n", date) - os.Exit(0) +func initGops() error { + if !flagGops && !config.Keys.Debug.EnableGops { + return nil } - startupTime := time.Now() - config.Init(configFile) - memorystore.Init(config.Keys.Metrics) - ms := memorystore.GetMemoryStore() + if err := agent.Listen(agent.Options{}); err != nil { + return fmt.Errorf("starting gops agent: %w", err) + } + return nil +} - if enableGopsAgent || config.Keys.Debug.EnableGops { - if err := agent.Listen(agent.Options{}); err != nil { - log.Fatal(err) - } +func initConfiguration() error { + ccconf.Init(flagConfigFile) + + cfg := ccconf.GetPackageConfig("main") + if cfg == nil { + return fmt.Errorf("main configuration must be present") } - d, err := time.ParseDuration(config.Keys.Checkpoints.Restore) - if err != nil { - log.Fatal(err) + config.Init(cfg) + return nil +} + +func initSubsystems() error { + // Initialize nats client + natsConfig := ccconf.GetPackageConfig("nats") + if err := nats.Init(natsConfig); err != nil { + cclog.Warnf("initializing (optional) nats client: %s", err.Error()) } + nats.Connect() - restoreFrom := startupTime.Add(-d) - log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339)) - files, err := ms.FromCheckpointFiles(config.Keys.Checkpoints.RootDir, restoreFrom.Unix()) - loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB - if err != nil { - log.Fatalf("Loading checkpoints failed: %s\n", err.Error()) - } else { - log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds()) - } - - // Try to use less memory by forcing a GC run here and then - // lowering the target percentage. The default of 100 means - // that only once the ratio of new allocations execeds the - // previously active heap, a GC is triggered. - // Forcing a GC here will set the "previously active heap" - // to a minumum. - runtime.GC() - if loadedData > 1000 && os.Getenv("GOGC") == "" { - debug.SetGCPercent(10) - } - - ctx, shutdown := context.WithCancel(context.Background()) + return nil +} +func runServer(ctx context.Context) error { var wg sync.WaitGroup - wg.Add(4) - memorystore.Retention(&wg, ctx) - memorystore.Checkpointing(&wg, ctx) - memorystore.Archiving(&wg, ctx) - avro.DataStaging(&wg, ctx) - - r := http.NewServeMux() - api.MountRoutes(r) - - if flagDev { - log.Print("Enable Swagger UI!") - r.HandleFunc("GET /swagger/", httpSwagger.Handler( - httpSwagger.URL("http://"+config.Keys.HttpConfig.Address+"/swagger/doc.json"))) - } - - server := &http.Server{ - Handler: r, - Addr: config.Keys.HttpConfig.Address, - WriteTimeout: 30 * time.Second, - ReadTimeout: 30 * time.Second, - } - - // Start http or https server - listener, err := net.Listen("tcp", config.Keys.HttpConfig.Address) - if err != nil { - log.Fatalf("starting http listener failed: %v", err) - } - - if config.Keys.HttpConfig.CertFile != "" && config.Keys.HttpConfig.KeyFile != "" { - cert, err := tls.LoadX509KeyPair(config.Keys.HttpConfig.CertFile, config.Keys.HttpConfig.KeyFile) - if err != nil { - log.Fatalf("loading X509 keypair failed: %v", err) - } - listener = tls.NewListener(listener, &tls.Config{ - Certificates: []tls.Certificate{cert}, - CipherSuites: []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - }, - MinVersion: tls.VersionTLS12, - PreferServerCipherSuites: true, - }) - fmt.Printf("HTTPS server listening at %s...\n", config.Keys.HttpConfig.Address) + // Initialize metric store if configuration is provided + mscfg := ccconf.GetPackageConfig("metric-store") + if mscfg != nil { + metricstore.Init(mscfg, &wg) } else { - fmt.Printf("HTTP server listening at %s...\n", config.Keys.HttpConfig.Address) + return fmt.Errorf("missing metricstore configuration") } + // Initialize HTTP server + srv, err := NewServer(version, commit, date) + if err != nil { + return fmt.Errorf("creating server: %w", err) + } + + // Channel to collect errors from server + errChan := make(chan error, 1) + + // Start HTTP server wg.Add(1) go func() { defer wg.Done() - if err = server.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatalf("starting server failed: %v", err) + if err := srv.Start(ctx); err != nil { + errChan <- err } }() + // Handle shutdown signals wg.Add(1) sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { defer wg.Done() - <-sigs - runtimeEnv.SystemdNotifiy(false, "Shutting down ...") - server.Shutdown(context.Background()) - shutdown() - memorystore.Shutdown() + select { + case <-sigs: + cclog.Info("Shutdown signal received") + case <-ctx.Done(): + } + + runtime.SystemdNotify(false, "Shutting down ...") + srv.Shutdown(ctx) }() - if config.Keys.Nats != nil { - for _, natsConf := range config.Keys.Nats { - // TODO: When multiple nats configs share a URL, do a single connect. - wg.Add(1) - nc := natsConf - go func() { - // err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx) - err := api.ReceiveNats(nc, ms, 1, ctx) - if err != nil { - log.Fatal(err) - } - wg.Done() - }() - } + runtime.SystemdNotify(true, "running") + + // Wait for completion or errors + go func() { + wg.Wait() + close(errChan) + }() + + // Wait for either server startup error or shutdown completion + if err := <-errChan; err != nil { + return err } - runtimeEnv.SystemdNotifiy(true, "running") - wg.Wait() - log.Print("Graceful shutdown completed!") + cclog.Print("Graceful shutdown completed!") + return nil +} + +func run() error { + cliInit() + + if flagVersion { + printVersion() + return nil + } + + // Initialize logger + cclog.Init(flagLogLevel, flagLogDateTime) + + // Initialize gops agent + if err := initGops(); err != nil { + return err + } + + // Initialize subsystems in dependency order: + // 1. Load configuration from config.json + // 2. Initialize subsystems like nats + + // Load configuration + if err := initConfiguration(); err != nil { + return err + } + + // Initialize subsystems (nats, etc.) + if err := initSubsystems(); err != nil { + return err + } + + // Run server with context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + return runServer(ctx) +} + +func main() { + if err := run(); err != nil { + cclog.Error(err.Error()) + os.Exit(1) + } } diff --git a/cmd/cc-metric-store/server.go b/cmd/cc-metric-store/server.go new file mode 100644 index 0000000..5e5ffd4 --- /dev/null +++ b/cmd/cc-metric-store/server.go @@ -0,0 +1,142 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package main provides the entry point for the ClusterCockpit metric store server. +// This file contains HTTP server setup, routing configuration. +package main + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "time" + + "github.com/ClusterCockpit/cc-backend/pkg/metricstore" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/nats" + "github.com/ClusterCockpit/cc-lib/v2/runtime" + "github.com/ClusterCockpit/cc-metric-store/internal/api" + "github.com/ClusterCockpit/cc-metric-store/internal/config" + httpSwagger "github.com/swaggo/http-swagger" +) + +// Server encapsulates the HTTP server state and dependencies +type Server struct { + router *http.ServeMux + server *http.Server +} + +// NewServer creates and initializes a new Server instance +func NewServer(version, commit, buildDate string) (*Server, error) { + s := &Server{ + router: http.NewServeMux(), + } + + if err := s.init(); err != nil { + return nil, err + } + + return s, nil +} + +func (s *Server) init() error { + api.MountRoutes(s.router) + + if flagDev { + cclog.Print("Enable Swagger UI!") + s.router.HandleFunc("GET /swagger/", httpSwagger.Handler( + httpSwagger.URL("http://"+config.Keys.Address+"/swagger/doc.json"))) + } + + return nil +} + +// Server timeout defaults (in seconds) +const ( + defaultReadTimeout = 30 + defaultWriteTimeout = 30 +) + +func (s *Server) Start(ctx context.Context) error { + // Use configurable timeouts with defaults + readTimeout := time.Duration(defaultReadTimeout) * time.Second + writeTimeout := time.Duration(defaultWriteTimeout) * time.Second + + s.server = &http.Server{ + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + Handler: s.router, + Addr: config.Keys.Address, + } + + // Start http or https server + listener, err := net.Listen("tcp", config.Keys.Address) + if err != nil { + return fmt.Errorf("starting listener on '%s': %w", config.Keys.Address, err) + } + + if config.Keys.CertFile != "" && config.Keys.KeyFile != "" { + cert, err := tls.LoadX509KeyPair( + config.Keys.CertFile, config.Keys.KeyFile) + if err != nil { + return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err) + } + listener = tls.NewListener(listener, &tls.Config{ + Certificates: []tls.Certificate{cert}, + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + }) + cclog.Infof("HTTPS server listening at %s...", config.Keys.Address) + } else { + cclog.Infof("HTTP server listening at %s...", config.Keys.Address) + } + + // Because this program will want to bind to a privileged port (like 80), the listener must + // be established first, then the user can be changed, and after that, + // the actual http server can be started. + if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { + return fmt.Errorf("dropping privileges: %w", err) + } + + // Handle context cancellation for graceful shutdown + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := s.server.Shutdown(shutdownCtx); err != nil { + cclog.Errorf("Server shutdown error: %v", err) + } + }() + + if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed { + return fmt.Errorf("server failed: %w", err) + } + return nil +} + +func (s *Server) Shutdown(ctx context.Context) { + // Create a shutdown context with timeout + shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + nc := nats.GetClient() + if nc != nil { + nc.Close() + } + + // First shut down the server gracefully (waiting for all ongoing requests) + if err := s.server.Shutdown(shutdownCtx); err != nil { + cclog.Errorf("Server shutdown error: %v", err) + } + + // Archive all the metric store data + metricstore.Shutdown() +} diff --git a/configs/config.json b/configs/config.json index 29d4d28..6211fa0 100644 --- a/configs/config.json +++ b/configs/config.json @@ -1,185 +1,203 @@ { - "metrics": { - "debug_metric": { - "frequency": 60, - "aggregation": "avg" - }, - "clock": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_idle": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_iowait": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_irq": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_system": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_user": { - "frequency": 60, - "aggregation": "avg" - }, - "nv_mem_util": { - "frequency": 60, - "aggregation": "avg" - }, - "nv_temp": { - "frequency": 60, - "aggregation": "avg" - }, - "nv_sm_clock": { - "frequency": 60, - "aggregation": "avg" - }, - "acc_utilization": { - "frequency": 60, - "aggregation": "avg" - }, - "acc_mem_used": { - "frequency": 60, - "aggregation": "sum" - }, - "acc_power": { - "frequency": 60, - "aggregation": "sum" - }, - "flops_any": { - "frequency": 60, - "aggregation": "sum" - }, - "flops_dp": { - "frequency": 60, - "aggregation": "sum" - }, - "flops_sp": { - "frequency": 60, - "aggregation": "sum" - }, - "ib_recv": { - "frequency": 60, - "aggregation": "sum" - }, - "ib_xmit": { - "frequency": 60, - "aggregation": "sum" - }, - "ib_recv_pkts": { - "frequency": 60, - "aggregation": "sum" - }, - "ib_xmit_pkts": { - "frequency": 60, - "aggregation": "sum" - }, - "cpu_power": { - "frequency": 60, - "aggregation": "sum" - }, - "core_power": { - "frequency": 60, - "aggregation": "sum" - }, - "mem_power": { - "frequency": 60, - "aggregation": "sum" - }, - "ipc": { - "frequency": 60, - "aggregation": "avg" - }, - "cpu_load": { - "frequency": 60, - "aggregation": null - }, - "lustre_close": { - "frequency": 60, - "aggregation": null - }, - "lustre_open": { - "frequency": 60, - "aggregation": null - }, - "lustre_statfs": { - "frequency": 60, - "aggregation": null - }, - "lustre_read_bytes": { - "frequency": 60, - "aggregation": null - }, - "lustre_write_bytes": { - "frequency": 60, - "aggregation": null - }, - "net_bw": { - "frequency": 60, - "aggregation": null - }, - "file_bw": { - "frequency": 60, - "aggregation": null - }, - "mem_bw": { - "frequency": 60, - "aggregation": "sum" - }, - "mem_cached": { - "frequency": 60, - "aggregation": null - }, - "mem_used": { - "frequency": 60, - "aggregation": null - }, - "net_bytes_in": { - "frequency": 60, - "aggregation": null - }, - "net_bytes_out": { - "frequency": 60, - "aggregation": null - }, - "nfs4_read": { - "frequency": 60, - "aggregation": null - }, - "nfs4_total": { - "frequency": 60, - "aggregation": null - }, - "nfs4_write": { - "frequency": 60, - "aggregation": null - }, - "vectorization_ratio": { - "frequency": 60, - "aggregation": "avg" - } - }, - "checkpoints": { - "interval": "12h", - "directory": "./var/checkpoints", - "restore": "48h" - }, - "archive": { - "interval": "50h", - "directory": "./var/archive" - }, - "http-api": { - "address": "localhost:8082", - "https-cert-file": null, - "https-key-file": null - }, - "retention-in-memory": "48h", - "nats": null, + "main": { + "addr": "0.0.0.0:443", + "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem", + "https-key-file": "/etc/letsencrypt/live/url/privkey.pem", "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0=" -} \ No newline at end of file + }, + "metrics": { + "debug_metric": { + "frequency": 60, + "aggregation": "avg" + }, + "clock": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_idle": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_iowait": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_irq": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_system": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_user": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_mem_util": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_temp": { + "frequency": 60, + "aggregation": "avg" + }, + "nv_sm_clock": { + "frequency": 60, + "aggregation": "avg" + }, + "acc_utilization": { + "frequency": 60, + "aggregation": "avg" + }, + "acc_mem_used": { + "frequency": 60, + "aggregation": "sum" + }, + "acc_power": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_any": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_dp": { + "frequency": 60, + "aggregation": "sum" + }, + "flops_sp": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_recv": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_xmit": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_recv_pkts": { + "frequency": 60, + "aggregation": "sum" + }, + "ib_xmit_pkts": { + "frequency": 60, + "aggregation": "sum" + }, + "cpu_power": { + "frequency": 60, + "aggregation": "sum" + }, + "core_power": { + "frequency": 60, + "aggregation": "sum" + }, + "mem_power": { + "frequency": 60, + "aggregation": "sum" + }, + "ipc": { + "frequency": 60, + "aggregation": "avg" + }, + "cpu_load": { + "frequency": 60, + "aggregation": null + }, + "lustre_close": { + "frequency": 60, + "aggregation": null + }, + "lustre_open": { + "frequency": 60, + "aggregation": null + }, + "lustre_statfs": { + "frequency": 60, + "aggregation": null + }, + "lustre_read_bytes": { + "frequency": 60, + "aggregation": null + }, + "lustre_write_bytes": { + "frequency": 60, + "aggregation": null + }, + "net_bw": { + "frequency": 60, + "aggregation": null + }, + "file_bw": { + "frequency": 60, + "aggregation": null + }, + "mem_bw": { + "frequency": 60, + "aggregation": "sum" + }, + "mem_cached": { + "frequency": 60, + "aggregation": null + }, + "mem_used": { + "frequency": 60, + "aggregation": null + }, + "net_bytes_in": { + "frequency": 60, + "aggregation": null + }, + "net_bytes_out": { + "frequency": 60, + "aggregation": null + }, + "nfs4_read": { + "frequency": 60, + "aggregation": null + }, + "nfs4_total": { + "frequency": 60, + "aggregation": null + }, + "nfs4_write": { + "frequency": 60, + "aggregation": null + }, + "vectorization_ratio": { + "frequency": 60, + "aggregation": "avg" + } + }, + "nats": { + "address": "nats://0.0.0.0:4222", + "username": "root", + "password": "root" + }, + "metric-store": { + "checkpoints": { + "interval": "12h", + "directory": "./var/checkpoints" + }, + "memory-cap": 100, + "retention-in-memory": "48h", + "cleanup": { + "mode": "archive", + "interval": "48h", + "directory": "./var/archive" + }, + "nats-subscriptions": [ + { + "subscribe-to": "hpc-nats", + "cluster-tag": "fritz" + }, + { + "subscribe-to": "hpc-nats", + "cluster-tag": "alex" + } + ] + } +} + diff --git a/go.mod b/go.mod index 100323d..8580571 100644 --- a/go.mod +++ b/go.mod @@ -1,39 +1,71 @@ module github.com/ClusterCockpit/cc-metric-store -go 1.23.0 +go 1.24.0 require ( + github.com/ClusterCockpit/cc-backend v1.4.4-0.20260126082752-084d00cb0d0c + github.com/ClusterCockpit/cc-lib/v2 v2.2.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/gops v0.3.28 github.com/influxdata/line-protocol/v2 v2.2.1 - github.com/linkedin/goavro/v2 v2.13.1 - github.com/nats-io/nats.go v1.47.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/swaggo/http-swagger v1.3.4 - github.com/swaggo/swag v1.16.3 + github.com/swaggo/swag v1.16.6 ) require ( github.com/KyleBanks/depth v1.2.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/spec v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/golang/snappy v0.0.1 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect + github.com/aws/smithy-go v1.24.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-openapi/jsonpointer v0.22.3 // indirect + github.com/go-openapi/jsonreference v0.21.3 // indirect + github.com/go-openapi/spec v0.22.1 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/linkedin/goavro/v2 v2.14.1 // indirect + github.com/mattn/go-sqlite3 v1.14.33 // indirect + github.com/nats-io/nats.go v1.47.0 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/swaggo/files v1.0.1 // indirect - github.com/urfave/cli/v2 v2.27.1 // indirect - github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect - golang.org/x/tools v0.22.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + github.com/urfave/cli/v2 v2.27.7 // indirect + github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.40.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 9b16981..6e598b3 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,116 @@ +github.com/ClusterCockpit/cc-backend v1.4.4-0.20260126082752-084d00cb0d0c h1:rN1M3afMjlW4GUsa5jiR5OKA23IVpoeMrkbVlpk2sWw= +github.com/ClusterCockpit/cc-backend v1.4.4-0.20260126082752-084d00cb0d0c/go.mod h1:RDlfymO/WgrcZ1eDxGpur2jTEFoMA8BfJUvV+Heb+E4= +github.com/ClusterCockpit/cc-lib/v2 v2.2.0 h1:gqMsh7zsJMUhaXviXzaZ3gqXcLVgerjRJHzIcwX4FmQ= +github.com/ClusterCockpit/cc-lib/v2 v2.2.0/go.mod h1:JuxMAuEOaLLNEnnL9U3ejha8kMvsSatLdKPZEgJw6iw= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw= +github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= +github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= +github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= -github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-openapi/jsonpointer v0.22.3 h1:dKMwfV4fmt6Ah90zloTbUKWMD+0he+12XYAsPotrkn8= +github.com/go-openapi/jsonpointer v0.22.3/go.mod h1:0lBbqeRsQ5lIanv3LHZBrmRGHLHcQoOXQnf88fHlGWo= +github.com/go-openapi/jsonreference v0.21.3 h1:96Dn+MRPa0nYAR8DR1E03SblB5FJvh7W6krPI0Z7qMc= +github.com/go-openapi/jsonreference v0.21.3/go.mod h1:RqkUP0MrLf37HqxZxrIAtTWW4ZJIK1VzduhXYBEeGc4= +github.com/go-openapi/spec v0.22.1 h1:beZMa5AVQzRspNjvhe5aG1/XyBSMeX1eEOs7dMoXh/k= +github.com/go-openapi/spec v0.22.1/go.mod h1:c7aeIQT175dVowfp7FeCvXXnjN/MrpaONStibD2WtDA= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark= github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4= +github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= +github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= +github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98= github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig= github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo= @@ -36,10 +118,8 @@ github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY= github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE= github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -47,65 +127,86 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linkedin/goavro/v2 v2.13.1 h1:4qZ5M0QzQFDRqccsroJlgOJznqAS/TpdvXg55h429+I= -github.com/linkedin/goavro/v2 v2.13.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI= +github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= +github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0= +github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= +github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= -github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= -github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= +github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= +github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= +github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= +github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= +github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -113,15 +214,17 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -130,5 +233,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/api/authentication.go b/internal/api/authentication.go index 6d8db4f..3f91054 100644 --- a/internal/api/authentication.go +++ b/internal/api/authentication.go @@ -1,3 +1,8 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package api import ( diff --git a/internal/api/docs.go b/internal/api/docs.go index c7bb10a..6af16ab 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1,3 +1,8 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + // Package api Code generated by swaggo/swag. DO NOT EDIT package api diff --git a/internal/api/lineprotocol.go b/internal/api/lineprotocol.go deleted file mode 100644 index 73e697a..0000000 --- a/internal/api/lineprotocol.go +++ /dev/null @@ -1,350 +0,0 @@ -package api - -import ( - "context" - "errors" - "fmt" - "log" - "net" - "sync" - "time" - - "github.com/ClusterCockpit/cc-metric-store/internal/avro" - "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" - "github.com/ClusterCockpit/cc-metric-store/internal/util" - "github.com/influxdata/line-protocol/v2/lineprotocol" - "github.com/nats-io/nats.go" -) - -// Each connection is handled in it's own goroutine. This is a blocking function. -func ReceiveRaw(ctx context.Context, - listener net.Listener, - handleLine func(*lineprotocol.Decoder, string) error, -) error { - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - <-ctx.Done() - if err := listener.Close(); err != nil { - log.Printf("listener.Close(): %s", err.Error()) - } - }() - - for { - conn, err := listener.Accept() - if err != nil { - if errors.Is(err, net.ErrClosed) { - break - } - - log.Printf("listener.Accept(): %s", err.Error()) - } - - wg.Add(2) - go func() { - defer wg.Done() - defer conn.Close() - - dec := lineprotocol.NewDecoder(conn) - connctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - defer wg.Done() - select { - case <-connctx.Done(): - conn.Close() - case <-ctx.Done(): - conn.Close() - } - }() - - if err := handleLine(dec, "default"); err != nil { - if errors.Is(err, net.ErrClosed) { - return - } - - log.Printf("ReceiveRaw Error from Address %s: %s", conn.RemoteAddr().String(), err.Error()) - errmsg := make([]byte, 128) - errmsg = append(errmsg, `error: `...) - errmsg = append(errmsg, err.Error()...) - errmsg = append(errmsg, '\n') - conn.Write(errmsg) - } - }() - } - - wg.Wait() - return nil -} - -// Connect to a nats server and subscribe to "updates". This is a blocking -// function. handleLine will be called for each line recieved via nats. -// Send `true` through the done channel for gracefull termination. -func ReceiveNats(conf *config.NatsConfig, - ms *memorystore.MemoryStore, - workers int, - ctx context.Context, -) error { - var opts []nats.Option - if conf.Username != "" && conf.Password != "" { - opts = append(opts, nats.UserInfo(conf.Username, conf.Password)) - } - - if conf.Credsfilepath != "" { - opts = append(opts, nats.UserCredentials(conf.Credsfilepath)) - } - - nc, err := nats.Connect(conf.Address, opts...) - if err != nil { - return err - } - defer nc.Close() - - var wg sync.WaitGroup - var subs []*nats.Subscription - - msgs := make(chan *nats.Msg, workers*2) - - for _, sc := range conf.Subscriptions { - clusterTag := sc.ClusterTag - var sub *nats.Subscription - if workers > 1 { - wg.Add(workers) - - for i := 0; i < workers; i++ { - go func() { - for m := range msgs { - dec := lineprotocol.NewDecoderWithBytes(m.Data) - if err := decodeLine(dec, ms, clusterTag); err != nil { - log.Printf("error: %s\n", err.Error()) - } - } - - wg.Done() - }() - } - - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - msgs <- m - }) - } else { - sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { - dec := lineprotocol.NewDecoderWithBytes(m.Data) - if err := decodeLine(dec, ms, clusterTag); err != nil { - log.Printf("error: %s\n", err.Error()) - } - }) - } - - if err != nil { - return err - } - log.Printf("NATS subscription to '%s' on '%s' established\n", sc.SubscribeTo, conf.Address) - subs = append(subs, sub) - } - - <-ctx.Done() - for _, sub := range subs { - err = sub.Unsubscribe() - if err != nil { - log.Printf("NATS unsubscribe failed: %s", err.Error()) - } - } - close(msgs) - wg.Wait() - - nc.Close() - log.Println("NATS connection closed") - return nil -} - -// Place `prefix` in front of `buf` but if possible, -// do that inplace in `buf`. -func reorder(buf, prefix []byte) []byte { - n := len(prefix) - m := len(buf) - if cap(buf) < m+n { - return append(prefix[:n:n], buf...) - } else { - buf = buf[:n+m] - for i := m - 1; i >= 0; i-- { - buf[i+n] = buf[i] - } - for i := 0; i < n; i++ { - buf[i] = prefix[i] - } - return buf - } -} - -// Decode lines using dec and make write calls to the MemoryStore. -// If a line is missing its cluster tag, use clusterDefault as default. -func decodeLine(dec *lineprotocol.Decoder, - ms *memorystore.MemoryStore, - clusterDefault string, -) error { - // Reduce allocations in loop: - t := time.Now() - metric, metricBuf := memorystore.Metric{}, make([]byte, 0, 16) - selector := make([]string, 0, 4) - typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0) - - // Optimize for the case where all lines in a "batch" are about the same - // cluster and host. By using `WriteToLevel` (level = host), we do not need - // to take the root- and cluster-level lock as often. - var lvl *memorystore.Level = nil - prevCluster, prevHost := "", "" - - var ok bool - for dec.Next() { - rawmeasurement, err := dec.Measurement() - if err != nil { - return err - } - - // Needs to be copied because another call to dec.* would - // invalidate the returned slice. - metricBuf = append(metricBuf[:0], rawmeasurement...) - - // The go compiler optimizes map[string(byteslice)] lookups: - metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)] - if !ok { - continue - } - - typeBuf, subTypeBuf := typeBuf[:0], subTypeBuf[:0] - cluster, host := clusterDefault, "" - for { - key, val, err := dec.NextTag() - if err != nil { - return err - } - if key == nil { - break - } - - // The go compiler optimizes string([]byte{...}) == "...": - switch string(key) { - case "cluster": - if string(val) == prevCluster { - cluster = prevCluster - } else { - cluster = string(val) - lvl = nil - } - case "hostname", "host": - if string(val) == prevHost { - host = prevHost - } else { - host = string(val) - lvl = nil - } - case "type": - if string(val) == "node" { - break - } - - // We cannot be sure that the "type" tag comes before the "type-id" tag: - if len(typeBuf) == 0 { - typeBuf = append(typeBuf, val...) - } else { - typeBuf = reorder(typeBuf, val) - } - case "type-id": - typeBuf = append(typeBuf, val...) - case "stype": - // We cannot be sure that the "stype" tag comes before the "stype-id" tag: - if len(subTypeBuf) == 0 { - subTypeBuf = append(subTypeBuf, val...) - } else { - subTypeBuf = reorder(subTypeBuf, val) - // subTypeBuf = reorder(typeBuf, val) - } - case "stype-id": - subTypeBuf = append(subTypeBuf, val...) - default: - // Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need) - // return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val)) - } - } - - // If the cluster or host changed, the lvl was set to nil - if lvl == nil { - selector = selector[:2] - selector[0], selector[1] = cluster, host - lvl = ms.GetLevel(selector) - prevCluster, prevHost = cluster, host - } - - // subtypes: - selector = selector[:0] - if len(typeBuf) > 0 { - selector = append(selector, string(typeBuf)) // <- Allocation :( - if len(subTypeBuf) > 0 { - selector = append(selector, string(subTypeBuf)) - } - } - - for { - key, val, err := dec.NextField() - if err != nil { - return err - } - - if key == nil { - break - } - - if string(key) != "value" { - return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val) - } - - if val.Kind() == lineprotocol.Float { - metric.Value = util.Float(val.FloatV()) - } else if val.Kind() == lineprotocol.Int { - metric.Value = util.Float(val.IntV()) - } else if val.Kind() == lineprotocol.Uint { - metric.Value = util.Float(val.UintV()) - } else { - return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String()) - } - } - - if t, err = dec.Time(lineprotocol.Second, t); err != nil { - t = time.Now() - if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil { - t = time.Now() - if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil { - t = time.Now() - if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil { - return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error()) - } - } - } - } - - if err != nil { - return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error()) - } - - time := t.Unix() - - if config.Keys.Checkpoints.FileFormat != "json" { - avro.LineProtocolMessages <- &avro.AvroStruct{ - MetricName: string(metricBuf), - Cluster: cluster, - Node: host, - Selector: append([]string{}, selector...), - Value: metric.Value, - Timestamp: time} - } - - if err := ms.WriteToLevel(lvl, selector, time, []memorystore.Metric{metric}); err != nil { - return err - } - } - return nil -} diff --git a/internal/api/api.go b/internal/api/metricstore.go similarity index 74% rename from internal/api/api.go rename to internal/api/metricstore.go index ff9596c..2239f12 100644 --- a/internal/api/api.go +++ b/internal/api/metricstore.go @@ -1,7 +1,8 @@ // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. +// All rights reserved. This file is part of cc-metric-store. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package api import ( @@ -16,29 +17,14 @@ import ( "strconv" "strings" - "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" - "github.com/ClusterCockpit/cc-metric-store/internal/util" + "github.com/ClusterCockpit/cc-backend/pkg/metricstore" + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/ClusterCockpit/cc-lib/v2/schema" + "github.com/ClusterCockpit/cc-lib/v2/util" + "github.com/influxdata/line-protocol/v2/lineprotocol" ) -// @title cc-metric-store REST API -// @version 1.0.0 -// @description API for cc-metric-store - -// @contact.name ClusterCockpit Project -// @contact.url https://clustercockpit.org -// @contact.email support@clustercockpit.org - -// @license.name MIT License -// @license.url https://opensource.org/licenses/MIT - -// @host localhost:8082 -// @basePath /api/ - -// @securityDefinitions.apikey ApiKeyAuth -// @in header -// @name X-Auth-Token - // ErrorResponse model type ErrorResponse struct { // Statustext of Errorcode @@ -46,29 +32,38 @@ type ErrorResponse struct { Error string `json:"error"` // Error Message } -type ApiMetricData struct { - Error *string `json:"error,omitempty"` - Data util.FloatArray `json:"data,omitempty"` - From int64 `json:"from"` - To int64 `json:"to"` - Resolution int64 `json:"resolution"` - Avg util.Float `json:"avg"` - Min util.Float `json:"min"` - Max util.Float `json:"max"` +// DefaultAPIResponse model +type DefaultAPIResponse struct { + Message string `json:"msg"` } +// handleError writes a standardized JSON error response with the given status code. +// It logs the error at WARN level and ensures proper Content-Type headers are set. func handleError(err error, statusCode int, rw http.ResponseWriter) { - // log.Warnf("REST ERROR : %s", err.Error()) + cclog.Warnf("REST ERROR : %s", err.Error()) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(statusCode) - json.NewEncoder(rw).Encode(ErrorResponse{ + if err := json.NewEncoder(rw).Encode(ErrorResponse{ Status: http.StatusText(statusCode), Error: err.Error(), - }) + }); err != nil { + cclog.Errorf("Failed to encode error response: %v", err) + } +} + +type APIMetricData struct { + Error *string `json:"error,omitempty"` + Data schema.FloatArray `json:"data,omitempty"` + From int64 `json:"from"` + To int64 `json:"to"` + Resolution int64 `json:"resolution"` + Avg schema.Float `json:"avg"` + Min schema.Float `json:"min"` + Max schema.Float `json:"max"` } // TODO: Optimize this, just like the stats endpoint! -func (data *ApiMetricData) AddStats() { +func (data *APIMetricData) AddStats() { n := 0 sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64 for _, x := range data.Data { @@ -84,15 +79,15 @@ func (data *ApiMetricData) AddStats() { if n > 0 { avg := sum / float64(n) - data.Avg = util.Float(avg) - data.Min = util.Float(min) - data.Max = util.Float(max) + data.Avg = schema.Float(avg) + data.Min = schema.Float(min) + data.Max = schema.Float(max) } else { - data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN + data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN } } -func (data *ApiMetricData) ScaleBy(f util.Float) { +func (data *APIMetricData) ScaleBy(f schema.Float) { if f == 0 || f == 1 { return } @@ -105,7 +100,7 @@ func (data *ApiMetricData) ScaleBy(f util.Float) { } } -func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) { +func (data *APIMetricData) PadDataWithNull(ms *metricstore.MemoryStore, from, to int64, metric string) { minfo, ok := ms.Metrics[metric] if !ok { return @@ -113,9 +108,9 @@ func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to if (data.From / minfo.Frequency) > (from / minfo.Frequency) { padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency)) - ndata := make([]util.Float, 0, padfront+len(data.Data)) - for i := 0; i < padfront; i++ { - ndata = append(ndata, util.NaN) + ndata := make([]schema.Float, 0, padfront+len(data.Data)) + for range padfront { + ndata = append(ndata, schema.NaN) } for j := 0; j < len(data.Data); j++ { ndata = append(ndata, data.Data[j]) @@ -124,102 +119,9 @@ func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to } } -// handleFree godoc -// @summary -// @tags free -// @description This endpoint allows the users to free the Buffers from the -// metric store. This endpoint offers the users to remove then systematically -// and also allows then to prune the data under node, if they do not want to -// remove the whole node. -// @produce json -// @param to query string false "up to timestamp" -// @success 200 {string} string "ok" -// @failure 400 {object} api.ErrorResponse "Bad Request" -// @failure 401 {object} api.ErrorResponse "Unauthorized" -// @failure 403 {object} api.ErrorResponse "Forbidden" -// @failure 500 {object} api.ErrorResponse "Internal Server Error" -// @security ApiKeyAuth -// @router /free/ [post] -func handleFree(rw http.ResponseWriter, r *http.Request) { - rawTo := r.URL.Query().Get("to") - if rawTo == "" { - handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw) - return - } - - to, err := strconv.ParseInt(rawTo, 10, 64) - if err != nil { - handleError(err, http.StatusInternalServerError, rw) - return - } - - // // TODO: lastCheckpoint might be modified by different go-routines. - // // Load it using the sync/atomic package? - // freeUpTo := lastCheckpoint.Unix() - // if to < freeUpTo { - // freeUpTo = to - // } - - bodyDec := json.NewDecoder(r.Body) - var selectors [][]string - err = bodyDec.Decode(&selectors) - if err != nil { - http.Error(rw, err.Error(), http.StatusBadRequest) - return - } - - ms := memorystore.GetMemoryStore() - n := 0 - for _, sel := range selectors { - bn, err := ms.Free(sel, to) - if err != nil { - handleError(err, http.StatusInternalServerError, rw) - return - } - - n += bn - } - - rw.WriteHeader(http.StatusOK) - fmt.Fprintf(rw, "buffers freed: %d\n", n) -} - -// handleWrite godoc -// @summary Receive metrics in InfluxDB line-protocol -// @tags write -// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md) - -// @accept plain -// @produce json -// @param cluster query string false "If the lines in the body do not have a cluster tag, use this value instead." -// @success 200 {string} string "ok" -// @failure 400 {object} api.ErrorResponse "Bad Request" -// @failure 401 {object} api.ErrorResponse "Unauthorized" -// @failure 403 {object} api.ErrorResponse "Forbidden" -// @failure 500 {object} api.ErrorResponse "Internal Server Error" -// @security ApiKeyAuth -// @router /write/ [post] -func handleWrite(rw http.ResponseWriter, r *http.Request) { - bytes, err := io.ReadAll(r.Body) - rw.Header().Add("Content-Type", "application/json") - if err != nil { - handleError(err, http.StatusInternalServerError, rw) - return - } - - ms := memorystore.GetMemoryStore() - dec := lineprotocol.NewDecoderWithBytes(bytes) - if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { - log.Printf("/api/write error: %s", err.Error()) - handleError(err, http.StatusBadRequest, rw) - return - } - rw.WriteHeader(http.StatusOK) -} - -type ApiQueryRequest struct { +type APIQueryRequest struct { Cluster string `json:"cluster"` - Queries []ApiQuery `json:"queries"` + Queries []APIQuery `json:"queries"` ForAllNodes []string `json:"for-all-nodes"` From int64 `json:"from"` To int64 `json:"to"` @@ -228,21 +130,21 @@ type ApiQueryRequest struct { WithPadding bool `json:"with-padding"` } -type ApiQueryResponse struct { - Queries []ApiQuery `json:"queries,omitempty"` - Results [][]ApiMetricData `json:"results"` +type APIQueryResponse struct { + Queries []APIQuery `json:"queries,omitempty"` + Results [][]APIMetricData `json:"results"` } -type ApiQuery struct { - Type *string `json:"type,omitempty"` - SubType *string `json:"subtype,omitempty"` - Metric string `json:"metric"` - Hostname string `json:"host"` - Resolution int64 `json:"resolution"` - TypeIds []string `json:"type-ids,omitempty"` - SubTypeIds []string `json:"subtype-ids,omitempty"` - ScaleFactor util.Float `json:"scale-by,omitempty"` - Aggregate bool `json:"aggreg"` +type APIQuery struct { + Type *string `json:"type,omitempty"` + SubType *string `json:"subtype,omitempty"` + Metric string `json:"metric"` + Hostname string `json:"host"` + Resolution int64 `json:"resolution"` + TypeIds []string `json:"type-ids,omitempty"` + SubTypeIds []string `json:"subtype-ids,omitempty"` + ScaleFactor schema.Float `json:"scale-by,omitempty"` + Aggregate bool `json:"aggreg"` } // handleQuery godoc @@ -267,22 +169,22 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { if ver == "" { ver = "v2" } - req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true} + req := APIQueryRequest{WithStats: true, WithData: true, WithPadding: true} if err := json.NewDecoder(r.Body).Decode(&req); err != nil { handleError(err, http.StatusBadRequest, rw) return } - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() - response := ApiQueryResponse{ - Results: make([][]ApiMetricData, 0, len(req.Queries)), + response := APIQueryResponse{ + Results: make([][]APIMetricData, 0, len(req.Queries)), } if req.ForAllNodes != nil { nodes := ms.ListChildren([]string{req.Cluster}) for _, node := range nodes { for _, metric := range req.ForAllNodes { - q := ApiQuery{ + q := APIQuery{ Metric: metric, Hostname: node, } @@ -321,21 +223,21 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { } sels = append(sels, sel) } else { - for _, typeId := range query.TypeIds { + for _, typeID := range query.TypeIds { if query.SubType != nil { - for _, subTypeId := range query.SubTypeIds { + for _, subTypeID := range query.SubTypeIds { sels = append(sels, util.Selector{ {String: req.Cluster}, {String: query.Hostname}, - {String: *query.Type + typeId}, - {String: *query.SubType + subTypeId}, + {String: *query.Type + typeID}, + {String: *query.SubType + subTypeID}, }) } } else { sels = append(sels, util.Selector{ {String: req.Cluster}, {String: query.Hostname}, - {String: *query.Type + typeId}, + {String: *query.Type + typeID}, }) } } @@ -344,9 +246,9 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { // log.Printf("query: %#v\n", query) // log.Printf("sels: %#v\n", sels) - res := make([]ApiMetricData, 0, len(sels)) + res := make([]APIMetricData, 0, len(sels)) for _, sel := range sels { - data := ApiMetricData{} + data := APIMetricData{} if ver == "v1" { data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, 0) } else { @@ -380,11 +282,97 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { bw := bufio.NewWriter(rw) defer bw.Flush() if err := json.NewEncoder(bw).Encode(response); err != nil { - log.Printf("handleQuery Response Encode Error: %s", err.Error()) + log.Print(err) return } } +// handleFree godoc +// @summary +// @tags free +// @description This endpoint allows the users to free the Buffers from the +// metric store. This endpoint offers the users to remove then systematically +// and also allows then to prune the data under node, if they do not want to +// remove the whole node. +// @produce json +// @param to query string false "up to timestamp" +// @success 200 {string} string "ok" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /free/ [post] +func freeMetrics(rw http.ResponseWriter, r *http.Request) { + rawTo := r.URL.Query().Get("to") + if rawTo == "" { + handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw) + return + } + + to, err := strconv.ParseInt(rawTo, 10, 64) + if err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } + + bodyDec := json.NewDecoder(r.Body) + var selectors [][]string + err = bodyDec.Decode(&selectors) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + ms := metricstore.GetMemoryStore() + n := 0 + for _, sel := range selectors { + bn, err := ms.Free(sel, to) + if err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } + + n += bn + } + + rw.WriteHeader(http.StatusOK) + fmt.Fprintf(rw, "buffers freed: %d\n", n) +} + +// handleWrite godoc +// @summary Receive metrics in InfluxDB line-protocol +// @tags write +// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md) + +// @accept plain +// @produce json +// @param cluster query string false "If the lines in the body do not have a cluster tag, use this value instead." +// @success 200 {string} string "ok" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /write/ [post] +func writeMetrics(rw http.ResponseWriter, r *http.Request) { + bytes, err := io.ReadAll(r.Body) + rw.Header().Add("Content-Type", "application/json") + if err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } + + ms := metricstore.GetMemoryStore() + dec := lineprotocol.NewDecoderWithBytes(bytes) + if err := metricstore.DecodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { + cclog.Errorf("/api/write error: %s", err.Error()) + handleError(err, http.StatusBadRequest, rw) + return + } + rw.WriteHeader(http.StatusOK) +} + // handleDebug godoc // @summary Debug endpoint // @tags debug @@ -399,7 +387,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth // @router /debug/ [post] -func handleDebug(rw http.ResponseWriter, r *http.Request) { +func debugMetrics(rw http.ResponseWriter, r *http.Request) { raw := r.URL.Query().Get("selector") rw.Header().Add("Content-Type", "application/json") selector := []string{} @@ -407,7 +395,7 @@ func handleDebug(rw http.ResponseWriter, r *http.Request) { selector = strings.Split(raw, ":") } - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil { handleError(err, http.StatusBadRequest, rw) return @@ -427,7 +415,7 @@ func handleDebug(rw http.ResponseWriter, r *http.Request) { // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth // @router /healthcheck/ [get] -func handleHealthCheck(rw http.ResponseWriter, r *http.Request) { +func metricsHealth(rw http.ResponseWriter, r *http.Request) { rawCluster := r.URL.Query().Get("cluster") rawNode := r.URL.Query().Get("node") @@ -440,7 +428,7 @@ func handleHealthCheck(rw http.ResponseWriter, r *http.Request) { selector := []string{rawCluster, rawNode} - ms := memorystore.GetMemoryStore() + ms := metricstore.GetMemoryStore() if err := ms.HealthCheck(bufio.NewWriter(rw), selector); err != nil { handleError(err, http.StatusBadRequest, rw) return diff --git a/internal/api/server.go b/internal/api/server.go index ea24d6f..ea26151 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1,7 +1,8 @@ // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. +// All rights reserved. This file is part of cc-metric-store. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package api import ( @@ -21,29 +22,29 @@ func MountRoutes(r *http.ServeMux) { } publicKey := ed25519.PublicKey(buf) // Compatibility - r.Handle("POST /api/free", authHandler(http.HandlerFunc(handleFree), publicKey)) - r.Handle("POST /api/write", authHandler(http.HandlerFunc(handleWrite), publicKey)) + r.Handle("POST /api/free", authHandler(http.HandlerFunc(freeMetrics), publicKey)) + r.Handle("POST /api/write", authHandler(http.HandlerFunc(writeMetrics), publicKey)) r.Handle("GET /api/query", authHandler(http.HandlerFunc(handleQuery), publicKey)) - r.Handle("GET /api/debug", authHandler(http.HandlerFunc(handleDebug), publicKey)) - r.Handle("GET /api/healthcheck", authHandler(http.HandlerFunc(handleHealthCheck), publicKey)) + r.Handle("GET /api/debug", authHandler(http.HandlerFunc(debugMetrics), publicKey)) + r.Handle("GET /api/healthcheck", authHandler(http.HandlerFunc(metricsHealth), publicKey)) // Refactor - r.Handle("POST /api/free/", authHandler(http.HandlerFunc(handleFree), publicKey)) - r.Handle("POST /api/write/", authHandler(http.HandlerFunc(handleWrite), publicKey)) + r.Handle("POST /api/free/", authHandler(http.HandlerFunc(freeMetrics), publicKey)) + r.Handle("POST /api/write/", authHandler(http.HandlerFunc(writeMetrics), publicKey)) r.Handle("GET /api/query/", authHandler(http.HandlerFunc(handleQuery), publicKey)) - r.Handle("GET /api/debug/", authHandler(http.HandlerFunc(handleDebug), publicKey)) - r.Handle("GET /api/healthcheck/", authHandler(http.HandlerFunc(handleHealthCheck), publicKey)) + r.Handle("GET /api/debug/", authHandler(http.HandlerFunc(debugMetrics), publicKey)) + r.Handle("GET /api/healthcheck/", authHandler(http.HandlerFunc(metricsHealth), publicKey)) } else { // Compatibility - r.HandleFunc("POST /api/free", handleFree) - r.HandleFunc("POST /api/write", handleWrite) + r.HandleFunc("POST /api/free", freeMetrics) + r.HandleFunc("POST /api/write", writeMetrics) r.HandleFunc("GET /api/query", handleQuery) - r.HandleFunc("GET /api/debug", handleDebug) - r.HandleFunc("GET /api/healthcheck", handleHealthCheck) + r.HandleFunc("GET /api/debug", debugMetrics) + r.HandleFunc("GET /api/healthcheck", metricsHealth) // Refactor - r.HandleFunc("POST /api/free/", handleFree) - r.HandleFunc("POST /api/write/", handleWrite) + r.HandleFunc("POST /api/free/", freeMetrics) + r.HandleFunc("POST /api/write/", writeMetrics) r.HandleFunc("GET /api/query/", handleQuery) - r.HandleFunc("GET /api/debug/", handleDebug) - r.HandleFunc("GET /api/healthcheck/", handleHealthCheck) + r.HandleFunc("GET /api/debug/", debugMetrics) + r.HandleFunc("GET /api/healthcheck/", metricsHealth) } } diff --git a/internal/avro/avroCheckpoint.go b/internal/avro/avroCheckpoint.go deleted file mode 100644 index 6563eb4..0000000 --- a/internal/avro/avroCheckpoint.go +++ /dev/null @@ -1,474 +0,0 @@ -package avro - -import ( - "bufio" - "encoding/json" - "errors" - "fmt" - "log" - "os" - "path" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/util" - "github.com/linkedin/goavro/v2" -) - -var NumWorkers int = 4 - -var ErrNoNewData error = errors.New("no data in the pool") - -func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) { - levels := make([]*AvroLevel, 0) - selectors := make([][]string, 0) - as.root.lock.RLock() - // Cluster - for sel1, l1 := range as.root.children { - l1.lock.RLock() - // Node - for sel2, l2 := range l1.children { - l2.lock.RLock() - // Frequency - for sel3, l3 := range l2.children { - levels = append(levels, l3) - selectors = append(selectors, []string{sel1, sel2, sel3}) - } - l2.lock.RUnlock() - } - l1.lock.RUnlock() - } - as.root.lock.RUnlock() - - type workItem struct { - level *AvroLevel - dir string - selector []string - } - - n, errs := int32(0), int32(0) - - var wg sync.WaitGroup - wg.Add(NumWorkers) - work := make(chan workItem, NumWorkers*2) - for range NumWorkers { - go func() { - defer wg.Done() - - for workItem := range work { - var from int64 = getTimestamp(workItem.dir) - - if err := workItem.level.toCheckpoint(workItem.dir, from, dumpAll); err != nil { - if err == ErrNoNewData { - continue - } - - log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error()) - atomic.AddInt32(&errs, 1) - } else { - atomic.AddInt32(&n, 1) - } - } - }() - } - - for i := range len(levels) { - dir := path.Join(dir, path.Join(selectors[i]...)) - work <- workItem{ - level: levels[i], - dir: dir, - selector: selectors[i], - } - } - - close(work) - wg.Wait() - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n) - } - return int(n), nil -} - -// getTimestamp returns the timestamp from the directory name -func getTimestamp(dir string) int64 { - // Extract the resolution and timestamp from the directory name - // The existing avro file will be in epoch timestamp format - // iterate over all the files in the directory and find the maximum timestamp - // and return it - - resolution := path.Base(dir) - dir = path.Dir(dir) - - files, err := os.ReadDir(dir) - if err != nil { - return 0 - } - var maxTs int64 = 0 - - if len(files) == 0 { - return 0 - } - - for _, file := range files { - if file.IsDir() { - continue - } - name := file.Name() - - if len(name) < 5 || !strings.HasSuffix(name, ".avro") || !strings.HasPrefix(name, resolution+"_") { - continue - } - - ts, err := strconv.ParseInt(name[strings.Index(name, "_")+1:len(name)-5], 10, 64) - if err != nil { - fmt.Printf("error while parsing timestamp: %s\n", err.Error()) - continue - } - - if ts > maxTs { - maxTs = ts - } - } - - interval, _ := time.ParseDuration(config.Keys.Checkpoints.Interval) - updateTime := time.Unix(maxTs, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix() - - if updateTime < time.Now().Unix() { - return 0 - } - - return maxTs -} - -func (l *AvroLevel) toCheckpoint(dir string, from int64, dumpAll bool) error { - l.lock.Lock() - defer l.lock.Unlock() - - // fmt.Printf("Checkpointing directory: %s\n", dir) - // filepath contains the resolution - int_res, _ := strconv.Atoi(path.Base(dir)) - - // find smallest overall timestamp in l.data map and delete it from l.data - var minTs int64 = int64(1<<63 - 1) - for ts, dat := range l.data { - if ts < minTs && len(dat) != 0 { - minTs = ts - } - } - - if from == 0 && minTs != int64(1<<63-1) { - from = minTs - } - - if from == 0 { - return ErrNoNewData - } - - var schema string - var codec *goavro.Codec - record_list := make([]map[string]interface{}, 0) - - var f *os.File - - filePath := dir + fmt.Sprintf("_%d.avro", from) - - var err error - - fp_, err_ := os.Stat(filePath) - if errors.Is(err_, os.ErrNotExist) { - err = os.MkdirAll(path.Dir(dir), 0o755) - if err != nil { - return fmt.Errorf("failed to create directory: %v", err) - } - } else if fp_.Size() != 0 { - f, err = os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open existing avro file: %v", err) - } - - br := bufio.NewReader(f) - - reader, err := goavro.NewOCFReader(br) - if err != nil { - return fmt.Errorf("failed to create OCF reader: %v", err) - } - codec = reader.Codec() - schema = codec.Schema() - - f.Close() - } - - time_ref := time.Now().Add(time.Duration(-CheckpointBufferMinutes+1) * time.Minute).Unix() - - if dumpAll { - time_ref = time.Now().Unix() - } - - // Empty values - if len(l.data) == 0 { - // we checkpoint avro files every 60 seconds - repeat := 60 / int_res - - for range repeat { - record_list = append(record_list, make(map[string]interface{})) - } - } - - readFlag := true - - for ts := range l.data { - flag := false - if ts < time_ref { - data := l.data[ts] - - schema_gen, err := generateSchema(data) - - if err != nil { - return err - } - - flag, schema, err = compareSchema(schema, schema_gen) - - if err != nil { - return fmt.Errorf("failed to compare read and generated schema: %v", err) - } - if flag && readFlag && !errors.Is(err_, os.ErrNotExist) { - - f.Close() - - f, err = os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open Avro file: %v", err) - } - - br := bufio.NewReader(f) - - ocfReader, err := goavro.NewOCFReader(br) - if err != nil { - return fmt.Errorf("failed to create OCF reader while changing schema: %v", err) - } - - for ocfReader.Scan() { - record, err := ocfReader.Read() - if err != nil { - return fmt.Errorf("failed to read record: %v", err) - } - - record_list = append(record_list, record.(map[string]interface{})) - } - - f.Close() - - err = os.Remove(filePath) - if err != nil { - return fmt.Errorf("failed to delete file: %v", err) - } - - readFlag = false - } - codec, err = goavro.NewCodec(schema) - if err != nil { - return fmt.Errorf("failed to create codec after merged schema: %v", err) - } - - record_list = append(record_list, generateRecord(data)) - delete(l.data, ts) - } - } - - if len(record_list) == 0 { - return ErrNoNewData - } - - f, err = os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644) - if err != nil { - return fmt.Errorf("failed to append new avro file: %v", err) - } - - // fmt.Printf("Codec : %#v\n", codec) - - writer, err := goavro.NewOCFWriter(goavro.OCFConfig{ - W: f, - Codec: codec, - CompressionName: goavro.CompressionDeflateLabel, - }) - if err != nil { - return fmt.Errorf("failed to create OCF writer: %v", err) - } - - // Append the new record - if err := writer.Append(record_list); err != nil { - return fmt.Errorf("failed to append record: %v", err) - } - - f.Close() - - return nil -} - -func compareSchema(schemaRead, schemaGen string) (bool, string, error) { - var genSchema, readSchema AvroSchema - - if schemaRead == "" { - return false, schemaGen, nil - } - - // Unmarshal the schema strings into AvroSchema structs - if err := json.Unmarshal([]byte(schemaGen), &genSchema); err != nil { - return false, "", fmt.Errorf("failed to parse generated schema: %v", err) - } - if err := json.Unmarshal([]byte(schemaRead), &readSchema); err != nil { - return false, "", fmt.Errorf("failed to parse read schema: %v", err) - } - - sort.Slice(genSchema.Fields, func(i, j int) bool { - return genSchema.Fields[i].Name < genSchema.Fields[j].Name - }) - - sort.Slice(readSchema.Fields, func(i, j int) bool { - return readSchema.Fields[i].Name < readSchema.Fields[j].Name - }) - - // Check if schemas are identical - schemasEqual := true - if len(genSchema.Fields) <= len(readSchema.Fields) { - - for i := range genSchema.Fields { - if genSchema.Fields[i].Name != readSchema.Fields[i].Name { - schemasEqual = false - break - } - } - - // If schemas are identical, return the read schema - if schemasEqual { - return false, schemaRead, nil - } - } - - // Create a map to hold unique fields from both schemas - fieldMap := make(map[string]AvroField) - - // Add fields from the read schema - for _, field := range readSchema.Fields { - fieldMap[field.Name] = field - } - - // Add or update fields from the generated schema - for _, field := range genSchema.Fields { - fieldMap[field.Name] = field - } - - // Create a union schema by collecting fields from the map - var mergedFields []AvroField - for _, field := range fieldMap { - mergedFields = append(mergedFields, field) - } - - // Sort fields by name for consistency - sort.Slice(mergedFields, func(i, j int) bool { - return mergedFields[i].Name < mergedFields[j].Name - }) - - // Create the merged schema - mergedSchema := AvroSchema{ - Type: "record", - Name: genSchema.Name, - Fields: mergedFields, - } - - // Check if schemas are identical - schemasEqual = len(mergedSchema.Fields) == len(readSchema.Fields) - if schemasEqual { - for i := range mergedSchema.Fields { - if mergedSchema.Fields[i].Name != readSchema.Fields[i].Name { - schemasEqual = false - break - } - } - - if schemasEqual { - return false, schemaRead, nil - } - } - - // Marshal the merged schema back to JSON - mergedSchemaJson, err := json.Marshal(mergedSchema) - if err != nil { - return false, "", fmt.Errorf("failed to marshal merged schema: %v", err) - } - - return true, string(mergedSchemaJson), nil -} - -func generateSchema(data map[string]util.Float) (string, error) { - - // Define the Avro schema structure - schema := map[string]interface{}{ - "type": "record", - "name": "DataRecord", - "fields": []map[string]interface{}{}, - } - - fieldTracker := make(map[string]struct{}) - - for key := range data { - if _, exists := fieldTracker[key]; !exists { - key = correctKey(key) - - field := map[string]interface{}{ - "name": key, - "type": "double", - "default": -1.0, - } - schema["fields"] = append(schema["fields"].([]map[string]interface{}), field) - fieldTracker[key] = struct{}{} - } - } - - schemaString, err := json.Marshal(schema) - if err != nil { - return "", fmt.Errorf("failed to marshal schema: %v", err) - } - - return string(schemaString), nil -} - -func generateRecord(data map[string]util.Float) map[string]interface{} { - - record := make(map[string]interface{}) - - // Iterate through each map in data - for key, value := range data { - key = correctKey(key) - - // Set the value in the record - record[key] = value.Double() - } - - return record -} - -func correctKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, ":", "___") - key = strings.ReplaceAll(key, ".", "__") - - return key -} - -func ReplaceKey(key string) string { - // Replace any invalid characters in the key - // For example, replace spaces with underscores - key = strings.ReplaceAll(key, "___", ":") - key = strings.ReplaceAll(key, "__", ".") - - return key -} diff --git a/internal/avro/avroHelper.go b/internal/avro/avroHelper.go deleted file mode 100644 index 1f7cbd3..0000000 --- a/internal/avro/avroHelper.go +++ /dev/null @@ -1,80 +0,0 @@ -package avro - -import ( - "context" - "fmt" - "strconv" - "sync" - - "github.com/ClusterCockpit/cc-metric-store/internal/config" -) - -func DataStaging(wg *sync.WaitGroup, ctx context.Context) { - - // AvroPool is a pool of Avro writers. - go func() { - if config.Keys.Checkpoints.FileFormat == "json" { - wg.Done() // Mark this goroutine as done - return // Exit the goroutine - } - - defer wg.Done() - - var avroLevel *AvroLevel - oldSelector := make([]string, 0) - - for { - select { - case <-ctx.Done(): - return - case val := <-LineProtocolMessages: - //Fetch the frequency of the metric from the global configuration - freq, err := config.Keys.GetMetricFrequency(val.MetricName) - if err != nil { - fmt.Printf("Error fetching metric frequency: %s\n", err) - continue - } - - metricName := "" - - for _, selector_name := range val.Selector { - metricName += selector_name + Delimiter - } - - metricName += val.MetricName - - // Create a new selector for the Avro level - // The selector is a slice of strings that represents the path to the - // Avro level. It is created by appending the cluster, node, and metric - // name to the selector. - var selector []string - selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10)) - - if !testEq(oldSelector, selector) { - // Get the Avro level for the metric - avroLevel = avroStore.root.findAvroLevelOrCreate(selector) - - // If the Avro level is nil, create a new one - if avroLevel == nil { - fmt.Printf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName) - } - oldSelector = append([]string{}, selector...) - } - - avroLevel.addMetric(metricName, val.Value, val.Timestamp, int(freq)) - } - } - }() -} - -func testEq(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/internal/avro/avroStruct.go b/internal/avro/avroStruct.go deleted file mode 100644 index 85aa3ba..0000000 --- a/internal/avro/avroStruct.go +++ /dev/null @@ -1,161 +0,0 @@ -package avro - -import ( - "sync" - - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -var LineProtocolMessages = make(chan *AvroStruct) -var Delimiter = "ZZZZZ" - -// CheckpointBufferMinutes should always be in minutes. -// Its controls the amount of data to hold for given amount of time. -var CheckpointBufferMinutes = 3 - -type AvroStruct struct { - MetricName string - Cluster string - Node string - Selector []string - Value util.Float - Timestamp int64 -} - -type AvroStore struct { - root AvroLevel -} - -var avroStore AvroStore - -type AvroLevel struct { - children map[string]*AvroLevel - data map[int64]map[string]util.Float - lock sync.RWMutex -} - -type AvroField struct { - Name string `json:"name"` - Type interface{} `json:"type"` - Default interface{} `json:"default,omitempty"` -} - -type AvroSchema struct { - Type string `json:"type"` - Name string `json:"name"` - Fields []AvroField `json:"fields"` -} - -func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel { - if len(selector) == 0 { - return l - } - - // Allow concurrent reads: - l.lock.RLock() - var child *AvroLevel - var ok bool - if l.children == nil { - // Children map needs to be created... - l.lock.RUnlock() - } else { - child, ok := l.children[selector[0]] - l.lock.RUnlock() - if ok { - return child.findAvroLevelOrCreate(selector[1:]) - } - } - - // The level does not exist, take write lock for unqiue access: - l.lock.Lock() - // While this thread waited for the write lock, another thread - // could have created the child node. - if l.children != nil { - child, ok = l.children[selector[0]] - if ok { - l.lock.Unlock() - return child.findAvroLevelOrCreate(selector[1:]) - } - } - - child = &AvroLevel{ - data: make(map[int64]map[string]util.Float, 0), - children: nil, - } - - if l.children != nil { - l.children[selector[0]] = child - } else { - l.children = map[string]*AvroLevel{selector[0]: child} - } - l.lock.Unlock() - return child.findAvroLevelOrCreate(selector[1:]) -} - -func (l *AvroLevel) addMetric(metricName string, value util.Float, timestamp int64, Freq int) { - l.lock.Lock() - defer l.lock.Unlock() - - KeyCounter := int(CheckpointBufferMinutes * 60 / Freq) - - // Create keys in advance for the given amount of time - if len(l.data) != KeyCounter { - if len(l.data) == 0 { - for i := range KeyCounter { - l.data[timestamp+int64(i*Freq)] = make(map[string]util.Float, 0) - } - } else { - //Get the last timestamp - var lastTs int64 - for ts := range l.data { - if ts > lastTs { - lastTs = ts - } - } - // Create keys for the next KeyCounter timestamps - l.data[lastTs+int64(Freq)] = make(map[string]util.Float, 0) - } - } - - closestTs := int64(0) - minDiff := int64(Freq) + 1 // Start with diff just outside the valid range - found := false - - // Iterate over timestamps and choose the one which is within range. - // Since its epoch time, we check if the difference is less than 60 seconds. - for ts, dat := range l.data { - // Check if timestamp is within range - diff := timestamp - ts - if diff < -int64(Freq) || diff > int64(Freq) { - continue - } - - // Metric already present at this timestamp — skip - if _, ok := dat[metricName]; ok { - continue - } - - // Check if this is the closest timestamp so far - if Abs(diff) < minDiff { - minDiff = Abs(diff) - closestTs = ts - found = true - } - } - - if found { - l.data[closestTs][metricName] = value - } -} - -func GetAvroStore() *AvroStore { - return &avroStore -} - -// Abs returns the absolute value of x. -func Abs(x int64) int64 { - if x < 0 { - return -x - } - return x -} diff --git a/internal/config/config.go b/internal/config/config.go index 0469924..e6ea240 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,10 +1,16 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package config import ( + "bytes" "encoding/json" "fmt" - "log" - "os" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" ) // For aggregation over multiple values at different cpus/sockets/..., not time! @@ -46,77 +52,43 @@ type MetricConfig struct { Offset int } -type HttpConfig struct { - // Address to bind to, for example "0.0.0.0:8081" - Address string `json:"address"` - - // If not the empty string, use https with this as the certificate file - CertFile string `json:"https-cert-file"` - - // If not the empty string, use https with this as the key file - KeyFile string `json:"https-key-file"` -} - -type NatsConfig struct { - // Address of the nats server - Address string `json:"address"` - - // Username/Password, optional - Username string `json:"username"` - Password string `json:"password"` - - //Creds file path - Credsfilepath string `json:"creds-file-path"` - - Subscriptions []struct { - // Channel name - SubscribeTo string `json:"subscribe-to"` - - // Allow lines without a cluster tag, use this as default, optional - ClusterTag string `json:"cluster-tag"` - } `json:"subscriptions"` -} +var metrics map[string]MetricConfig type Config struct { - Metrics map[string]MetricConfig `json:"metrics"` - HttpConfig *HttpConfig `json:"http-api"` - Checkpoints struct { - FileFormat string `json:"file-format"` - Interval string `json:"interval"` - RootDir string `json:"directory"` - Restore string `json:"restore"` - } `json:"checkpoints"` - Debug struct { + Address string `json:"addr"` + CertFile string `json:"https-cert-file"` + KeyFile string `json:"https-key-file"` + User string `json:"user"` + Group string `json:"group"` + Debug struct { DumpToFile string `json:"dump-to-file"` EnableGops bool `json:"gops"` } `json:"debug"` - RetentionInMemory string `json:"retention-in-memory"` - JwtPublicKey string `json:"jwt-public-key"` - Archive struct { - Interval string `json:"interval"` - RootDir string `json:"directory"` - DeleteInstead bool `json:"delete-instead"` - } `json:"archive"` - Nats []*NatsConfig `json:"nats"` + JwtPublicKey string `json:"jwt-public-key"` } var Keys Config -func Init(file string) { - configFile, err := os.Open(file) - if err != nil { - log.Fatal(err) - } - defer configFile.Close() - dec := json.NewDecoder(configFile) +func InitMetrics(metricConfig json.RawMessage) { + Validate(metricConfigSchema, metricConfig) + dec := json.NewDecoder(bytes.NewReader(metricConfig)) dec.DisallowUnknownFields() - if err := dec.Decode(&Keys); err != nil { - log.Fatal(err) + if err := dec.Decode(&metrics); err != nil { + cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", metricConfig, err.Error()) } } -func (c *Config) GetMetricFrequency(metricName string) (int64, error) { - if metric, ok := c.Metrics[metricName]; ok { +func Init(mainConfig json.RawMessage) { + Validate(configSchema, mainConfig) + dec := json.NewDecoder(bytes.NewReader(mainConfig)) + dec.DisallowUnknownFields() + if err := dec.Decode(&Keys); err != nil { + cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error()) + } +} + +func GetMetricFrequency(metricName string) (int64, error) { + if metric, ok := metrics[metricName]; ok { return metric.Frequency, nil } return 0, fmt.Errorf("metric %s not found", metricName) diff --git a/internal/config/metricSchema.go b/internal/config/metricSchema.go new file mode 100644 index 0000000..be6244f --- /dev/null +++ b/internal/config/metricSchema.go @@ -0,0 +1,135 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package config + +var metricConfigSchema = ` +{ + "type": "object", + "properties": { + "addr": { + "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", + "type": "string" + }, + "api-allowed-ips": { + "description": "Addresses from which secured API endpoints can be reached", + "type": "array", + "items": { + "type": "string" + } + }, + "user": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "group": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "disable-authentication": { + "description": "Disable authentication (for everything: API, Web-UI, ...).", + "type": "boolean" + }, + "embed-static-files": { + "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.", + "type": "boolean" + }, + "static-files": { + "description": "Folder where static assets can be found, if embed-static-files is false.", + "type": "string" + }, + "db": { + "description": "Path to SQLite database file (e.g., './var/job.db')", + "type": "string" + }, + "enable-job-taggers": { + "description": "Turn on automatic application and jobclass taggers", + "type": "boolean" + }, + "validate": { + "description": "Validate all input json documents against json schema.", + "type": "boolean" + }, + "session-max-age": { + "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "https-cert-file": { + "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", + "type": "string" + }, + "https-key-file": { + "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", + "type": "string" + }, + "redirect-http-to": { + "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", + "type": "string" + }, + "stop-jobs-exceeding-walltime": { + "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "type": "integer" + }, + "short-running-jobs-duration": { + "description": "Do not show running jobs shorter than X seconds.", + "type": "integer" + }, + "emission-constant": { + "description": ".", + "type": "integer" + }, + "cron-frequency": { + "description": "Frequency of cron job workers.", + "type": "object", + "properties": { + "duration-worker": { + "description": "Duration Update Worker [Defaults to '5m']", + "type": "string" + }, + "footprint-worker": { + "description": "Metric-Footprint Update Worker [Defaults to '10m']", + "type": "string" + } + } + }, + "enable-resampling": { + "description": "Enable dynamic zoom in frontend metric plots.", + "type": "object", + "properties": { + "minimum-points": { + "description": "Minimum points to trigger resampling of time-series data.", + "type": "integer" + }, + "trigger": { + "description": "Trigger next zoom level at less than this many visible datapoints.", + "type": "integer" + }, + "resolutions": { + "description": "Array of resampling target resolutions, in seconds.", + "type": "array", + "items": { + "type": "integer" + } + } + }, + "required": ["trigger", "resolutions"] + }, + "api-subjects": { + "description": "NATS subjects configuration for subscribing to job and node events.", + "type": "object", + "properties": { + "subject-job-event": { + "description": "NATS subject for job events (start_job, stop_job)", + "type": "string" + }, + "subject-node-state": { + "description": "NATS subject for node state updates", + "type": "string" + } + }, + "required": ["subject-job-event", "subject-node-state"] + } + } +}` diff --git a/internal/config/schema.go b/internal/config/schema.go new file mode 100644 index 0000000..64b02f1 --- /dev/null +++ b/internal/config/schema.go @@ -0,0 +1,135 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package config + +var configSchema = ` +{ + "type": "object", + "properties": { + "addr": { + "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').", + "type": "string" + }, + "api-allowed-ips": { + "description": "Addresses from which secured API endpoints can be reached", + "type": "array", + "items": { + "type": "string" + } + }, + "user": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "group": { + "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.", + "type": "string" + }, + "disable-authentication": { + "description": "Disable authentication (for everything: API, Web-UI, ...).", + "type": "boolean" + }, + "embed-static-files": { + "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.", + "type": "boolean" + }, + "static-files": { + "description": "Folder where static assets can be found, if embed-static-files is false.", + "type": "string" + }, + "db": { + "description": "Path to SQLite database file (e.g., './var/job.db')", + "type": "string" + }, + "enable-job-taggers": { + "description": "Turn on automatic application and jobclass taggers", + "type": "boolean" + }, + "validate": { + "description": "Validate all input json documents against json schema.", + "type": "boolean" + }, + "session-max-age": { + "description": "Specifies for how long a session shall be valid as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!", + "type": "string" + }, + "https-cert-file": { + "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.", + "type": "string" + }, + "https-key-file": { + "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.", + "type": "string" + }, + "redirect-http-to": { + "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.", + "type": "string" + }, + "stop-jobs-exceeding-walltime": { + "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", + "type": "integer" + }, + "short-running-jobs-duration": { + "description": "Do not show running jobs shorter than X seconds.", + "type": "integer" + }, + "emission-constant": { + "description": ".", + "type": "integer" + }, + "cron-frequency": { + "description": "Frequency of cron job workers.", + "type": "object", + "properties": { + "duration-worker": { + "description": "Duration Update Worker [Defaults to '5m']", + "type": "string" + }, + "footprint-worker": { + "description": "Metric-Footprint Update Worker [Defaults to '10m']", + "type": "string" + } + } + }, + "enable-resampling": { + "description": "Enable dynamic zoom in frontend metric plots.", + "type": "object", + "properties": { + "minimum-points": { + "description": "Minimum points to trigger resampling of time-series data.", + "type": "integer" + }, + "trigger": { + "description": "Trigger next zoom level at less than this many visible datapoints.", + "type": "integer" + }, + "resolutions": { + "description": "Array of resampling target resolutions, in seconds.", + "type": "array", + "items": { + "type": "integer" + } + } + }, + "required": ["trigger", "resolutions"] + }, + "api-subjects": { + "description": "NATS subjects configuration for subscribing to job and node events.", + "type": "object", + "properties": { + "subject-job-event": { + "description": "NATS subject for job events (start_job, stop_job)", + "type": "string" + }, + "subject-node-state": { + "description": "NATS subject for node state updates", + "type": "string" + } + }, + "required": ["subject-job-event", "subject-node-state"] + } + } +}` diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 0000000..ce99ed1 --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,29 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. This file is part of cc-metric-store. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package config + +import ( + "encoding/json" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + "github.com/santhosh-tekuri/jsonschema/v5" +) + +func Validate(schema string, instance json.RawMessage) { + sch, err := jsonschema.CompileString("schema.json", schema) + if err != nil { + cclog.Fatalf("%#v", err) + } + + var v any + if err := json.Unmarshal([]byte(instance), &v); err != nil { + cclog.Fatal(err) + } + + if err = sch.Validate(v); err != nil { + cclog.Fatalf("%#v", err) + } +} diff --git a/internal/memorystore/archive.go b/internal/memorystore/archive.go deleted file mode 100644 index b751b58..0000000 --- a/internal/memorystore/archive.go +++ /dev/null @@ -1,185 +0,0 @@ -package memorystore - -import ( - "archive/zip" - "bufio" - "context" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "sync" - "sync/atomic" - "time" - - "github.com/ClusterCockpit/cc-metric-store/internal/config" -) - -func Archiving(wg *sync.WaitGroup, ctx context.Context) { - go func() { - defer wg.Done() - d, err := time.ParseDuration(config.Keys.Archive.Interval) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() - for { - select { - case <-ctx.Done(): - return - case <-ticks: - t := time.Now().Add(-d) - log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339)) - n, err := ArchiveCheckpoints(config.Keys.Checkpoints.RootDir, config.Keys.Archive.RootDir, t.Unix(), config.Keys.Archive.DeleteInstead) - if err != nil { - log.Printf("archiving failed: %s\n", err.Error()) - } else { - log.Printf("done: %d files zipped and moved to archive\n", n) - } - } - } - }() -} - -var ErrNoNewData error = errors.New("all data already archived") - -// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`, -// deleting them from the `checkpointsDir`. -func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) { - entries1, err := os.ReadDir(checkpointsDir) - if err != nil { - return 0, err - } - - type workItem struct { - cdir, adir string - cluster, host string - } - - var wg sync.WaitGroup - n, errs := int32(0), int32(0) - work := make(chan workItem, NumWorkers) - - wg.Add(NumWorkers) - for worker := 0; worker < NumWorkers; worker++ { - go func() { - defer wg.Done() - for workItem := range work { - m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead) - if err != nil { - log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error()) - atomic.AddInt32(&errs, 1) - } - atomic.AddInt32(&n, int32(m)) - } - }() - } - - for _, de1 := range entries1 { - entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name())) - if e != nil { - err = e - } - - for _, de2 := range entries2 { - cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name()) - adir := filepath.Join(archiveDir, de1.Name(), de2.Name()) - work <- workItem{ - adir: adir, cdir: cdir, - cluster: de1.Name(), host: de2.Name(), - } - } - } - - close(work) - wg.Wait() - - if err != nil { - return int(n), err - } - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n) - } - return int(n), nil -} - -// Helper function for `ArchiveCheckpoints`. -func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return 0, err - } - - extension := config.Keys.Checkpoints.FileFormat - files, err := findFiles(entries, from, extension, false) - if err != nil { - return 0, err - } - - if deleteInstead { - n := 0 - for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - if err = os.Remove(filename); err != nil { - return n, err - } - n += 1 - } - return n, nil - } - - filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from)) - f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(archiveDir, 0o755) - if err == nil { - f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644) - } - } - if err != nil { - return 0, err - } - defer f.Close() - bw := bufio.NewWriter(f) - defer bw.Flush() - zw := zip.NewWriter(bw) - defer zw.Close() - - n := 0 - for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - r, err := os.Open(filename) - if err != nil { - return n, err - } - defer r.Close() - - w, err := zw.Create(checkpoint) - if err != nil { - return n, err - } - - if _, err = io.Copy(w, r); err != nil { - return n, err - } - - if err = os.Remove(filename); err != nil { - return n, err - } - n += 1 - } - - return n, nil -} diff --git a/internal/memorystore/buffer.go b/internal/memorystore/buffer.go deleted file mode 100644 index 34fee5d..0000000 --- a/internal/memorystore/buffer.go +++ /dev/null @@ -1,233 +0,0 @@ -package memorystore - -import ( - "errors" - "sync" - - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -// Default buffer capacity. -// `buffer.data` will only ever grow up to it's capacity and a new link -// in the buffer chain will be created if needed so that no copying -// of data or reallocation needs to happen on writes. -const ( - BUFFER_CAP int = 512 -) - -// So that we can reuse allocations -var bufferPool sync.Pool = sync.Pool{ - New: func() interface{} { - return &buffer{ - data: make([]util.Float, 0, BUFFER_CAP), - } - }, -} - -var ( - ErrNoData error = errors.New("no data for this metric/level") - ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align") -) - -// Each metric on each level has it's own buffer. -// This is where the actual values go. -// If `cap(data)` is reached, a new buffer is created and -// becomes the new head of a buffer list. -type buffer struct { - prev *buffer - next *buffer - data []util.Float - frequency int64 - start int64 - archived bool - closed bool -} - -func newBuffer(ts, freq int64) *buffer { - b := bufferPool.Get().(*buffer) - b.frequency = freq - b.start = ts - (freq / 2) - b.prev = nil - b.next = nil - b.archived = false - b.closed = false - b.data = b.data[:0] - return b -} - -// If a new buffer was created, the new head is returnd. -// Otherwise, the existing buffer is returnd. -// Normaly, only "newer" data should be written, but if the value would -// end up in the same buffer anyways it is allowed. -func (b *buffer) write(ts int64, value util.Float) (*buffer, error) { - if ts < b.start { - return nil, errors.New("cannot write value to buffer from past") - } - - // idx := int((ts - b.start + (b.frequency / 3)) / b.frequency) - idx := int((ts - b.start) / b.frequency) - if idx >= cap(b.data) { - newbuf := newBuffer(ts, b.frequency) - newbuf.prev = b - b.next = newbuf - b.close() - b = newbuf - idx = 0 - } - - // Overwriting value or writing value from past - if idx < len(b.data) { - b.data[idx] = value - return b, nil - } - - // Fill up unwritten slots with NaN - for i := len(b.data); i < idx; i++ { - b.data = append(b.data, util.NaN) - } - - b.data = append(b.data, value) - return b, nil -} - -func (b *buffer) end() int64 { - return b.firstWrite() + int64(len(b.data))*b.frequency -} - -func (b *buffer) firstWrite() int64 { - return b.start + (b.frequency / 2) -} - -func (b *buffer) close() {} - -/* -func (b *buffer) close() { - if b.closed { - return - } - - b.closed = true - n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64 - for _, x := range b.data { - if x.IsNaN() { - continue - } - - n += 1 - f := float64(x) - sum += f - min = math.Min(min, f) - max = math.Max(max, f) - } - - b.statisticts.samples = n - if n > 0 { - b.statisticts.avg = Float(sum / float64(n)) - b.statisticts.min = Float(min) - b.statisticts.max = Float(max) - } else { - b.statisticts.avg = NaN - b.statisticts.min = NaN - b.statisticts.max = NaN - } -} -*/ - -// func interpolate(idx int, data []Float) Float { -// if idx == 0 || idx+1 == len(data) { -// return NaN -// } -// return (data[idx-1] + data[idx+1]) / 2.0 -// } - -// Return all known values from `from` to `to`. Gaps of information are represented as NaN. -// Simple linear interpolation is done between the two neighboring cells if possible. -// If values at the start or end are missing, instead of NaN values, the second and thrid -// return values contain the actual `from`/`to`. -// This function goes back the buffer chain if `from` is older than the currents buffer start. -// The loaded values are added to `data` and `data` is returned, possibly with a shorter length. -// If `data` is not long enough to hold all values, this function will panic! -func (b *buffer) read(from, to int64, data []util.Float) ([]util.Float, int64, int64, error) { - if from < b.firstWrite() { - if b.prev != nil { - return b.prev.read(from, to, data) - } - from = b.firstWrite() - } - - i := 0 - t := from - for ; t < to; t += b.frequency { - idx := int((t - b.start) / b.frequency) - if idx >= cap(b.data) { - if b.next == nil { - break - } - b = b.next - idx = 0 - } - - if idx >= len(b.data) { - if b.next == nil || to <= b.next.start { - break - } - data[i] += util.NaN - } else if t < b.start { - data[i] += util.NaN - // } else if b.data[idx].IsNaN() { - // data[i] += interpolate(idx, b.data) - } else { - data[i] += b.data[idx] - } - i++ - } - - return data[:i], from, t, nil -} - -// Returns true if this buffer needs to be freed. -func (b *buffer) free(t int64) (delme bool, n int) { - if b.prev != nil { - delme, m := b.prev.free(t) - n += m - if delme { - b.prev.next = nil - if cap(b.prev.data) == BUFFER_CAP { - bufferPool.Put(b.prev) - } - b.prev = nil - } - } - - end := b.end() - if end < t { - return true, n + 1 - } - - return false, n -} - -// Call `callback` on every buffer that contains data in the range from `from` to `to`. -func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error { - if b == nil { - return nil - } - - if err := b.prev.iterFromTo(from, to, callback); err != nil { - return err - } - - if from <= b.end() && b.start <= to { - return callback(b) - } - - return nil -} - -func (b *buffer) count() int64 { - res := int64(len(b.data)) - if b.prev != nil { - res += b.prev.count() - } - return res -} diff --git a/internal/memorystore/checkpoint.go b/internal/memorystore/checkpoint.go deleted file mode 100644 index 7c7d467..0000000 --- a/internal/memorystore/checkpoint.go +++ /dev/null @@ -1,767 +0,0 @@ -package memorystore - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "log" - "os" - "path" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/ClusterCockpit/cc-metric-store/internal/avro" - "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/util" - "github.com/linkedin/goavro/v2" -) - -// Whenever changed, update MarshalJSON as well! -type CheckpointMetrics struct { - Data []util.Float `json:"data"` - Frequency int64 `json:"frequency"` - Start int64 `json:"start"` -} - -type CheckpointFile struct { - Metrics map[string]*CheckpointMetrics `json:"metrics"` - Children map[string]*CheckpointFile `json:"children"` - From int64 `json:"from"` - To int64 `json:"to"` -} - -var lastCheckpoint time.Time - -func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { - lastCheckpoint = time.Now() - - if config.Keys.Checkpoints.FileFormat == "json" { - ms := GetMemoryStore() - - go func() { - defer wg.Done() - d, err := time.ParseDuration(config.Keys.Checkpoints.Interval) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() - for { - select { - case <-ctx.Done(): - return - case <-ticks: - log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339)) - now := time.Now() - n, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, - lastCheckpoint.Unix(), now.Unix()) - if err != nil { - log.Printf("checkpointing failed: %s\n", err.Error()) - } else { - log.Printf("done: %d checkpoint files created\n", n) - lastCheckpoint = now - } - } - } - }() - } else { - go func() { - defer wg.Done() - d, _ := time.ParseDuration("1m") - - select { - case <-ctx.Done(): - return - case <-time.After(time.Duration(avro.CheckpointBufferMinutes) * time.Minute): - // This is the first tick untill we collect the data for given minutes. - avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) - - } - - ticks := func() <-chan time.Time { - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() - - for { - select { - case <-ctx.Done(): - return - case <-ticks: - // Regular ticks of 1 minute to write data. - avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, false) - // log.Printf("Checkpointing %d avro files", count) - } - } - }() - } -} - -// As `Float` implements a custom MarshalJSON() function, -// serializing an array of such types has more overhead -// than one would assume (because of extra allocations, interfaces and so on). -func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) { - buf := make([]byte, 0, 128+len(cm.Data)*8) - buf = append(buf, `{"frequency":`...) - buf = strconv.AppendInt(buf, cm.Frequency, 10) - buf = append(buf, `,"start":`...) - buf = strconv.AppendInt(buf, cm.Start, 10) - buf = append(buf, `,"data":[`...) - for i, x := range cm.Data { - if i != 0 { - buf = append(buf, ',') - } - if x.IsNaN() { - buf = append(buf, `null`...) - } else { - buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32) - } - } - buf = append(buf, `]}`...) - return buf, nil -} - -// Metrics stored at the lowest 2 levels are not stored away (root and cluster)! -// On a per-host basis a new JSON file is created. I have no idea if this will scale. -// The good thing: Only a host at a time is locked, so this function can run -// in parallel to writes/reads. -func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) { - levels := make([]*Level, 0) - selectors := make([][]string, 0) - m.root.lock.RLock() - for sel1, l1 := range m.root.children { - l1.lock.RLock() - for sel2, l2 := range l1.children { - levels = append(levels, l2) - selectors = append(selectors, []string{sel1, sel2}) - } - l1.lock.RUnlock() - } - m.root.lock.RUnlock() - - type workItem struct { - level *Level - dir string - selector []string - } - - n, errs := int32(0), int32(0) - - var wg sync.WaitGroup - wg.Add(NumWorkers) - work := make(chan workItem, NumWorkers*2) - for worker := 0; worker < NumWorkers; worker++ { - go func() { - defer wg.Done() - - for workItem := range work { - if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil { - if err == ErrNoNewData { - continue - } - - log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error()) - atomic.AddInt32(&errs, 1) - } else { - atomic.AddInt32(&n, 1) - } - } - }() - } - - for i := 0; i < len(levels); i++ { - dir := path.Join(dir, path.Join(selectors[i]...)) - work <- workItem{ - level: levels[i], - dir: dir, - selector: selectors[i], - } - } - - close(work) - wg.Wait() - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n) - } - return int(n), nil -} - -func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) { - l.lock.RLock() - defer l.lock.RUnlock() - - retval := &CheckpointFile{ - From: from, - To: to, - Metrics: make(map[string]*CheckpointMetrics), - Children: make(map[string]*CheckpointFile), - } - - for metric, minfo := range m.Metrics { - b := l.metrics[minfo.Offset] - if b == nil { - continue - } - - allArchived := true - b.iterFromTo(from, to, func(b *buffer) error { - if !b.archived { - allArchived = false - } - return nil - }) - - if allArchived { - continue - } - - data := make([]util.Float, (to-from)/b.frequency+1) - data, start, end, err := b.read(from, to, data) - if err != nil { - return nil, err - } - - for i := int((end - start) / b.frequency); i < len(data); i++ { - data[i] = util.NaN - } - - retval.Metrics[metric] = &CheckpointMetrics{ - Frequency: b.frequency, - Start: start, - Data: data, - } - } - - for name, child := range l.children { - val, err := child.toCheckpointFile(from, to, m) - if err != nil { - return nil, err - } - - if val != nil { - retval.Children[name] = val - } - } - - if len(retval.Children) == 0 && len(retval.Metrics) == 0 { - return nil, nil - } - - return retval, nil -} - -func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error { - cf, err := l.toCheckpointFile(from, to, m) - if err != nil { - return err - } - - if cf == nil { - return ErrNoNewData - } - - filepath := path.Join(dir, fmt.Sprintf("%d.json", from)) - f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(dir, 0o755) - if err == nil { - f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644) - } - } - if err != nil { - return err - } - defer f.Close() - - bw := bufio.NewWriter(f) - if err = json.NewEncoder(bw).Encode(cf); err != nil { - return err - } - - return bw.Flush() -} - -func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) (int, error) { - var wg sync.WaitGroup - work := make(chan [2]string, NumWorkers) - n, errs := int32(0), int32(0) - - wg.Add(NumWorkers) - for worker := 0; worker < NumWorkers; worker++ { - go func() { - defer wg.Done() - for host := range work { - lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics)) - nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension) - if err != nil { - log.Fatalf("error while loading checkpoints: %s", err.Error()) - atomic.AddInt32(&errs, 1) - } - atomic.AddInt32(&n, int32(nn)) - } - }() - } - - i := 0 - clustersDir, err := os.ReadDir(dir) - for _, clusterDir := range clustersDir { - if !clusterDir.IsDir() { - err = errors.New("expected only directories at first level of checkpoints/ directory") - goto done - } - - hostsDir, e := os.ReadDir(filepath.Join(dir, clusterDir.Name())) - if e != nil { - err = e - goto done - } - - for _, hostDir := range hostsDir { - if !hostDir.IsDir() { - err = errors.New("expected only directories at second level of checkpoints/ directory") - goto done - } - - i++ - if i%NumWorkers == 0 && i > 100 { - // Forcing garbage collection runs here regulary during the loading of checkpoints - // will decrease the total heap size after loading everything back to memory is done. - // While loading data, the heap will grow fast, so the GC target size will double - // almost always. By forcing GCs here, we can keep it growing more slowly so that - // at the end, less memory is wasted. - runtime.GC() - } - - work <- [2]string{clusterDir.Name(), hostDir.Name()} - } - } -done: - close(work) - wg.Wait() - - if err != nil { - return int(n), err - } - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n) - } - return int(n), nil -} - -// Metrics stored at the lowest 2 levels are not loaded (root and cluster)! -// This function can only be called once and before the very first write or read. -// Different host's data is loaded to memory in parallel. -func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) { - - if _, err := os.Stat(dir); os.IsNotExist(err) { - // The directory does not exist, so create it using os.MkdirAll() - err := os.MkdirAll(dir, 0755) // 0755 sets the permissions for the directory - if err != nil { - log.Fatalf("Error creating directory: %#v\n", err) - } - fmt.Printf("%#v Directory created successfully.\n", dir) - } - - // Config read (replace with your actual config read) - fileFormat := config.Keys.Checkpoints.FileFormat - if fileFormat == "" { - fileFormat = "avro" - } - - // Map to easily get the fallback format - oppositeFormat := map[string]string{ - "json": "avro", - "avro": "json", - } - - // First, attempt to load the specified format - if found, err := checkFilesWithExtension(dir, fileFormat); err != nil { - return 0, fmt.Errorf("error checking files with extension: %v", err) - } else if found { - log.Printf("Loading %s files because fileformat is %s\n", fileFormat, fileFormat) - return m.FromCheckpoint(dir, from, fileFormat) - } - - // If not found, attempt the opposite format - altFormat := oppositeFormat[fileFormat] - if found, err := checkFilesWithExtension(dir, altFormat); err != nil { - return 0, fmt.Errorf("error checking files with extension: %v", err) - } else if found { - log.Printf("Loading %s files but fileformat is %s\n", altFormat, fileFormat) - return m.FromCheckpoint(dir, from, altFormat) - } - - log.Println("No valid checkpoint files found in the directory.") - return 0, nil -} - -func checkFilesWithExtension(dir string, extension string) (bool, error) { - found := false - - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error accessing path %s: %v", path, err) - } - if !info.IsDir() && filepath.Ext(info.Name()) == "."+extension { - found = true - return nil - } - return nil - }) - - if err != nil { - return false, fmt.Errorf("error walking through directories: %s", err) - } - - return found, nil -} - -func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error { - br := bufio.NewReader(f) - - fileName := f.Name()[strings.LastIndex(f.Name(), "/")+1:] - resolution, err := strconv.ParseInt(fileName[0:strings.Index(fileName, "_")], 10, 64) - if err != nil { - return fmt.Errorf("error while reading avro file (resolution parsing) : %s", err) - } - - from_timestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64) - - // Same logic according to lineprotocol - from_timestamp -= (resolution / 2) - - if err != nil { - return fmt.Errorf("error converting timestamp from the avro file : %s", err) - } - - // fmt.Printf("File : %s with resolution : %d\n", fileName, resolution) - - var recordCounter int64 = 0 - - // Create a new OCF reader from the buffered reader - ocfReader, err := goavro.NewOCFReader(br) - if err != nil { - panic(err) - } - - metricsData := make(map[string]util.FloatArray) - - for ocfReader.Scan() { - datum, err := ocfReader.Read() - if err != nil { - return fmt.Errorf("error while reading avro file : %s", err) - } - - record, ok := datum.(map[string]interface{}) - if !ok { - panic("failed to assert datum as map[string]interface{}") - } - - for key, value := range record { - metricsData[key] = append(metricsData[key], util.ConvertToFloat(value.(float64))) - } - - recordCounter += 1 - } - - to := (from_timestamp + (recordCounter / (60 / resolution) * 60)) - if to < from { - return nil - } - - for key, floatArray := range metricsData { - metricName := avro.ReplaceKey(key) - - if strings.Contains(metricName, avro.Delimiter) { - subString := strings.Split(metricName, avro.Delimiter) - - lvl := l - - for i := 0; i < len(subString)-1; i++ { - - sel := subString[i] - - if lvl.children == nil { - lvl.children = make(map[string]*Level) - } - - child, ok := lvl.children[sel] - if !ok { - child = &Level{ - metrics: make([]*buffer, len(m.Metrics)), - children: nil, - } - lvl.children[sel] = child - } - lvl = child - } - - leafMetricName := subString[len(subString)-1] - err = lvl.createBuffer(m, leafMetricName, floatArray, from_timestamp, resolution) - if err != nil { - return fmt.Errorf("error while creating buffers from avroReader : %s", err) - } - } else { - err = l.createBuffer(m, metricName, floatArray, from_timestamp, resolution) - if err != nil { - return fmt.Errorf("error while creating buffers from avroReader : %s", err) - } - } - - } - - return nil -} - -func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray util.FloatArray, from int64, resolution int64) error { - n := len(floatArray) - b := &buffer{ - frequency: resolution, - start: from, - data: floatArray[0:n:n], - prev: nil, - next: nil, - archived: true, - } - b.close() - - minfo, ok := m.Metrics[metricName] - if !ok { - return nil - // return errors.New("Unkown metric: " + name) - } - - prev := l.metrics[minfo.Offset] - if prev == nil { - l.metrics[minfo.Offset] = b - } else { - if prev.start > b.start { - return errors.New("wooops") - } - - b.prev = prev - prev.next = b - - missingCount := ((int(b.start) - int(prev.start)) - len(prev.data)*int(b.frequency)) - if missingCount > 0 { - missingCount /= int(b.frequency) - - for range missingCount { - prev.data = append(prev.data, util.NaN) - } - - prev.data = prev.data[0:len(prev.data):len(prev.data)] - } - } - l.metrics[minfo.Offset] = b - - return nil -} - -func (l *Level) loadJsonFile(m *MemoryStore, f *os.File, from int64) error { - br := bufio.NewReader(f) - cf := &CheckpointFile{} - if err := json.NewDecoder(br).Decode(cf); err != nil { - return err - } - - if cf.To != 0 && cf.To < from { - return nil - } - - if err := l.loadFile(cf, m); err != nil { - return err - } - - return nil -} - -func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error { - for name, metric := range cf.Metrics { - n := len(metric.Data) - b := &buffer{ - frequency: metric.Frequency, - start: metric.Start, - data: metric.Data[0:n:n], // Space is wasted here :( - prev: nil, - next: nil, - archived: true, - } - b.close() - - minfo, ok := m.Metrics[name] - if !ok { - continue - // return errors.New("Unkown metric: " + name) - } - - prev := l.metrics[minfo.Offset] - if prev == nil { - l.metrics[minfo.Offset] = b - } else { - if prev.start > b.start { - return errors.New("wooops") - } - - b.prev = prev - prev.next = b - } - l.metrics[minfo.Offset] = b - } - - if len(cf.Children) > 0 && l.children == nil { - l.children = make(map[string]*Level) - } - - for sel, childCf := range cf.Children { - child, ok := l.children[sel] - if !ok { - child = &Level{ - metrics: make([]*buffer, len(m.Metrics)), - children: nil, - } - l.children[sel] = child - } - - if err := child.loadFile(childCf, m); err != nil { - return err - } - } - - return nil -} - -func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64, extension string) (int, error) { - direntries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return 0, nil - } - - return 0, err - } - - allFiles := make([]fs.DirEntry, 0) - filesLoaded := 0 - for _, e := range direntries { - if e.IsDir() { - child := &Level{ - metrics: make([]*buffer, len(m.Metrics)), - children: make(map[string]*Level), - } - - files, err := child.fromCheckpoint(m, path.Join(dir, e.Name()), from, extension) - filesLoaded += files - if err != nil { - return filesLoaded, err - } - - l.children[e.Name()] = child - } else if strings.HasSuffix(e.Name(), "."+extension) { - allFiles = append(allFiles, e) - } else { - continue - } - } - - files, err := findFiles(allFiles, from, extension, true) - if err != nil { - return filesLoaded, err - } - - loaders := map[string]func(*MemoryStore, *os.File, int64) error{ - "json": l.loadJsonFile, - "avro": l.loadAvroFile, - } - - loader := loaders[extension] - - for _, filename := range files { - f, err := os.Open(path.Join(dir, filename)) - if err != nil { - return filesLoaded, err - } - defer f.Close() - - if err = loader(m, f, from); err != nil { - return filesLoaded, err - } - - filesLoaded += 1 - } - - return filesLoaded, nil -} - -// This will probably get very slow over time! -// A solution could be some sort of an index file in which all other files -// and the timespan they contain is listed. -func findFiles(direntries []fs.DirEntry, t int64, extension string, findMoreRecentFiles bool) ([]string, error) { - nums := map[string]int64{} - for _, e := range direntries { - if !strings.HasSuffix(e.Name(), "."+extension) { - continue - } - - ts, err := strconv.ParseInt(e.Name()[strings.Index(e.Name(), "_")+1:len(e.Name())-5], 10, 64) - if err != nil { - return nil, err - } - nums[e.Name()] = ts - } - - sort.Slice(direntries, func(i, j int) bool { - a, b := direntries[i], direntries[j] - return nums[a.Name()] < nums[b.Name()] - }) - - filenames := make([]string, 0) - for i := 0; i < len(direntries); i++ { - e := direntries[i] - ts1 := nums[e.Name()] - - if findMoreRecentFiles && t <= ts1 { - filenames = append(filenames, e.Name()) - } - if i == len(direntries)-1 { - continue - } - - enext := direntries[i+1] - ts2 := nums[enext.Name()] - - if findMoreRecentFiles { - if ts1 < t && t < ts2 { - filenames = append(filenames, e.Name()) - } - } else { - if ts2 < t { - filenames = append(filenames, e.Name()) - } - } - } - - return filenames, nil -} diff --git a/internal/memorystore/debug.go b/internal/memorystore/debug.go deleted file mode 100644 index 2743a45..0000000 --- a/internal/memorystore/debug.go +++ /dev/null @@ -1,107 +0,0 @@ -package memorystore - -import ( - "bufio" - "fmt" - "strconv" -) - -func (b *buffer) debugDump(buf []byte) []byte { - if b.prev != nil { - buf = b.prev.debugDump(buf) - } - - start, len, end := b.start, len(b.data), b.start+b.frequency*int64(len(b.data)) - buf = append(buf, `{"start":`...) - buf = strconv.AppendInt(buf, start, 10) - buf = append(buf, `,"len":`...) - buf = strconv.AppendInt(buf, int64(len), 10) - buf = append(buf, `,"end":`...) - buf = strconv.AppendInt(buf, end, 10) - if b.archived { - buf = append(buf, `,"saved":true`...) - } - if b.next != nil { - buf = append(buf, `},`...) - } else { - buf = append(buf, `}`...) - } - return buf -} - -func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) { - l.lock.RLock() - defer l.lock.RUnlock() - for i := 0; i < depth; i++ { - buf = append(buf, '\t') - } - buf = append(buf, '"') - buf = append(buf, lvlname...) - buf = append(buf, "\":{\n"...) - depth += 1 - objitems := 0 - for name, mc := range m.Metrics { - if b := l.metrics[mc.Offset]; b != nil { - for i := 0; i < depth; i++ { - buf = append(buf, '\t') - } - - buf = append(buf, '"') - buf = append(buf, name...) - buf = append(buf, `":[`...) - buf = b.debugDump(buf) - buf = append(buf, "],\n"...) - objitems++ - } - } - - for name, lvl := range l.children { - _, err := w.Write(buf) - if err != nil { - return nil, err - } - - buf = buf[0:0] - buf, err = lvl.debugDump(m, w, name, buf, depth) - if err != nil { - return nil, err - } - - buf = append(buf, ',', '\n') - objitems++ - } - - // remove final `,`: - if objitems > 0 { - buf = append(buf[0:len(buf)-1], '\n') - } - - depth -= 1 - for i := 0; i < depth; i++ { - buf = append(buf, '\t') - } - buf = append(buf, '}') - return buf, nil -} - -func (m *MemoryStore) DebugDump(w *bufio.Writer, selector []string) error { - lvl := m.root.findLevel(selector) - if lvl == nil { - return fmt.Errorf("not found: %#v", selector) - } - - buf := make([]byte, 0, 2048) - buf = append(buf, "{"...) - - buf, err := lvl.debugDump(m, w, "data", buf, 0) - if err != nil { - return err - } - - buf = append(buf, "}\n"...) - if _, err = w.Write(buf); err != nil { - return err - } - - return w.Flush() -} diff --git a/internal/memorystore/healthcheck.go b/internal/memorystore/healthcheck.go deleted file mode 100644 index cb22d49..0000000 --- a/internal/memorystore/healthcheck.go +++ /dev/null @@ -1,88 +0,0 @@ -package memorystore - -import ( - "bufio" - "fmt" - "time" -) - -// This is a threshold that allows a node to be healthy with certain number of data points missing. -// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a -// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy. -const MaxMissingDataPoints int64 = 5 - -// This is a threshold which allows upto certain number of metrics in a node to be unhealthly. -// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last -// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does -// not receive data for MaxMissingDataPoints data points will deem the node unhealthy. -const MaxUnhealthyMetrics int64 = 5 - -func (b *buffer) healthCheck() int64 { - - // Check if the buffer is empty - if b.data == nil { - return 1 - } - - buffer_end := b.start + b.frequency*int64(len(b.data)) - t := time.Now().Unix() - - // Check if the buffer is too old - if t-buffer_end > MaxMissingDataPoints*b.frequency { - return 1 - } - - return 0 -} - -func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) { - l.lock.RLock() - defer l.lock.RUnlock() - - for _, mc := range m.Metrics { - if b := l.metrics[mc.Offset]; b != nil { - count += b.healthCheck() - } - } - - for _, lvl := range l.children { - c, err := lvl.healthCheck(m, 0) - if err != nil { - return 0, err - } - count += c - } - - return count, nil -} - -func (m *MemoryStore) HealthCheck(w *bufio.Writer, selector []string) error { - lvl := m.root.findLevel(selector) - if lvl == nil { - return fmt.Errorf("not found: %#v", selector) - } - - buf := make([]byte, 0, 25) - // buf = append(buf, "{"...) - - var count int64 = 0 - - unhealthyMetricsCount, err := lvl.healthCheck(m, count) - if err != nil { - return err - } - - if unhealthyMetricsCount < MaxUnhealthyMetrics { - buf = append(buf, "Healthy"...) - } else { - buf = append(buf, "Unhealthy"...) - } - - // buf = append(buf, "}\n"...) - - if _, err = w.Write(buf); err != nil { - return err - } - - return w.Flush() -} diff --git a/internal/memorystore/level.go b/internal/memorystore/level.go deleted file mode 100644 index f5f133a..0000000 --- a/internal/memorystore/level.go +++ /dev/null @@ -1,187 +0,0 @@ -package memorystore - -import ( - "sync" - "unsafe" - - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -// Could also be called "node" as this forms a node in a tree structure. -// Called Level because "node" might be confusing here. -// Can be both a leaf or a inner node. In this tree structue, inner nodes can -// also hold data (in `metrics`). -type Level struct { - children map[string]*Level - metrics []*buffer - lock sync.RWMutex -} - -// Find the correct level for the given selector, creating it if -// it does not exist. Example selector in the context of the -// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }. -// This function would probably benefit a lot from `level.children` beeing a `sync.Map`? -func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level { - if len(selector) == 0 { - return l - } - - // Allow concurrent reads: - l.lock.RLock() - var child *Level - var ok bool - if l.children == nil { - // Children map needs to be created... - l.lock.RUnlock() - } else { - child, ok := l.children[selector[0]] - l.lock.RUnlock() - if ok { - return child.findLevelOrCreate(selector[1:], nMetrics) - } - } - - // The level does not exist, take write lock for unqiue access: - l.lock.Lock() - // While this thread waited for the write lock, another thread - // could have created the child node. - if l.children != nil { - child, ok = l.children[selector[0]] - if ok { - l.lock.Unlock() - return child.findLevelOrCreate(selector[1:], nMetrics) - } - } - - child = &Level{ - metrics: make([]*buffer, nMetrics), - children: nil, - } - - if l.children != nil { - l.children[selector[0]] = child - } else { - l.children = map[string]*Level{selector[0]: child} - } - l.lock.Unlock() - return child.findLevelOrCreate(selector[1:], nMetrics) -} - -func (l *Level) free(t int64) (int, error) { - l.lock.Lock() - defer l.lock.Unlock() - - n := 0 - for i, b := range l.metrics { - if b != nil { - delme, m := b.free(t) - n += m - if delme { - if cap(b.data) == BUFFER_CAP { - bufferPool.Put(b) - } - l.metrics[i] = nil - } - } - } - - for _, l := range l.children { - m, err := l.free(t) - n += m - if err != nil { - return n, err - } - } - - return n, nil -} - -func (l *Level) sizeInBytes() int64 { - l.lock.RLock() - defer l.lock.RUnlock() - size := int64(0) - - for _, b := range l.metrics { - if b != nil { - size += b.count() * int64(unsafe.Sizeof(util.Float(0))) - } - } - - for _, child := range l.children { - size += child.sizeInBytes() - } - - return size -} - -func (l *Level) findLevel(selector []string) *Level { - if len(selector) == 0 { - return l - } - - l.lock.RLock() - defer l.lock.RUnlock() - - lvl := l.children[selector[0]] - if lvl == nil { - return nil - } - - return lvl.findLevel(selector[1:]) -} - -func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error { - l.lock.RLock() - defer l.lock.RUnlock() - - if len(selector) == 0 { - b := l.metrics[offset] - if b != nil { - return f(b) - } - - for _, lvl := range l.children { - err := lvl.findBuffers(nil, offset, f) - if err != nil { - return err - } - } - return nil - } - - sel := selector[0] - if len(sel.String) != 0 && l.children != nil { - lvl, ok := l.children[sel.String] - if ok { - err := lvl.findBuffers(selector[1:], offset, f) - if err != nil { - return err - } - } - return nil - } - - if sel.Group != nil && l.children != nil { - for _, key := range sel.Group { - lvl, ok := l.children[key] - if ok { - err := lvl.findBuffers(selector[1:], offset, f) - if err != nil { - return err - } - } - } - return nil - } - - if sel.Any && l.children != nil { - for _, lvl := range l.children { - if err := lvl.findBuffers(selector[1:], offset, f); err != nil { - return err - } - } - return nil - } - - return nil -} diff --git a/internal/memorystore/memorystore.go b/internal/memorystore/memorystore.go deleted file mode 100644 index 959a582..0000000 --- a/internal/memorystore/memorystore.go +++ /dev/null @@ -1,373 +0,0 @@ -package memorystore - -import ( - "context" - "errors" - "log" - "runtime" - "sync" - "time" - - "github.com/ClusterCockpit/cc-metric-store/internal/avro" - "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/util" - "github.com/ClusterCockpit/cc-metric-store/pkg/resampler" -) - -var ( - singleton sync.Once - msInstance *MemoryStore -) - -var NumWorkers int = 4 - -func init() { - maxWorkers := 10 - NumWorkers = runtime.NumCPU()/2 + 1 - if NumWorkers > maxWorkers { - NumWorkers = maxWorkers - } -} - -type Metric struct { - Name string - Value util.Float - MetricConfig config.MetricConfig -} - -type MemoryStore struct { - Metrics map[string]config.MetricConfig - root Level -} - -// Create a new, initialized instance of a MemoryStore. -// Will panic if values in the metric configurations are invalid. -func Init(metrics map[string]config.MetricConfig) { - singleton.Do(func() { - offset := 0 - for key, cfg := range metrics { - if cfg.Frequency == 0 { - panic("invalid frequency") - } - - metrics[key] = config.MetricConfig{ - Frequency: cfg.Frequency, - Aggregation: cfg.Aggregation, - Offset: offset, - } - offset += 1 - } - - msInstance = &MemoryStore{ - root: Level{ - metrics: make([]*buffer, len(metrics)), - children: make(map[string]*Level), - }, - Metrics: metrics, - } - }) -} - -func GetMemoryStore() *MemoryStore { - if msInstance == nil { - log.Fatalf("MemoryStore not initialized!") - } - - return msInstance -} - -func Shutdown() { - log.Printf("Writing to '%s'...\n", config.Keys.Checkpoints.RootDir) - var files int - var err error - - ms := GetMemoryStore() - - if config.Keys.Checkpoints.FileFormat == "json" { - files, err = ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix()) - } else { - files, err = avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, true) - close(avro.LineProtocolMessages) - } - - if err != nil { - log.Printf("Writing checkpoint failed: %s\n", err.Error()) - } - log.Printf("Done! (%d files written)\n", files) - - // ms.PrintHeirarchy() -} - -// func (m *MemoryStore) PrintHeirarchy() { -// m.root.lock.Lock() -// defer m.root.lock.Unlock() - -// fmt.Printf("Root : \n") - -// for lvl1, sel1 := range m.root.children { -// fmt.Printf("\t%s\n", lvl1) -// for lvl2, sel2 := range sel1.children { -// fmt.Printf("\t\t%s\n", lvl2) -// if lvl1 == "fritz" && lvl2 == "f0201" { - -// for name, met := range m.Metrics { -// mt := sel2.metrics[met.Offset] - -// fmt.Printf("\t\t\t\t%s\n", name) -// fmt.Printf("\t\t\t\t") - -// for mt != nil { -// // if name == "cpu_load" { -// fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data) -// // } -// mt = mt.prev -// } -// fmt.Printf("\n") - -// } -// } -// for lvl3, sel3 := range sel2.children { -// if lvl1 == "fritz" && lvl2 == "f0201" && lvl3 == "hwthread70" { - -// fmt.Printf("\t\t\t\t\t%s\n", lvl3) - -// for name, met := range m.Metrics { -// mt := sel3.metrics[met.Offset] - -// fmt.Printf("\t\t\t\t\t\t%s\n", name) - -// fmt.Printf("\t\t\t\t\t\t") - -// for mt != nil { -// // if name == "clock" { -// fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data) - -// mt = mt.prev -// } -// fmt.Printf("\n") - -// } - -// // for i, _ := range sel3.metrics { -// // fmt.Printf("\t\t\t\t\t%s\n", getName(configmetrics, i)) -// // } -// } -// } -// } -// } - -// } - -func getName(m *MemoryStore, i int) string { - for key, val := range m.Metrics { - if val.Offset == i { - return key - } - } - return "" -} - -func Retention(wg *sync.WaitGroup, ctx context.Context) { - ms := GetMemoryStore() - - go func() { - defer wg.Done() - d, err := time.ParseDuration(config.Keys.RetentionInMemory) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := func() <-chan time.Time { - d := d / 2 - if d <= 0 { - return nil - } - return time.NewTicker(d).C - }() - for { - select { - case <-ctx.Done(): - return - case <-ticks: - t := time.Now().Add(-d) - log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) - freed, err := ms.Free(nil, t.Unix()) - if err != nil { - log.Printf("freeing up buffers failed: %s\n", err.Error()) - } else { - log.Printf("done: %d buffers freed\n", freed) - } - } - } - }() -} - -// Write all values in `metrics` to the level specified by `selector` for time `ts`. -// Look at `findLevelOrCreate` for how selectors work. -func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error { - var ok bool - for i, metric := range metrics { - if metric.MetricConfig.Frequency == 0 { - metric.MetricConfig, ok = m.Metrics[metric.Name] - if !ok { - metric.MetricConfig.Frequency = 0 - } - metrics[i] = metric - } - } - - return m.WriteToLevel(&m.root, selector, ts, metrics) -} - -func (m *MemoryStore) GetLevel(selector []string) *Level { - return m.root.findLevelOrCreate(selector, len(m.Metrics)) -} - -// Assumes that `minfo` in `metrics` is filled in! -func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error { - l = l.findLevelOrCreate(selector, len(m.Metrics)) - l.lock.Lock() - defer l.lock.Unlock() - - for _, metric := range metrics { - if metric.MetricConfig.Frequency == 0 { - continue - } - - b := l.metrics[metric.MetricConfig.Offset] - if b == nil { - // First write to this metric and level - b = newBuffer(ts, metric.MetricConfig.Frequency) - l.metrics[metric.MetricConfig.Offset] = b - } - - nb, err := b.write(ts, metric.Value) - if err != nil { - return err - } - - // Last write created a new buffer... - if b != nb { - l.metrics[metric.MetricConfig.Offset] = nb - } - } - return nil -} - -// Returns all values for metric `metric` from `from` to `to` for the selected level(s). -// If the level does not hold the metric itself, the data will be aggregated recursively from the children. -// The second and third return value are the actual from/to for the data. Those can be different from -// the range asked for if no data was available. -func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]util.Float, int64, int64, int64, error) { - if from > to { - return nil, 0, 0, 0, errors.New("invalid time range") - } - - minfo, ok := m.Metrics[metric] - if !ok { - return nil, 0, 0, 0, errors.New("unkown metric: " + metric) - } - - n, data := 0, make([]util.Float, (to-from)/minfo.Frequency+1) - - err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error { - cdata, cfrom, cto, err := b.read(from, to, data) - if err != nil { - return err - } - - if n == 0 { - from, to = cfrom, cto - } else if from != cfrom || to != cto || len(data) != len(cdata) { - missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency) - if missingfront != 0 { - return ErrDataDoesNotAlign - } - - newlen := len(cdata) - missingback - if newlen < 1 { - return ErrDataDoesNotAlign - } - cdata = cdata[0:newlen] - if len(cdata) != len(data) { - return ErrDataDoesNotAlign - } - - from, to = cfrom, cto - } - - data = cdata - n += 1 - return nil - }) - - if err != nil { - return nil, 0, 0, 0, err - } else if n == 0 { - return nil, 0, 0, 0, errors.New("metric or host not found") - } else if n > 1 { - if minfo.Aggregation == config.AvgAggregation { - normalize := 1. / util.Float(n) - for i := 0; i < len(data); i++ { - data[i] *= normalize - } - } else if minfo.Aggregation != config.SumAggregation { - return nil, 0, 0, 0, errors.New("invalid aggregation") - } - } - - data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution) - - if err != nil { - return nil, 0, 0, 0, err - } - - return data, from, to, resolution, nil -} - -// Release all buffers for the selected level and all its children that contain only -// values older than `t`. -func (m *MemoryStore) Free(selector []string, t int64) (int, error) { - return m.GetLevel(selector).free(t) -} - -func (m *MemoryStore) FreeAll() error { - for k := range m.root.children { - delete(m.root.children, k) - } - - return nil -} - -func (m *MemoryStore) SizeInBytes() int64 { - return m.root.sizeInBytes() -} - -// Given a selector, return a list of all children of the level selected. -func (m *MemoryStore) ListChildren(selector []string) []string { - lvl := &m.root - for lvl != nil && len(selector) != 0 { - lvl.lock.RLock() - next := lvl.children[selector[0]] - lvl.lock.RUnlock() - lvl = next - selector = selector[1:] - } - - if lvl == nil { - return nil - } - - lvl.lock.RLock() - defer lvl.lock.RUnlock() - - children := make([]string, 0, len(lvl.children)) - for child := range lvl.children { - children = append(children, child) - } - - return children -} diff --git a/internal/memorystore/stats.go b/internal/memorystore/stats.go deleted file mode 100644 index 5ddecfc..0000000 --- a/internal/memorystore/stats.go +++ /dev/null @@ -1,120 +0,0 @@ -package memorystore - -import ( - "errors" - "math" - - "github.com/ClusterCockpit/cc-metric-store/internal/config" - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -type Stats struct { - Samples int - Avg util.Float - Min util.Float - Max util.Float -} - -func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) { - if from < b.start { - if b.prev != nil { - return b.prev.stats(from, to) - } - from = b.start - } - - // TODO: Check if b.closed and if so and the full buffer is queried, - // use b.statistics instead of iterating over the buffer. - - samples := 0 - sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32 - - var t int64 - for t = from; t < to; t += b.frequency { - idx := int((t - b.start) / b.frequency) - if idx >= cap(b.data) { - b = b.next - if b == nil { - break - } - idx = 0 - } - - if t < b.start || idx >= len(b.data) { - continue - } - - xf := float64(b.data[idx]) - if math.IsNaN(xf) { - continue - } - - samples += 1 - sum += xf - min = math.Min(min, xf) - max = math.Max(max, xf) - } - - return Stats{ - Samples: samples, - Avg: util.Float(sum) / util.Float(samples), - Min: util.Float(min), - Max: util.Float(max), - }, from, t, nil -} - -// Returns statistics for the requested metric on the selected node/level. -// Data is aggregated to the selected level the same way as in `MemoryStore.Read`. -// If `Stats.Samples` is zero, the statistics should not be considered as valid. -func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) { - if from > to { - return nil, 0, 0, errors.New("invalid time range") - } - - minfo, ok := m.Metrics[metric] - if !ok { - return nil, 0, 0, errors.New("unkown metric: " + metric) - } - - n, samples := 0, 0 - avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32 - err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error { - stats, cfrom, cto, err := b.stats(from, to) - if err != nil { - return err - } - - if n == 0 { - from, to = cfrom, cto - } else if from != cfrom || to != cto { - return ErrDataDoesNotAlign - } - - samples += stats.Samples - avg += stats.Avg - min = math.Min(min, float64(stats.Min)) - max = math.Max(max, float64(stats.Max)) - n += 1 - return nil - }) - if err != nil { - return nil, 0, 0, err - } - - if n == 0 { - return nil, 0, 0, ErrNoData - } - - if minfo.Aggregation == config.AvgAggregation { - avg /= util.Float(n) - } else if n > 1 && minfo.Aggregation != config.SumAggregation { - return nil, 0, 0, errors.New("invalid aggregation") - } - - return &Stats{ - Samples: samples, - Avg: avg, - Min: util.Float(min), - Max: util.Float(max), - }, from, to, nil -} diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go deleted file mode 100644 index e999154..0000000 --- a/internal/runtimeEnv/setup.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. -// All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. -package runtimeEnv - -import ( - "bufio" - "errors" - "fmt" - "os" - "os/exec" - "os/user" - "strconv" - "strings" - "syscall" -) - -// Very simple and limited .env file reader. -// All variable definitions found are directly -// added to the processes environment. -func LoadEnv(file string) error { - f, err := os.Open(file) - if err != nil { - // log.Error("Error while opening .env file") - return err - } - - defer f.Close() - s := bufio.NewScanner(bufio.NewReader(f)) - for s.Scan() { - line := s.Text() - if strings.HasPrefix(line, "#") || len(line) == 0 { - continue - } - - if strings.Contains(line, "#") { - return errors.New("'#' are only supported at the start of a line") - } - - line = strings.TrimPrefix(line, "export ") - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line) - } - - key := strings.TrimSpace(parts[0]) - val := strings.TrimSpace(parts[1]) - if strings.HasPrefix(val, "\"") { - if !strings.HasSuffix(val, "\"") { - return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line) - } - - runes := []rune(val[1 : len(val)-1]) - sb := strings.Builder{} - for i := 0; i < len(runes); i++ { - if runes[i] == '\\' { - i++ - switch runes[i] { - case 'n': - sb.WriteRune('\n') - case 'r': - sb.WriteRune('\r') - case 't': - sb.WriteRune('\t') - case '"': - sb.WriteRune('"') - default: - return fmt.Errorf("RUNTIME/SETUP > unsupported escape sequence in quoted string: backslash %#v", runes[i]) - } - continue - } - sb.WriteRune(runes[i]) - } - - val = sb.String() - } - - os.Setenv(key, val) - } - - return s.Err() -} - -// Changes the processes user and group to that -// specified in the config.json. The go runtime -// takes care of all threads (and not only the calling one) -// executing the underlying systemcall. -func DropPrivileges(username string, group string) error { - if group != "" { - g, err := user.LookupGroup(group) - if err != nil { - // log.Warn("Error while looking up group") - return err - } - - gid, _ := strconv.Atoi(g.Gid) - if err := syscall.Setgid(gid); err != nil { - // log.Warn("Error while setting gid") - return err - } - } - - if username != "" { - u, err := user.Lookup(username) - if err != nil { - // log.Warn("Error while looking up user") - return err - } - - uid, _ := strconv.Atoi(u.Uid) - if err := syscall.Setuid(uid); err != nil { - // log.Warn("Error while setting uid") - return err - } - } - - return nil -} - -// If started via systemd, inform systemd that we are running: -// https://www.freedesktop.org/software/systemd/man/sd_notify.html -func SystemdNotifiy(ready bool, status string) { - if os.Getenv("NOTIFY_SOCKET") == "" { - // Not started using systemd - return - } - - args := []string{fmt.Sprintf("--pid=%d", os.Getpid())} - if ready { - args = append(args, "--ready") - } - - if status != "" { - args = append(args, fmt.Sprintf("--status=%s", status)) - } - - cmd := exec.Command("systemd-notify", args...) - cmd.Run() // errors ignored on purpose, there is not much to do anyways. -} diff --git a/internal/util/float.go b/internal/util/float.go deleted file mode 100644 index 1ebd36b..0000000 --- a/internal/util/float.go +++ /dev/null @@ -1,76 +0,0 @@ -package util - -import ( - "math" - "strconv" -) - -// Go's JSON encoder for floats does not support NaN (https://github.com/golang/go/issues/3480). -// This program uses NaN as a signal for missing data. -// For the HTTP JSON API to be able to handle NaN values, -// we have to use our own type which implements encoding/json.Marshaler itself. -type Float float64 - -var ( - NaN Float = Float(math.NaN()) - nullAsBytes []byte = []byte("null") -) - -func (f Float) IsNaN() bool { - return math.IsNaN(float64(f)) -} - -func (f Float) MarshalJSON() ([]byte, error) { - if math.IsNaN(float64(f)) { - return nullAsBytes, nil - } - - return strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 3, 64), nil -} - -func (f Float) Double() float64 { - return float64(f) -} - -func (f *Float) UnmarshalJSON(input []byte) error { - if string(input) == "null" { - *f = NaN - return nil - } - - val, err := strconv.ParseFloat(string(input), 64) - if err != nil { - return err - } - *f = Float(val) - return nil -} - -// Same as `[]Float`, but can be marshaled to JSON with less allocations. -type FloatArray []Float - -func ConvertToFloat(input float64) Float { - if input == -1.0 { - return NaN - } else { - return Float(input) - } -} - -func (fa FloatArray) MarshalJSON() ([]byte, error) { - buf := make([]byte, 0, 2+len(fa)*8) - buf = append(buf, '[') - for i := 0; i < len(fa); i++ { - if i != 0 { - buf = append(buf, ',') - } - - if fa[i].IsNaN() { - buf = append(buf, `null`...) - } else { - buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64) - } - } - buf = append(buf, ']') - return buf, nil -} diff --git a/internal/util/selector.go b/internal/util/selector.go deleted file mode 100644 index 27557ef..0000000 --- a/internal/util/selector.go +++ /dev/null @@ -1,51 +0,0 @@ -package util - -import ( - "encoding/json" - "errors" -) - -type SelectorElement struct { - String string - Group []string - Any bool -} - -func (se *SelectorElement) UnmarshalJSON(input []byte) error { - if input[0] == '"' { - if err := json.Unmarshal(input, &se.String); err != nil { - return err - } - - if se.String == "*" { - se.Any = true - se.String = "" - } - - return nil - } - - if input[0] == '[' { - return json.Unmarshal(input, &se.Group) - } - - return errors.New("the Go SelectorElement type can only be a string or an array of strings") -} - -func (se *SelectorElement) MarshalJSON() ([]byte, error) { - if se.Any { - return []byte("\"*\""), nil - } - - if se.String != "" { - return json.Marshal(se.String) - } - - if se.Group != nil { - return json.Marshal(se.Group) - } - - return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings") -} - -type Selector []SelectorElement diff --git a/pkg/resampler/resampler.go b/pkg/resampler/resampler.go deleted file mode 100644 index c641670..0000000 --- a/pkg/resampler/resampler.go +++ /dev/null @@ -1,122 +0,0 @@ -package resampler - -import ( - "errors" - "fmt" - "math" - - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) { - if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { - return data, old_frequency, nil - } - - if new_frequency%old_frequency != 0 { - return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency") - } - - var step int = int(new_frequency / old_frequency) - var new_data_length = len(data) / step - - if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) { - return data, old_frequency, nil - } - - new_data := make([]util.Float, new_data_length) - - for i := 0; i < new_data_length; i++ { - new_data[i] = data[i*step] - } - - return new_data, new_frequency, nil -} - -// Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf -// Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go -func LargestTriangleThreeBucket(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) { - - if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency { - return data, old_frequency, nil - } - - if new_frequency%old_frequency != 0 { - return nil, 0, fmt.Errorf("new sampling frequency : %d should be multiple of the old frequency : %d", new_frequency, old_frequency) - } - - var step int = int(new_frequency / old_frequency) - var new_data_length = len(data) / step - - if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) { - return data, old_frequency, nil - } - - new_data := make([]util.Float, 0, new_data_length) - - // Bucket size. Leave room for start and end data points - bucketSize := float64(len(data)-2) / float64(new_data_length-2) - - new_data = append(new_data, data[0]) // Always add the first point - - // We have 3 pointers represent for - // > bucketLow - the current bucket's beginning location - // > bucketMiddle - the current bucket's ending location, - // also the beginning location of next bucket - // > bucketHight - the next bucket's ending location. - bucketLow := 1 - bucketMiddle := int(math.Floor(bucketSize)) + 1 - - var prevMaxAreaPoint int - - for i := 0; i < new_data_length-2; i++ { - - bucketHigh := int(math.Floor(float64(i+2)*bucketSize)) + 1 - if bucketHigh >= len(data)-1 { - bucketHigh = len(data) - 2 - } - - // Calculate point average for next bucket (containing c) - avgPointX, avgPointY := calculateAverageDataPoint(data[bucketMiddle:bucketHigh+1], int64(bucketMiddle)) - - // Get the range for current bucket - currBucketStart := bucketLow - currBucketEnd := bucketMiddle - - // Point a - pointX := prevMaxAreaPoint - pointY := data[prevMaxAreaPoint] - - maxArea := -1.0 - - var maxAreaPoint int - flag_ := 0 - for ; currBucketStart < currBucketEnd; currBucketStart++ { - - area := calculateTriangleArea(util.Float(pointX), pointY, avgPointX, avgPointY, util.Float(currBucketStart), data[currBucketStart]) - if area > maxArea { - maxArea = area - maxAreaPoint = currBucketStart - } - if math.IsNaN(float64(avgPointY)) { - flag_ = 1 - } - } - - if flag_ == 1 { - new_data = append(new_data, util.NaN) // Pick this point from the bucket - - } else { - new_data = append(new_data, data[maxAreaPoint]) // Pick this point from the bucket - } - prevMaxAreaPoint = maxAreaPoint // This MaxArea point is the next's prevMAxAreaPoint - - //move to the next window - bucketLow = bucketMiddle - bucketMiddle = bucketHigh - } - - new_data = append(new_data, data[len(data)-1]) // Always add last - - return new_data, new_frequency, nil -} diff --git a/pkg/resampler/util.go b/pkg/resampler/util.go deleted file mode 100644 index 61397bf..0000000 --- a/pkg/resampler/util.go +++ /dev/null @@ -1,35 +0,0 @@ -package resampler - -import ( - "math" - - "github.com/ClusterCockpit/cc-metric-store/internal/util" -) - -func calculateTriangleArea(paX, paY, pbX, pbY, pcX, pcY util.Float) float64 { - area := ((paX-pcX)*(pbY-paY) - (paX-pbX)*(pcY-paY)) * 0.5 - return math.Abs(float64(area)) -} - -func calculateAverageDataPoint(points []util.Float, xStart int64) (avgX util.Float, avgY util.Float) { - flag := 0 - for _, point := range points { - avgX += util.Float(xStart) - avgY += point - xStart++ - if math.IsNaN(float64(point)) { - flag = 1 - } - } - - l := util.Float(len(points)) - - avgX /= l - avgY /= l - - if flag == 1 { - return avgX, util.NaN - } else { - return avgX, avgY - } -} diff --git a/tools.go b/tools.go deleted file mode 100644 index bcee2f6..0000000 --- a/tools.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build tools -// +build tools - -package tools - -import ( - _ "github.com/swaggo/swag/cmd/swag" -)