     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"bufio"
    10→	"encoding/json"
    11→	"errors"
    12→	"fmt"
    13→	"net/http"
    14→	"strconv"
    15→	"strings"
    16→
    17→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→
    20→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    21→)
    22→
    23→// handleFree godoc
    24→// @summary
    25→// @tags free
    26→// @description This endpoint allows the users to free the Buffers from the
    27→// metric store. This endpoint offers the users to remove then systematically
    28→// and also allows then to prune the data under node, if they do not want to
    29→// remove the whole node.
    30→// @produce     json
    31→// @param       to        query    string        false  "up to timestamp"
    32→// @success     200            {string} string  "ok"
    33→// @failure     400            {object} api.ErrorResponse       "Bad Request"
    34→// @failure     401            {object} api.ErrorResponse       "Unauthorized"
    35→// @failure     403            {object} api.ErrorResponse       "Forbidden"
    36→// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
    37→// @security    ApiKeyAuth
    38→// @router      /free/ [post]
    39→func freeMetrics(rw http.ResponseWriter, r *http.Request) {
    40→	rawTo := r.URL.Query().Get("to")
    41→	if rawTo == "" {
    42→		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
    43→		return
    44→	}
    45→
    46→	to, err := strconv.ParseInt(rawTo, 10, 64)
    47→	if err != nil {
    48→		handleError(err, http.StatusInternalServerError, rw)
    49→		return
    50→	}
    51→
    52→	bodyDec := json.NewDecoder(r.Body)
    53→	var selectors [][]string
    54→	err = bodyDec.Decode(&selectors)
    55→	if err != nil {
    56→		http.Error(rw, err.Error(), http.StatusBadRequest)
    57→		return
    58→	}
    59→
    60→	ms := metricstore.GetMemoryStore()
    61→	n := 0
    62→	for _, sel := range selectors {
    63→		bn, err := ms.Free(sel, to)
    64→		if err != nil {
    65→			handleError(err, http.StatusInternalServerError, rw)
    66→			return
    67→		}
    68→
    69→		n += bn
    70→	}
    71→
    72→	rw.WriteHeader(http.StatusOK)
    73→	fmt.Fprintf(rw, "buffers freed: %d\n", n)
    74→}
    75→
    76→// handleWrite godoc
    77→// @summary Receive metrics in InfluxDB line-protocol
    78→// @tags write
    79→// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
    80→
    81→// @accept      plain
    82→// @produce     json
    83→// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
    84→// @success     200            {string} string  "ok"
    85→// @failure     400            {object} api.ErrorResponse       "Bad Request"
    86→// @failure     401            {object} api.ErrorResponse       "Unauthorized"
    87→// @failure     403            {object} api.ErrorResponse       "Forbidden"
    88→// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
    89→// @security    ApiKeyAuth
    90→// @router      /write/ [post]
    91→func writeMetrics(rw http.ResponseWriter, r *http.Request) {
    92→	rw.Header().Add("Content-Type", "application/json")
    93→
    94→	// Extract the "cluster" query parameter without allocating a url.Values map.
    95→	cluster := queryParam(r.URL.RawQuery, "cluster")
    96→
    97→	// Stream directly from the request body instead of copying it into a
    98→	// temporary buffer via io.ReadAll. The line-protocol decoder supports
    99→	// io.Reader natively, so this avoids the largest heap allocation.
   100→	ms := metricstore.GetMemoryStore()
   101→	dec := lineprotocol.NewDecoder(r.Body)
   102→	if err := metricstore.DecodeLine(dec, ms, cluster); err != nil {
   103→		cclog.Errorf("/api/write error: %s", err.Error())
   104→		handleError(err, http.StatusBadRequest, rw)
   105→		return
   106→	}
   107→	rw.WriteHeader(http.StatusOK)
   108→}
   109→
   110→// queryParam extracts a single query-parameter value from a raw query string
   111→// without allocating a url.Values map. Returns "" if the key is not present.
   112→func queryParam(raw, key string) string {
   113→	for raw != "" {
   114→		var kv string
   115→		kv, raw, _ = strings.Cut(raw, "&")
   116→		k, v, _ := strings.Cut(kv, "=")
   117→		if k == key {
   118→			return v
   119→		}
   120→	}
   121→	return ""
   122→}
   123→
   124→// handleDebug godoc
   125→// @summary Debug endpoint
   126→// @tags debug
   127→// @description This endpoint allows the users to print the content of
   128→// nodes/clusters/metrics to review the state of the data.
   129→// @produce     json
   130→// @param       selector        query    string            false "Selector"
   131→// @success     200            {string} string  "Debug dump"
   132→// @failure     400            {object} api.ErrorResponse       "Bad Request"
   133→// @failure     401            {object} api.ErrorResponse       "Unauthorized"
   134→// @failure     403            {object} api.ErrorResponse       "Forbidden"
   135→// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
   136→// @security    ApiKeyAuth
   137→// @router      /debug/ [post]
   138→func debugMetrics(rw http.ResponseWriter, r *http.Request) {
   139→	raw := r.URL.Query().Get("selector")
   140→	rw.Header().Add("Content-Type", "application/json")
   141→	selector := []string{}
   142→	if len(raw) != 0 {
   143→		selector = strings.Split(raw, ":")
   144→	}
   145→
   146→	ms := metricstore.GetMemoryStore()
   147→	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
   148→		handleError(err, http.StatusBadRequest, rw)
   149→		return
   150→	}
   151→}
   152→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"

	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// handleFree godoc
// @summary
// @tags free
// @description This endpoint allows the users to free the Buffers from the
// metric store. This endpoint offers the users to remove then systematically
// and also allows then to prune the data under node, if they do not want to
// remove the whole node.
// @produce     json
// @param       to        query    string        false  "up to timestamp"
// @success     200            {string} string  "ok"
// @failure     400            {object} api.ErrorResponse       "Bad Request"
// @failure     401            {object} api.ErrorResponse       "Unauthorized"
// @failure     403            {object} api.ErrorResponse       "Forbidden"
// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
// @security    ApiKeyAuth
// @router      /free/ [post]
func freeMetrics(rw http.ResponseWriter, r *http.Request) {
	rawTo := r.URL.Query().Get("to")
	if rawTo == "" {
		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
		return
	}

	to, err := strconv.ParseInt(rawTo, 10, 64)
	if err != nil {
		handleError(err, http.StatusInternalServerError, rw)
		return
	}

	bodyDec := json.NewDecoder(r.Body)
	var selectors [][]string
	err = bodyDec.Decode(&selectors)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusBadRequest)
		return
	}

	ms := metricstore.GetMemoryStore()
	n := 0
	for _, sel := range selectors {
		bn, err := ms.Free(sel, to)
		if err != nil {
			handleError(err, http.StatusInternalServerError, rw)
			return
		}

		n += bn
	}

	rw.WriteHeader(http.StatusOK)
	fmt.Fprintf(rw, "buffers freed: %d\n", n)
}

// handleWrite godoc
// @summary Receive metrics in InfluxDB line-protocol
// @tags write
// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)

// @accept      plain
// @produce     json
// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
// @success     200            {string} string  "ok"
// @failure     400            {object} api.ErrorResponse       "Bad Request"
// @failure     401            {object} api.ErrorResponse       "Unauthorized"
// @failure     403            {object} api.ErrorResponse       "Forbidden"
// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
// @security    ApiKeyAuth
// @router      /write/ [post]
func writeMetrics(rw http.ResponseWriter, r *http.Request) {
	rw.Header().Add("Content-Type", "application/json")

	// Extract the "cluster" query parameter without allocating a url.Values map.
	cluster := queryParam(r.URL.RawQuery, "cluster")

	// Stream directly from the request body instead of copying it into a
	// temporary buffer via io.ReadAll. The line-protocol decoder supports
	// io.Reader natively, so this avoids the largest heap allocation.
	ms := metricstore.GetMemoryStore()
	dec := lineprotocol.NewDecoder(r.Body)
	if err := metricstore.DecodeLine(dec, ms, cluster); err != nil {
		cclog.Errorf("/api/write error: %s", err.Error())
		handleError(err, http.StatusBadRequest, rw)
		return
	}
	rw.WriteHeader(http.StatusOK)
}

// queryParam extracts a single query-parameter value from a raw query string
// without allocating a url.Values map. Returns "" if the key is not present.
func queryParam(raw, key string) string {
	for raw != "" {
		var kv string
		kv, raw, _ = strings.Cut(raw, "&")
		k, v, _ := strings.Cut(kv, "=")
		if k == key {
			return v
		}
	}
	return ""
}

// handleDebug godoc
// @summary Debug endpoint
// @tags debug
// @description This endpoint allows the users to print the content of
// nodes/clusters/metrics to review the state of the data.
// @produce     json
// @param       selector        query    string            false "Selector"
// @success     200            {string} string  "Debug dump"
// @failure     400            {object} api.ErrorResponse       "Bad Request"
// @failure     401            {object} api.ErrorResponse       "Unauthorized"
// @failure     403            {object} api.ErrorResponse       "Forbidden"
// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
// @security    ApiKeyAuth
// @router      /debug/ [post]
func debugMetrics(rw http.ResponseWriter, r *http.Request) {
	raw := r.URL.Query().Get("selector")
	rw.Header().Add("Content-Type", "application/json")
	selector := []string{}
	if len(raw) != 0 {
		selector = strings.Split(raw, ":")
	}

	ms := metricstore.GetMemoryStore()
	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
		handleError(err, http.StatusBadRequest, rw)
		return
	}
}


No, the REST path does **not** have the same problem.

Looking at `internal/api/metricstore.go:91-108`, the REST `writeMetrics` handler calls `DecodeLine()` synchronously within the HTTP request goroutine. Each HTTP request gets its own goroutine from Go's `net/http` server, so:

1. **No single-threaded bottleneck** — Go's HTTP server spawns a goroutine per request, so multiple concurrent POST requests to `/api/write/` decode in parallel naturally.
2. **No callback blocking** — There's no subscription callback to block. If `DecodeLine` is slow, it only delays that one HTTP response, which is fine.
3. **WAL channel send still applies** — The blocking WAL send (now fixed to non-blocking) could have stalled an HTTP handler goroutine, but it wouldn't cause message drops — it would just increase HTTP response latency. With the non-blocking change, this is no longer an issue either.

The slow consumer problem is specific to the NATS ingestion path because the nats.go client uses a single callback delivery model with an internal pending buffer that drops messages when the callback can't keep up.


How many os level threads are used in this application? It is currently configurable?


pkg/metricstore/metricstore.go:109://  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
configs/startJobPayload.json:18:    "slurmInfo": "JobId=398759\nJobName=myJob\nUserId=dummyUser\nGroupId=dummyGroup\nAccount=dummyAccount\nQOS=normal Requeue=False Restarts=0 BatchFlag=True\nTimeLimit=1439'\nSubmitTime=2023-02-09T14:10:18\nPartition=singlenode\nNodeList=xx\nNumNodes=xx NumCPUs=72 NumTasks=72 CPUs/Task=1\nNTasksPerNode:Socket:Core=0:None:None\nTRES_req=cpu=72,mem=250000M,node=1,billing=72\nTRES_alloc=cpu=72,node=1,billing=72\nCommand=myCmd\nWorkDir=myDir\nStdErr=\nStdOut=\n",
pkg/metricstore/config.go:114://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
pkg/metricstore/config.go:123:	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
internal/graph/schema.resolvers.go:788:		numThreadsInt := int(job.NumHWThreads)
internal/graph/schema.resolvers.go:798:			NumHWThreads:    &numThreadsInt,
internal/importer/testdata/meta-fritzError.input:1:[Omitted long matching line]
web/frontend/src/generic/filters/Resources.svelte:262:        <Icon id="numthreads-info" style="cursor:help; padding-right: 10px;" size="sm" name="info-circle"/>
web/frontend/src/generic/filters/Resources.svelte:264:      <Tooltip target={`numthreads-info`} placement="right">


**No, `GOMAXPROCS` is never set explicitly in this application.** It uses the Go default, which equals the number of logical CPUs visible to the process (i.e., `runtime.NumCPU()`). It is not configurable via `config.json` or any flag.

To clarify the distinction:

- **OS threads (`GOMAXPROCS`)**: Controls how many goroutines can execute *simultaneously* on OS threads. Defaults to `runtime.NumCPU()`. Not set or configurable anywhere in cc-backend.
- **`NumWorkers`** (`config.go:124`): Controls the number of *goroutines* for checkpoint/archive worker pools. Defaults to `min(NumCPU/2+1, 10)`. Configurable via `"num-workers"` in `config.json`. After our change, this also controls the NATS decode worker count.

So on a 16-core machine you'd get `GOMAXPROCS=16` (up to 16 OS threads running goroutines) and `NumWorkers=9` (9 goroutines in each worker pool). The goroutines are multiplexed onto the OS threads by the Go scheduler — they're not 1:1.

Would you like to make `GOMAXPROCS` configurable, or is the current default sufficient?