   118→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   119→	wg.Go(func() {
   120→		if Keys.Checkpoints.FileFormat == "json" {
   121→			return
   122→		}
   123→
   124→		hostFiles := make(map[string]*walFileState)
   125→
   126→		defer func() {
   127→			for _, ws := range hostFiles {
   128→				if ws.f != nil {
   129→					ws.w.Flush()
   130→					ws.f.Close()
   131→				}
   132→			}
   133→		}()
   134→
   135→		getOrOpenWAL := func(hostDir string) *walFileState {
   136→			ws, ok := hostFiles[hostDir]
   137→			if ok {
   138→				return ws
   139→			}
   140→
   141→			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   142→				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   143→				return nil
   144→			}
   145→
   146→			walPath := path.Join(hostDir, "current.wal")
   147→			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   148→			if err != nil {
   149→				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   150→				return nil
   151→			}
   152→
   153→			w := bufio.NewWriter(f)
   154→
   155→			// Write file header magic if file is new (empty).
   156→			info, err := f.Stat()
   157→			if err == nil && info.Size() == 0 {
   158→				var hdr [4]byte
   159→				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   160→				if _, err := w.Write(hdr[:]); err != nil {
   161→					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   162→					f.Close()
   163→					return nil
   164→				}
   165→			}
   166→
   167→			ws = &walFileState{f: f, w: w}
   168→			hostFiles[hostDir] = ws
   169→			return ws
   170→		}
   171→
   172→		processMsg := func(msg *WALMessage) {
   173→			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   174→			ws := getOrOpenWAL(hostDir)
   175→			if ws == nil {
   176→				return
   177→			}
   178→			if err := writeWALRecord(ws.w, msg); err != nil {
   179→				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   180→			}
   181→		}
   182→
   183→		processRotate := func(req walRotateReq) {
   184→			ws, ok := hostFiles[req.hostDir]
   185→			if ok && ws.f != nil {
   186→				ws.w.Flush()
   187→				ws.f.Close()
   188→				walPath := path.Join(req.hostDir, "current.wal")
   189→				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   190→					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   191→				}
   192→				delete(hostFiles, req.hostDir)
   193→			}
   194→			close(req.done)
   195→		}
   196→
   197→		drain := func() {
   198→			for {
   199→				select {
   200→				case msg, ok := <-WALMessages:
   201→					if !ok {
   202→						return
   203→					}
   204→					processMsg(msg)
   205→				case req := <-walRotateCh:
   206→					processRotate(req)
   207→				default:
   208→					// Flush all buffered writers after draining remaining messages.
   209→					for _, ws := range hostFiles {
   210→						if ws.f != nil {
   211→							ws.w.Flush()
   212→						}
   213→					}
   214→					return
   215→				}
   216→			}
   217→		}


908:// Execute runs a Squirrel UpdateBuilder statement against the database.
915:func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
924:// MarkArchived adds monitoring status update to an existing UpdateBuilder statement.
932:func (r *JobRepository) MarkArchived(
939:// UpdateEnergy calculates and updates the energy consumption for a job.
951:func (r *JobRepository) UpdateEnergy(
1001:// UpdateFootprint calculates and updates the performance footprint for a job.
1010:func (r *JobRepository) UpdateFootprint(


Bash

grep -n "func.*Job\|func.*Node\|func.*User" /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go | head -30


122:func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
252:func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request) {
345:func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) {
545:func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) {
609:func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) {
717:func (api *RestAPI) startJob(rw http.ResponseWriter, r *http.Request) {
805:func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
852:func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) {
897:func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) {
950:func (api *RestAPI) deleteJobBefore(rw http.ResponseWriter, r *http.Request) {
993:func (api *RestAPI) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobAPIRequest, isCached bool) {
1055:func (api *RestAPI) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
1120:func (api *RestAPI) getUsedNodes(rw http.ResponseWriter, r *http.Request) {


/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-124-
internal/archiver/archiveWorker.go-125-			start := time.Now()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-126-			// not using meta data, called to load JobMeta into Cache?
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-127-			// will fail if job meta not in repository
internal/archiver/archiveWorker.go-128-			if _, err := jobRepo.FetchMetadata(job); err != nil {
internal/archiver/archiveWorker.go-129-				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", *job.ID, err.Error())
internal/archiver/archiveWorker.go:130:				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-131-				archivePending.Done()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-132-				continue
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-133-			}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-134-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-135-			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-136-			// Use shutdown context to allow cancellation
internal/archiver/archiveWorker.go-137-			jobMeta, err := ArchiveJob(job, shutdownCtx)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-138-			if err != nil {
internal/archiver/archiveWorker.go-139-				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", *job.ID, err.Error())
internal/archiver/archiveWorker.go:140:				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-141-				archivePending.Done()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-142-				continue
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-143-			}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-144-
internal/archiver/archiveWorker.go-145-			stmt := sq.Update("job").Where("job.id = ?", job.ID)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-146-
internal/archiver/archiveWorker.go:147:			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
internal/archiver/archiveWorker.go-148-				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", *job.ID, err.Error())
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-149-				archivePending.Done()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-150-				continue
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-151-			}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-152-			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
internal/archiver/archiveWorker.go-153-				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", *job.ID, err.Error())
--
internal/taskmanager/updateFootprintService.go-108-								Max: (math.Round(max*100) / 100),
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-109-							}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-110-						}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-111-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-112-						// Build Statement per Job, Add to Pending Array
internal/taskmanager/updateFootprintService.go-113-						stmt := sq.Update("job")
internal/taskmanager/updateFootprintService.go:114:						stmt, err = jobRepo.UpdateFootprint(stmt, job)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-115-						if err != nil {
internal/taskmanager/updateFootprintService.go-116-							cclog.Errorf("update job (dbid: %d) statement build failed at footprint step: %s", *job.ID, err.Error())
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-117-							ce++
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-118-							continue
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-119-						}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/updateFootprintService.go-120-						stmt = stmt.Where("job.id = ?", job.ID)
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-878-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-879-
internal/repository/job.go-880-	cclog.Debugf("JobRepository.FindJobsBetween(): Return job count %d (omitTagged: %v)", len(jobs), omitTagged)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-881-	return jobs, nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-882-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-883-
internal/repository/job.go:884:// UpdateMonitoringStatus updates the monitoring status for a job and invalidates its cache entries.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-885-// Cache invalidation affects both metadata and energy footprint to ensure consistency.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-886-//
internal/repository/job.go-887-/ Parameters:
internal/repository/job.go-888-/   - job: Database ID of the job to update
internal/repository/job.go-889-/   - monitoringStatus: New monitoring status value (see schema.MonitoringStatus constants)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-890-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-891-// Returns an error if the database update fails.
internal/repository/job.go:892:func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-893-	// Invalidate cache entries as monitoring status affects job state
internal/repository/job.go-894-	r.cache.Del(fmt.Sprintf("metadata:%d", job))
internal/repository/job.go-895-	r.cache.Del(fmt.Sprintf("energyFootprint:%d", job))
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-896-
internal/repository/job.go-897-	stmt := sq.Update("job").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-898-		Set("monitoring_status", monitoringStatus).
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-995-		return stmt, err
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-996-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-997-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-998-	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-999-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1000-
internal/repository/job.go:1001:// UpdateFootprint calculates and updates the performance footprint for a job.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1002-// This is called for running jobs during intermediate updates or when archiving.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1003-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1004-// A footprint is a summary statistic (avg/min/max) for each monitored metric.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1005-// The specific statistic type is defined in the cluster config's Footprint field.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1006-// Results are stored as JSON with keys like "metric_avg", "metric_max", etc.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1007-//
internal/repository/job.go-1008-/ Example: For a "cpu_load" metric with Footprint="avg", this stores
internal/repository/job.go-1009-/ the average CPU load across all nodes as "cpu_load_avg": 85.3
internal/repository/job.go:1010:func (r *JobRepository) UpdateFootprint(
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1011-	stmt sq.UpdateBuilder,
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1012-	jobMeta *schema.Job,
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1013-) (sq.UpdateBuilder, error) {
internal/repository/job.go-1014-	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-1015-	if err != nil {
internal/repository/job.go-1016-		cclog.Errorf("cannot get subcluster: %s", err.Error())


   908→// Execute runs a Squirrel UpdateBuilder statement against the database.
   909→// This is a generic helper for executing pre-built update queries.
   910→//
   911→// Parameters:
   912→//   - stmt: Squirrel UpdateBuilder with prepared update query
   913→//
   914→// Returns an error if the execution fails.
   915→func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
   916→	if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil {
   917→		cclog.Errorf("Error while executing statement: %v", err)
   918→		return fmt.Errorf("failed to execute update statement: %w", err)
   919→	}
   920→
   921→	return nil
   922→}
   923→
   924→// MarkArchived adds monitoring status update to an existing UpdateBuilder statement.
   925→// This is a builder helper used when constructing multi-field update queries.
   926→//
   927→// Parameters:
   928→//   - stmt: Existing UpdateBuilder to modify
   929→//   - monitoringStatus: Monitoring status value to set
   930→//
   931→// Returns the modified UpdateBuilder for method chaining.
   932→func (r *JobRepository) MarkArchived(
   933→	stmt sq.UpdateBuilder,
   934→	monitoringStatus int32,
   935→) sq.UpdateBuilder {
   936→	return stmt.Set("monitoring_status", monitoringStatus)
   937→}
   938→
   939→// UpdateEnergy calculates and updates the energy consumption for a job.
   940→// This is called for running jobs during intermediate updates or when archiving.
   941→//
   942→// Energy calculation formula:
   943→//   - For "power" metrics: Energy (kWh) = (Power_avg * NumNodes * Duration_hours) / 1000
   944→//   - For "energy" metrics: Currently not implemented (would need sum statistics)
   945→//
   946→// The calculation accounts for:
   947→//   - Multi-node jobs: Multiplies by NumNodes to get total cluster energy
   948→//   - Shared jobs: Node average is already based on partial resources, so NumNodes=1
   949→//   - Unit conversion: Watts * hours / 1000 = kilowatt-hours (kWh)
   950→//   - Rounding: Results rounded to 2 decimal places
   951→func (r *JobRepository) UpdateEnergy(
   952→	stmt sq.UpdateBuilder,
   953→	jobMeta *schema.Job,
   954→) (sq.UpdateBuilder, error) {
   955→	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
   956→	if err != nil {
   957→		cclog.Errorf("cannot get subcluster: %s", err.Error())
   958→		return stmt, err
   959→	}
   960→	energyFootprint := make(map[string]float64)
   961→
   962→	// Accumulate total energy across all energy-related metrics
   963→	totalEnergy := 0.0
   964→	for _, fp := range sc.EnergyFootprint {
   965→		// Calculate energy for this specific metric
   966→		metricEnergy := 0.0
   967→		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
   968→			switch sc.MetricConfig[i].Energy {
   969→			case "energy": // Metric already in energy units (Joules or Wh)
   970→				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
   971→				// FIXME: Needs sum as stats type to accumulate energy values over time
   972→			case "power": // Metric in power units (Watts)
   973→				// Energy (kWh) = Power (W) × Time (h) / 1000
   974→				// Formula: (avg_power_per_node * num_nodes) * (duration_sec / 3600) / 1000
   975→				//
   976→				// Breakdown:
   977→				//   LoadJobStat(jobMeta, fp, "avg") = average power per node (W)
   978→				//   jobMeta.NumNodes = number of nodes (1 for shared jobs)
   979→				//   jobMeta.Duration / 3600.0 = duration in hours
   980→				//   / 1000.0 = convert Wh to kWh
   981→				rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
   982→				metricEnergy = math.Round(rawEnergy*100.0) / 100.0 // Round to 2 decimal places
   983→			}
   984→		} else {
   985→			cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
   986→		}
   987→
   988→		energyFootprint[fp] = metricEnergy
   989→		totalEnergy += metricEnergy
   990→	}
   991→
   992→	var rawFootprint []byte
   993→	if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
   994→		cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
   995→		return stmt, err
   996→	}
   997→
   998→	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
   999→}
  1000→
  1001→// UpdateFootprint calculates and updates the performance footprint for a job.
  1002→// This is called for running jobs during intermediate updates or when archiving.
  1003→//
  1004→// A footprint is a summary statistic (avg/min/max) for each monitored metric.
  1005→// The specific statistic type is defined in the cluster config's Footprint field.
  1006→// Results are stored as JSON with keys like "metric_avg", "metric_max", etc.
  1007→//


   717→func (api *RestAPI) startJob(rw http.ResponseWriter, r *http.Request) {
   718→	req := schema.Job{
   719→		Shared:           "none",
   720→		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
   721→	}
   722→	if err := decode(r.Body, &req); err != nil {
   723→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   724→		return
   725→	}
   726→
   727→	cclog.Debugf("REST: %s", req.GoString())
   728→	req.State = schema.JobStateRunning
   729→
   730→	if err := importer.SanityChecks(&req); err != nil {
   731→		handleError(err, http.StatusBadRequest, rw)
   732→		return
   733→	}
   734→
   735→	// aquire lock to avoid race condition between API calls
   736→	var unlockOnce sync.Once
   737→	api.RepositoryMutex.Lock()
   738→	defer unlockOnce.Do(api.RepositoryMutex.Unlock)
   739→
   740→	// Check if combination of (job_id, cluster_id, start_time) already exists:
   741→	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
   742→	if err != nil && err != sql.ErrNoRows {
   743→		handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw)
   744→		return
   745→	}
   746→	if err == nil {
   747→		for _, job := range jobs {
   748→			// Check if jobs are within the same day (prevent duplicates)
   749→			if (req.StartTime - job.StartTime) < secondsPerDay {
   750→				handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d, jobid: %d", *job.ID, job.JobID), http.StatusUnprocessableEntity, rw)
   751→				return
   752→			}
   753→		}
   754→	}
   755→
   756→	// When tags are present, insert directly into the job table so that the
   757→	// returned ID can be used with AddTagOrCreate (which queries the job table).
   758→	// Jobs without tags use the cache path as before.
   759→	var id int64
   760→	if len(req.Tags) > 0 {
   761→		id, err = api.JobRepository.StartDirect(&req)
   762→	} else {
   763→		id, err = api.JobRepository.Start(&req)
   764→	}
   765→	if err != nil {
   766→		handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw)
   767→		return
   768→	}
   769→	// unlock here, adding Tags can be async
   770→	unlockOnce.Do(api.RepositoryMutex.Unlock)
   771→
   772→	for _, tag := range req.Tags {
   773→		if _, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), id, tag.Type, tag.Name, tag.Scope); err != nil {
   774→			handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw)
   775→			return
   776→		}
   777→	}
   778→
   779→	cclog.Infof("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime)
   780→	rw.Header().Add("Content-Type", "application/json")
   781→	rw.WriteHeader(http.StatusCreated)
   782→	if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{
   783→		Message: "success",
   784→	}); err != nil {
   785→		cclog.Errorf("Failed to encode response: %v", err)
   786→	}
   787→}
   788→
   789→// stopJobByRequest godoc
   790→// @summary     Marks job as completed and triggers archiving
   791→// @tags Job add and modify
   792→// @description Job to stop is specified by request body. All fields are required in this case.
   793→// @description Returns full job resource information according to 'Job' scheme.
   794→// @produce     json
   795→// @param       request body     api.StopJobAPIRequest true "All fields required"
   796→// @success     200     {object} schema.Job                 "Success message"
   797→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   798→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   799→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   800→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   801→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: job has already been stopped"
   802→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   803→// @security    ApiKeyAuth
   804→// @router      /api/jobs/stop_job/ [post]
   805→func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
   806→	// Parse request body
   807→	req := StopJobAPIRequest{}
   808→	if err := decode(r.Body, &req); err != nil {
   809→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   810→		return
   811→	}
   812→
   813→	// Fetch job (that will be stopped) from db
   814→	var job *schema.Job
   815→	var err error
   816→	if req.JobID == nil {


   805→func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
   806→	// Parse request body
   807→	req := StopJobAPIRequest{}
   808→	if err := decode(r.Body, &req); err != nil {
   809→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   810→		return
   811→	}
   812→
   813→	// Fetch job (that will be stopped) from db
   814→	var job *schema.Job
   815→	var err error
   816→	if req.JobID == nil {
   817→		handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw)
   818→		return
   819→	}
   820→
   821→	isCached := false
   822→	job, err = api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
   823→	if err != nil {
   824→		// Not in cache, try main job table
   825→		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
   826→		if err != nil {
   827→			handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
   828→			return
   829→		}
   830→	} else {
   831→		isCached = true
   832→	}
   833→
   834→	api.checkAndHandleStopJob(rw, job, req, isCached)
   835→}
   836→
   837→// deleteJobByID godoc
   838→// @summary     Remove a job from the sql database
   839→// @tags Job remove
   840→// @description Job to remove is specified by database ID. This will not remove the job from the job archive.
   841→// @produce     json
   842→// @param       id      path     int                   true "Database ID of Job"
   843→// @success     200     {object} api.DefaultAPIResponse  "Success message"
   844→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   845→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   846→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   847→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   848→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
   849→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   850→// @security    ApiKeyAuth
   851→// @router      /api/jobs/delete_job/{id} [delete]
   852→func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) {
   853→	// Fetch job (that will be stopped) from db
   854→	id := chi.URLParam(r, "id")


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"fmt"
    10→	"maps"
    11→	"net/http"
    12→	"strings"
    13→	"time"
    14→
    15→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    16→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    17→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    18→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    21→)
    22→
    23→type UpdateNodeStatesRequest struct {
    24→	Nodes   []schema.NodePayload `json:"nodes"`
    25→	Cluster string               `json:"cluster" example:"fritz"`
    26→}
    27→
    28→// metricListToNames converts a map of metric configurations to a list of metric names
    29→func metricListToNames(metricList map[string]*schema.Metric) []string {
    30→	names := make([]string, 0, len(metricList))
    31→	for name := range metricList {
    32→		names = append(names, name)
    33→	}
    34→	return names
    35→}
    36→
    37→// this routine assumes that only one of them exists per node
    38→func determineState(states []string) schema.SchedulerState {
    39→	for _, state := range states {
    40→		switch strings.ToLower(state) {
    41→		case "allocated":
    42→			return schema.NodeStateAllocated
    43→		case "reserved":
    44→			return schema.NodeStateReserved
    45→		case "idle":
    46→			return schema.NodeStateIdle
    47→		case "down":
    48→			return schema.NodeStateDown
    49→		case "mixed":
    50→			return schema.NodeStateMixed
    51→		}
    52→	}
    53→
    54→	return schema.NodeStateUnknown
    55→}
    56→
    57→// updateNodeStates godoc
    58→// @summary     Deliver updated Slurm node states
    59→// @tags Nodestates
    60→// @description Returns a JSON-encoded list of users.
    61→// @description Required query-parameter defines if all users or only users with additional special roles are returned.
    62→// @produce     json
    63→// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
    64→// @success     200     {object} api.DefaultAPIResponse "Success message"
    65→// @failure     400     {object} api.ErrorResponse      "Bad Request"
    66→// @failure     401     {object} api.ErrorResponse      "Unauthorized"
    67→// @failure     403     {object} api.ErrorResponse      "Forbidden"
    68→// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
    69→// @security    ApiKeyAuth
    70→// @router      /api/nodestats/ [post]
    71→func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
    72→	// Parse request body
    73→	req := UpdateNodeStatesRequest{}
    74→	if err := decode(r.Body, &req); err != nil {
    75→		handleError(fmt.Errorf("parsing request body failed: %w", err),
    76→			http.StatusBadRequest, rw)
    77→		return
    78→	}
    79→	requestReceived := time.Now().Unix()
    80→	repo := repository.GetNodeRepository()
    81→
    82→	m := make(map[string][]string)
    83→	metricNames := make(map[string][]string)
    84→	healthResults := make(map[string]metricstore.HealthCheckResult)
    85→
    86→	startMs := time.Now()
    87→
    88→	// Step 1: Build nodeList and metricList per subcluster
    89→	for _, node := range req.Nodes {
    90→		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
    91→			m[sc] = append(m[sc], node.Hostname)
    92→		}
    93→	}
    94→
    95→	for sc := range m {
    96→		if sc != "" {
    97→			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
    98→			metricNames[sc] = metricListToNames(metricList)
    99→		}
   100→	}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"database/sql"
    10→	"encoding/json"
    11→	"strings"
    12→	"sync"
    13→	"time"
    14→
    15→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    16→	"github.com/ClusterCockpit/cc-backend/internal/config"
    17→	"github.com/ClusterCockpit/cc-backend/internal/importer"
    18→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
    21→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    22→	"github.com/ClusterCockpit/cc-lib/v2/receivers"
    23→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    24→	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    25→)
    26→
    27→// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
    28→// It mirrors the functionality of the REST API but uses NATS messaging with
    29→// InfluxDB line protocol as the message format.
    30→//
    31→// # Message Format
    32→//
    33→// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
    34→// with the following structure:
    35→//
    36→//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
    37→//
    38→// # Job Events
    39→//
    40→// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
    41→//
    42→//	job,function=start_job event="{...JSON payload...}" <timestamp>
    43→//	job,function=stop_job event="{...JSON payload...}" <timestamp>
    44→//
    45→// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
    46→//
    47→// Example job start message:
    48→//
    49→//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
    50→//
    51→// # Node State Events
    52→//
    53→// Node state updates use the "nodestate" measurement with cluster information:
    54→//
    55→//	nodestate event="{...JSON payload...}" <timestamp>
    56→//
    57→// The JSON payload follows the UpdateNodeStatesRequest structure.
    58→//
    59→// Example node state message:
    60→//
    61→//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
    62→type NatsAPI struct {
    63→	JobRepository *repository.JobRepository
    64→	// RepositoryMutex protects job creation operations from race conditions
    65→	// when checking for duplicate jobs during startJob calls.
    66→	RepositoryMutex sync.Mutex
    67→	// jobSem limits concurrent NATS job event processing to prevent goroutine accumulation
    68→	// when the database is under contention.
    69→	jobSem chan struct{}
    70→	// nodeSem limits concurrent NATS node state processing.
    71→	nodeSem chan struct{}
    72→}
    73→
    74→// NewNatsAPI creates a new NatsAPI instance with default dependencies.
    75→func NewNatsAPI() *NatsAPI {
    76→	return &NatsAPI{
    77→		JobRepository: repository.GetJobRepository(),
    78→		jobSem:        make(chan struct{}, 8),
    79→		nodeSem:       make(chan struct{}, 2),
    80→	}
    81→}
    82→
    83→// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
    84→// Returns an error if the NATS client is not available or subscription fails.
    85→func (api *NatsAPI) StartSubscriptions() error {
    86→	client := nats.GetClient()
    87→	if client == nil {
    88→		cclog.Warn("NATS client not available, skipping API subscriptions")
    89→		return nil
    90→	}
    91→
    92→	if config.Keys.APISubjects != nil {
    93→
    94→		s := config.Keys.APISubjects
    95→
    96→		if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil {
    97→			return err
    98→		}
    99→
   100→		if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {
   101→			return err
   102→		}
   103→
   104→		cclog.Info("NATS API subscriptions started")
   105→	}
   106→	return nil
   107→}
   108→
   109→// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
   110→// Validates that required tags and fields are present before processing.
   111→func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
   112→	function, ok := msg.GetTag("function")
   113→	if !ok {
   114→		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
   115→		return
   116→	}
   117→
   118→	switch function {
   119→	case "start_job":
   120→		v, ok := msg.GetEventValue()
   121→		if !ok {
   122→			cclog.Errorf("Job start event is missing event field with JSON payload")
   123→			return
   124→		}
   125→		api.handleStartJob(v)
   126→
   127→	case "stop_job":
   128→		v, ok := msg.GetEventValue()
   129→		if !ok {
   130→			cclog.Errorf("Job stop event is missing event field with JSON payload")
   131→			return
   132→		}
   133→		api.handleStopJob(v)
   134→
   135→	default:
   136→		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
   137→	}
   138→}
   139→
   140→// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
   141→// The message must be in line protocol format with measurement="job" and include:
   142→//   - tag "function" with value "start_job" or "stop_job"
   143→//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
   144→//
   145→// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
   146→func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
   147→	api.jobSem <- struct{}{}
   148→	defer func() { <-api.jobSem }()
   149→
   150→	if len(data) == 0 {


418 /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go


grep -n "AddTag\|RemoveTag\|ImportTag" /Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/*.go 2>/dev/null | head -20


/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:165:// AddTagsToJob is the resolver for the addTagsToJob field.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:166:func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:199:			if tags, err = r.Repo.AddTag(user, jid, tid); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:212:// RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:213:func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:246:			if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:260:// RemoveTagFromList is the resolver for the removeTagFromList field.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:261:func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:287:			if err = r.Repo.RemoveTagByID(tid); err != nil {


find /Users/jan/prg/ClusterCockpit/cc-backend -name "*resolver*" -type f | head -5


   165→// AddTagsToJob is the resolver for the addTagsToJob field.
   166→func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
   167→	user := repository.GetUserFromContext(ctx)
   168→	if user == nil {
   169→		return nil, fmt.Errorf("no user in context")
   170→	}
   171→
   172→	jid, err := strconv.ParseInt(job, 10, 64)
   173→	if err != nil {
   174→		cclog.Warn("Error while adding tag to job")
   175→		return nil, err
   176→	}
   177→
   178→	tags := []*schema.Tag{}
   179→	for _, tagID := range tagIds {
   180→		// Get ID
   181→		tid, err := strconv.ParseInt(tagID, 10, 64)
   182→		if err != nil {
   183→			cclog.Warn("Error while parsing tag id")
   184→			return nil, err
   185→		}
   186→
   187→		// Test Exists
   188→		_, _, tscope, exists := r.Repo.TagInfo(tid)
   189→		if !exists {
   190→			cclog.Warnf("Tag does not exist (ID): %d", tid)
   191→			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
   192→		}
   193→
   194→		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
   195→		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
   196→			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
   197→			user.Username == tscope {
   198→			// Add to Job
   199→			if tags, err = r.Repo.AddTag(user, jid, tid); err != nil {
   200→				cclog.Warn("Error while adding tag")
   201→				return nil, err
   202→			}
   203→		} else {
   204→			cclog.Warnf("Not authorized to add tag: %d", tid)
   205→			return nil, fmt.Errorf("not authorized to add tag: %d", tid)
   206→		}
   207→	}
   208→
   209→	return tags, nil
   210→}
   211→
   212→// RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
   213→func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
   214→	user := repository.GetUserFromContext(ctx)
   215→	if user == nil {
   216→		return nil, fmt.Errorf("no user in context")
   217→	}
   218→
   219→	jid, err := strconv.ParseInt(job, 10, 64)
   220→	if err != nil {
   221→		cclog.Warn("Error while parsing job id")
   222→		return nil, err
   223→	}
   224→
   225→	tags := []*schema.Tag{}
   226→	for _, tagID := range tagIds {
   227→		// Get ID
   228→		tid, err := strconv.ParseInt(tagID, 10, 64)
   229→		if err != nil {
   230→			cclog.Warn("Error while parsing tag id")
   231→			return nil, err
   232→		}
   233→
   234→		// Test Exists
   235→		_, _, tscope, exists := r.Repo.TagInfo(tid)
   236→		if !exists {
   237→			cclog.Warnf("Tag does not exist (ID): %d", tid)
   238→			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
   239→		}
   240→
   241→		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
   242→		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
   243→			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
   244→			user.Username == tscope {
   245→			// Remove from Job
   246→			if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil {
   247→				cclog.Warn("Error while removing tag")
   248→				return nil, err
   249→			}
   250→		} else {
   251→			cclog.Warnf("Not authorized to remove tag: %d", tid)
   252→			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
   253→		}
   254→
   255→	}
   256→
   257→	return tags, nil
   258→}
   259→
   260→// RemoveTagFromList is the resolver for the removeTagFromList field.
   261→func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) {
   262→	// Needs Contextuser
   263→	user := repository.GetUserFromContext(ctx)
   264→	if user == nil {
   265→		return nil, fmt.Errorf("no user in context")
   266→	}
   267→
   268→	tags := []int{}
   269→	for _, tagID := range tagIds {
   270→		// Get ID
   271→		tid, err := strconv.ParseInt(tagID, 10, 64)
   272→		if err != nil {
   273→			cclog.Warn("Error while parsing tag id for removal")
   274→			return nil, err
   275→		}
   276→
   277→		// Test Exists
   278→		_, _, tscope, exists := r.Repo.TagInfo(tid)
   279→		if !exists {
   280→			cclog.Warnf("Tag does not exist (ID): %d", tid)
   281→			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
   282→		}
   283→
   284→		// Test Access: Admins && Admin Tag OR Everyone && Private Tag
   285→		if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope {
   286→			// Remove from DB
   287→			if err = r.Repo.RemoveTagByID(tid); err != nil {
   288→				cclog.Warn("Error while removing tag")
   289→				return nil, err
   290→			} else {
   291→				tags = append(tags, int(tid))
   292→			}
   293→		} else {
   294→			cclog.Warnf("Not authorized to remove tag: %d", tid)
   295→			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
   296→		}
   297→	}
   298→	return tags, nil
   299→}
   300→
   301→// UpdateConfiguration is the resolver for the updateConfiguration field.
   302→func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) {
   303→	if err := repository.GetUserCfgRepo().UpdateConfig(name, value, repository.GetUserFromContext(ctx)); err != nil {
   304→		cclog.Warn("Error while updating user config")
   305→		return nil, err
   306→	}
   307→
   308→	return nil, nil
   309→}
   310→
   311→// ID is the resolver for the id field.
   312→func (r *nodeResolver) ID(ctx context.Context, obj *schema.Node) (string, error) {
   313→	panic(fmt.Errorf("not implemented: ID - id"))
   314→}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"sync"
    10→
    11→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    12→)
    13→
    14→// JobHook interface allows external components to hook into job lifecycle events.
    15→// Implementations can perform actions when jobs start or stop, such as tagging,
    16→// logging, notifications, or triggering external workflows.
    17→//
    18→// Example implementation:
    19→//
    20→//	type MyJobTagger struct{}
    21→//
    22→//	func (t *MyJobTagger) JobStartCallback(job *schema.Job) {
    23→//	    if job.NumNodes > 100 {
    24→//	        // Tag large jobs automatically
    25→//	    }
    26→//	}
    27→//
    28→//	func (t *MyJobTagger) JobStopCallback(job *schema.Job) {
    29→//	    if job.State == schema.JobStateFailed {
    30→//	        // Log or alert on failed jobs
    31→//	    }
    32→//	}
    33→//
    34→// Register hooks during application initialization:
    35→//
    36→//	repository.RegisterJobHook(&MyJobTagger{})
    37→type JobHook interface {
    38→	// JobStartCallback is invoked when one or more jobs start.
    39→	// This is called synchronously, so implementations should be fast.
    40→	JobStartCallback(job *schema.Job)
    41→
    42→	// JobStopCallback is invoked when a job completes.
    43→	// This is called synchronously, so implementations should be fast.
    44→	JobStopCallback(job *schema.Job)
    45→}
    46→
    47→var (
    48→	initOnce sync.Once
    49→	hooks    []JobHook
    50→)
    51→
    52→// RegisterJobHook registers a JobHook to receive job lifecycle callbacks.
    53→// Multiple hooks can be registered and will be called in registration order.
    54→// This function is safe to call multiple times and is typically called during
    55→// application initialization.
    56→//
    57→// Nil hooks are silently ignored to simplify conditional registration.
    58→func RegisterJobHook(hook JobHook) {
    59→	initOnce.Do(func() {
    60→		hooks = make([]JobHook, 0)
    61→	})
    62→
    63→	if hook != nil {
    64→		hooks = append(hooks, hook)
    65→	}
    66→}
    67→
    68→// CallJobStartHooks invokes all registered JobHook.JobStartCallback methods
    69→// for each job in the provided slice. This is called internally by the repository
    70→// when jobs are started (e.g., via StartJob or batch job imports).
    71→//
    72→// Hooks are called synchronously in registration order. If a hook panics,
    73→// the panic will propagate to the caller.
    74→func CallJobStartHooks(jobs []*schema.Job) {
    75→	if hooks == nil {
    76→		return
    77→	}
    78→
    79→	for _, hook := range hooks {
    80→		if hook != nil {
    81→			for _, job := range jobs {
    82→				hook.JobStartCallback(job)
    83→			}
    84→		}
    85→	}
    86→}
    87→
    88→// CallJobStopHooks invokes all registered JobHook.JobStopCallback methods
    89→// for the provided job. This is called internally by the repository when a
    90→// job completes (e.g., via StopJob or job state updates).
    91→//
    92→// Hooks are called synchronously in registration order. If a hook panics,
    93→// the panic will propagate to the caller.
    94→func CallJobStopHooks(job *schema.Job) {
    95→	if hooks == nil {
    96→		return
    97→	}
    98→
    99→	for _, hook := range hooks {
   100→		if hook != nil {
   101→			hook.JobStopCallback(job)
   102→		}
   103→	}
   104→}
   105→


grep -n "CallJobStopHooks\|TriggerArchiving\|jobSem\|nodeSem" /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go | head -20


67:	// jobSem limits concurrent NATS job event processing to prevent goroutine accumulation
69:	jobSem chan struct{}
70:	// nodeSem limits concurrent NATS node state processing.
71:	nodeSem chan struct{}
78:		jobSem:        make(chan struct{}, 8),
79:		nodeSem:       make(chan struct{}, 2),
147:	api.jobSem <- struct{}{}
148:	defer func() { <-api.jobSem }()
334:	archiver.TriggerArchiving(job)
390:	api.nodeSem <- struct{}{}
391:	defer func() { <-api.nodeSem }()


Perfect. Now I have enough information to provide a comprehensive analysis. Let me compile my findings:

## DB Write Patterns Analysis - ClusterCockpit cc-backend

Based on my thorough exploration of the codebase, I've identified several DB write patterns and opportunities for transaction-based optimization. Here's my detailed analysis:

---

## 1. TAG OPERATIONS - HIGH TRANSACTION OPPORTUNITY

### Pattern: AddTagOrCreate (Internal/repository/tags.go:424)
When adding tags to a job with multiple tags, each tag operation is **separate and non-transactional**:

```
startJob() → for each tag {
  → TagID lookup (SELECT)
  → CreateTag (INSERT) if missing
  → AddTag (INSERT into jobtag)
  → GetTags (SELECT)
  → archive.UpdateTags() (file write)
}
```

**Current Issues:**
- **Line 760-777** in `/internal/api/job.go`: When startJob is called with `req.Tags`, each tag is added individually via `AddTagOrCreate()` inside a loop
- **Line 570-583** in `/internal/api/job.go`: tagJob endpoint adds multiple tags one-by-one with individual archive updates
- **Lines 165-210** in `/internal/graph/schema.resolvers.go`: GraphQL AddTagsToJob loops through tag IDs, calling `AddTag()` for each without batching

**Batch Potential:** ⭐⭐⭐⭐⭐ **HIGHEST**
- A job with 5 tags = 5 separate transactions (INSERT + SELECT + archive write)
- Could be batched into single transaction: 1 INSERT many jobtag + 1 archive write

### Key File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/tags.go`

**addJobTag() helper (line 398):** Performs 3 operations per tag:
1. INSERT into jobtag
2. SELECT tags
3. archive.UpdateTags() (file I/O)

**AddTagOrCreate() (line 424):** Chains multiple operations:
1. TagID lookup
2. CreateTag if missing (conditional INSERT)
3. AddTag (calls addJobTag, which does 3 more ops)

---

## 2. JOB ARCHIVING WRITES - MODERATE TRANSACTION OPPORTUNITY

### Pattern: Archiving Workflow (archiveWorker in /internal/archiver/archiveWorker.go)

When a job is archived, **3-4 separate UPDATE statements** execute sequentially:

```
archivingWorker() → for job in archiveChannel {
  → FetchMetadata (SELECT)
  → ArchiveJob() (file writes + calculations)
  → UpdateFootprint (UPDATE)
  → UpdateEnergy (UPDATE)
  → MarkArchived (UPDATE)
  → Execute(stmt) (single UPDATE with 3 SET clauses)
  → CallJobStopHooks()
}
```

**Current Issues:**
- **Lines 145-159** in `/internal/archiver/archiveWorker.go`: Three separate `.Set()` calls chained together but executed as ONE statement via `jobRepo.Execute(stmt)`
- Actually **GOOD**: The code chains multiple SET operations into a single UPDATE via Squirrel builder pattern
- However, the statement is still a single UPDATE - could be a transaction for safety

**Transaction Benefit:** ⭐⭐⭐ **MODERATE**
- Currently chains SETs into one UPDATE (good)
- Could add transaction wrapper for atomicity guarantee
- Already uses statement builder pattern to combine updates

### Key Files:
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go:145-163`
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go:908-999` (UpdateEnergy, UpdateFootprint, MarkArchived, Execute)

---

## 3. JOB IMPORT/RESTORE - HIGH TRANSACTION OPPORTUNITY

### Pattern: HandleImportFlag (internal/importer/handleImport.go:38)

Imports job metadata + tags without batching:

```
HandleImportFlag(flag) → for each job in flag {
  → Read JSON files
  → archive.ImportJob() (file write)
  → InsertJobDirect (INSERT)
  → for each tag in job.Tags {
    → ImportTag() (SELECT + INSERT per tag)
  }
}
```

**Current Issues:**
- **Lines 105-116** in `/internal/importer/handleImport.go`:
  - 1 job INSERT per job
  - N tag operations (SELECT + INSERT) per job
  - No transaction wrapping job + all its tags

**Batch Potential:** ⭐⭐⭐⭐⭐ **HIGHEST**
- If importing 100 jobs with 3 tags each = 100 INSERTs + 300 tag lookups/inserts
- Could batch into transaction: INSERT 100 jobs + 300 tag associations in single txn

### Key File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/importer/handleImport.go:111-116`

---

## 4. JOB CACHE TO MAIN TABLE TRANSFER - IDENTIFIED ISSUE

### Pattern: TransferCachedJobToMain (internal/repository/jobCreate.go:143)

**EXACT ISSUE YOU MENTIONED:**

```go
// TransferCachedJobToMain - Lines 143-162
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    res, err := r.DB.Exec(
        "INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?", cacheID)
    if err != nil { return 0, ... }

    newID, err := res.LastInsertId()
    if err != nil { return 0, ... }

    _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)  // SEPARATE OPERATION
    if err != nil { return 0, ... }

    return newID, nil
}
```

**Issue:** ⚠ **TWO SEPARATE EXEC CALLS**
- Line 144-146: INSERT
- Line 156: DELETE
- No transaction wrapping both operations
- **Risk:** If DELETE fails, job orphaned in main table and cache

**Transaction Fix:** Wrap both operations in `tx.Beginx()` / `tx.Commit()`

### Key File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobCreate.go:143-162`

---

## 5. JOB STOP WORKFLOW - MILD CONSOLIDATION OPPORTUNITY

### Pattern: stopJobByRequest (internal/api/job.go:805)

Calls `checkAndHandleStopJob()` which likely:
1. Updates job state (Stop)
2. Transfers from cache to main table (INSERT + DELETE as shown above)
3. Triggers archiving

**Issue:** Multiple separate repository calls that could be transactional

### Key Files:
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go:805-835`
- The actual logic is in `checkAndHandleStopJob()` - would need to read that

---

## 6. TAG REMOVAL - DISTRIBUTED WRITES

### Pattern: RemoveTagByID (internal/repository/tags.go:216)

```go
// RemoveTagByID - Lines 216-244
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)  // SELECT all job IDs

    qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID)
    if _, err := qJobTag.RunWith(r.stmtCache).Exec(); err != nil { ... }

    qTag := sq.Delete("tag").Where("tag.id = ?", tagID)
    if _, err := qTag.RunWith(r.stmtCache).Exec(); err != nil { ... }

    go r.removeTagFromArchiveJobs(jobIds)  // ASYNC goroutine

    return nil
}
```

**Issues:**
- **2 separate DELETE statements** (jobtag, then tag)
- Could be 1 transaction
- Archive updates run **asynchronously in goroutine** (line 241)
- **Lines 183-199** show async archive updates without error handling

**Transaction Opportunity:** ⭐⭐⭐ **MODERATE**
- Can wrap 2 DELETEs in single transaction

### Key Files:
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/tags.go:216-244`
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/tags.go:183-199`

---

## 7. GRAPHQL TAG MUTATIONS - LOOP-BASED WRITES

### Pattern: AddTagsToJob, RemoveTagsFromJob (internal/graph/schema.resolvers.go:166, 213)

**Lines 179-207** and **Lines 226-255**:
```go
for _, tagID := range tagIds {
    // Lookup tag
    _, _, tscope, exists := r.Repo.TagInfo(tid)
    // Authorization check
    // Add/Remove
    if tags, err = r.Repo.AddTag(...); err != nil { ... }
}
```

**Issue:**
- **Loop per tag** with individual `AddTag()` or `RemoveTag()` calls
- 5 tags = 5 separate database transactions with archive updates
- Could batch into single transaction

**Batch Potential:** ⭐⭐⭐⭐ **HIGH**

### Key File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/graph/schema.resolvers.go:165-210, 212-258`

---

## 8. NATS API - SEMAPHORE-PROTECTED WRITES

### Pattern: NATS Job/Node event handlers (internal/api/nats.go)

Uses semaphores to limit concurrency:
```go
type NatsAPI struct {
    jobSem chan struct{} // capacity 8
    nodeSem chan struct{} // capacity 2
}

func (api *NatsAPI) handleJobEvent(...) {
    api.jobSem <- struct{}{}      // acquire semaphore
    defer func() { <-api.jobSem }() // release
    // ... process job (same write patterns as REST)
}
```

**No Issues:** Semaphores are for concurrency control, not transactional. Underlying writes are same as REST API (non-transactional tag adds).

### Key File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:62-80, 147-148`

---

## 9. STATISTICS/ENERGY FOOTPRINT UPDATES - AGGREGATION-ONLY

### Pattern: Stats queries (internal/repository/stats.go)

All **READ-ONLY** SELECT with aggregations. No write batching opportunities.

Key observation: Post-archiving stats updates are computed by `UpdateFootprint()` and `UpdateEnergy()` which are already in a single UPDATE statement (item #2).

---

## TRANSACTION SUPPORT STATUS

**Good news:** The codebase **already has transaction support**:

### Available Transaction API (internal/repository/transaction.go):
```go
type Transaction struct { tx *sqlx.Tx }

func (r *JobRepository) TransactionInit() (*Transaction, error)
func (t *Transaction) Commit() error
func (t *Transaction) Rollback() error
func (r *JobRepository) TransactionAddNamed(t *Transaction, query string, args ...any) (int64, error)
func (r *JobRepository) TransactionAdd(t *Transaction, query string, args ...any) (int64, error)
```

**Currently used in:** `/internal/repository/jobCreate.go:96-120` (SyncJobs function uses `tx.Beginx()`)

**Not used in:** Tag operations, individual archiving, import loops

---

## SUMMARY TABLE

| Pattern | Location | Issue | Batch Potential | Transaction Support | Priority |
|---------|----------|-------|-----------------|---------------------|----------|
| AddTagOrCreate in loop | api/job.go:760-777, api/job.go:570-583 | 1 txn per tag | ⭐⭐⭐⭐⭐ | Exists | HIGH |
| ImportTag per tag | importer/handleImport.go:111-116 | 1 txn per tag | ⭐⭐⭐⭐⭐ | Exists | HIGH |
| TransferCachedJobToMain | repository/jobCreate.go:143-162 | INSERT + DELETE separate | ⭐⭐⭐⭐ | Exists | **CRITICAL** |
| RemoveTagByID | repository/tags.go:216-244 | 2 separate DELETEs | ⭐⭐⭐ | Exists | MEDIUM |
| GraphQL AddTagsToJob | graph/schema.resolvers.go:179-207 | 1 txn per tag | ⭐⭐⭐⭐ | Exists | HIGH |
| Archiving updates | archiver/archiveWorker.go:145-163 | 3 SETs chained (OK) | ⭐⭐ | Already good | LOW |
| Job stats queries | repository/stats.go | Read-only aggregations | N/A | N/A | N/A |

---

## RECOMMENDATIONS

1. **CRITICAL**: Fix `TransferCachedJobToMain()` - wrap INSERT + DELETE in transaction
2. **HIGH**: Batch tag operations in `tagJob()` and `AddTagsToJob()` - add `BatchAddTags()` method
3. **HIGH**: Batch import tag operations - wrap job insert + all tags in single transaction
4. **MEDIUM**: Consolidate `RemoveTagByID()` - use single transaction for both DELETEs
5. **MEDIUM**: Add async tag removal with error handling - current goroutine silently fails

---

These patterns show that while transaction support exists, it's **underutilized** in high-frequency write paths, particularly around tag management where multiple individual operations could be safely batched.

agentId: ae6e94e07c8dc030e (for resuming to continue this agent's work if needed)
<usage>total_tokens: 81850
tool_uses: 29
duration_ms: 64839</usage>

Prompt:
Explore the cc-backend codebase for DB write patterns specifically around:

1. **Tag operations** - AddTagOrCreate, adding tags to jobs. These are often called in loops (e.g., when starting a job with multiple tags). Check internal/repository/tags.go or similar files.

2. **Job archiving writes** - What happens when a job is archived? Does it update the job table? Check internal/archiver/ for the archiving flow and what DB writes it does.

3. **Job import/restore** - Check internal/importer/ for bulk import operations.

4. **Energy footprint / statistics updates** - Any post-processing that updates job rows after archiving.

5. **TransferCachedJobToMain** in internal/repository/jobCreate.go - this does INSERT + DELETE as separate operations. Could these be in a transaction?

For each pattern, identify if multiple individual writes could be combined into transactions or batch operations.

Also check internal/api/rest.go for the REST endpoint handlers (startJob, stopJob, etc.) to see their DB write patterns.


Excellent! Now I have comprehensive information. Let me compile my final report based on all the data I've gathered.

---

## COMPREHENSIVE DATABASE WRITE OPERATIONS ANALYSIS - cc-backend

Based on my thorough exploration of the codebase, here is a complete inventory of all database WRITE operations organized by contention patterns and optimization opportunities:

### SECTION 1: JOB LIFECYCLE WRITES (CRITICAL CONTENTION POINTS)

#### 1.1 Job Start/Insert Operations
**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobCreate.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `InsertJob()` (line 54-70) | NamedExec to job_cache | Per-request (REST/NATS) | Mutex-protected only | CONTENTION: Single insert per request, uses mutex not transactions |
| `InsertJobDirect()` (line 36-52) | NamedExec to job table | Per-request (with tags) | Mutex-protected only | CONTENTION: Mutex serializes all tag insertions sequentially |
| `Start()` (line 166-183) | Marshals then InsertJob | Per-request | Indirect (via InsertJob) | THREE writes: footprint + resources + metadata marshaling before DB write |
| `StartDirect()` (line 188-205) | Marshals then InsertJobDirect | Per-request (with tags) | Indirect (via InsertJobDirect) | SAME: Three marshals before single DB insert |
| `SyncJobs()` (line 72-139) | INSERT OR IGNORE + DELETE in transaction | Periodic (cache flush) | **Transactional** ✓ | Good: Batches cache-to-main table transfer, but job-cache is per-request |
| `TransferCachedJobToMain()` (line 143-162) | INSERT + DELETE (separate) | Per-job transfer | **NOT in transaction** ✗ | **ISSUE**: Two separate Exec() calls without transaction - race condition risk |

**Key Findings:**
- Job inserts use coarse `Mutex` locking instead of transactions
- Every job start with tags locks the entire JobRepository for the duration
- `TransferCachedJobToMain()` has two separate writes without transactional guarantees

#### 1.2 Job Stop/Update Operations
**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `Stop()` (line 208-226) | UPDATE job SET state/duration/monitoring_status | Per job-stop | No transaction | Single update, but needs cache invalidation |
| `UpdateMonitoringStatus()` (line 892-902) | UPDATE job SET monitoring_status | Archiving failures | No transaction | Called frequently during archiving errors |
| `UpdateMetadata()` (line 268-306) | UPDATE job SET meta_data + archive.UpdateMetadata() | Per metadata edit | No transaction | Two writes: DB + archive file (potential inconsistency) |
| `DeleteJobByID()` (line 470-477) | DELETE from job | On-demand cleanup | No transaction | Single delete but should cascade with jobtag |
| `DeleteJobsBefore()` (line 399-461) | DELETE from job WHERE start_time < ? | Periodic cleanup | No transaction | Bulk delete (potentially thousands of rows) without transaction |

#### 1.3 Footprint & Energy Updates
**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `UpdateFootprint()` (line 1010-1045) | Builds UPDATE stmt (not executed) | Archiving, footprint service | Returns builder only | Caller must execute; no transaction wrapping |
| `UpdateEnergy()` (line 960-999) | Builds UPDATE stmt (not executed) | Archiving, footprint service | Returns builder only | Caller must execute; no transaction wrapping |

**Critical Issue:** These return unexecuted UPDATE builders, caller is responsible for execution:
- Archiver calls both in sequence (archiveWorker.go line 147-152) - **2 separate statements**
- updateFootprintService calls them per-job in a loop - **potential N separate statements**

---

### SECTION 2: NODE STATE WRITES

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `AddNode()` (line 182-197) | NamedExec INSERT node | On first state update | No transaction | Single insert, minimal issue |
| `UpdateNodeState()` (line 208-245) | SELECT node, AddNode if missing, then NamedExec node_state | Per state update | No transaction | **CONTENTION**: Two writes (node + state) without transaction |
| `BatchUpdateNodeStates()` (line 257-316) | **Transactional batch** | Per REST /api/nodestats | **YES - Full transaction** ✓ | **GOOD**: Entire batch in single transaction |
| `DeleteNodeStatesBefore()` (line 383-396) | DELETE node_state WHERE time_stamp < ? | Periodic cleanup | No transaction | Bulk delete without transaction |

**Key Finding:**
- `BatchUpdateNodeStates()` is well-optimized (transaction + prepared statements)
- But `UpdateNodeState()` is called per-request and NOT transactional (called from NATS)

---

### SECTION 3: USER MANAGEMENT WRITES

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/user.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `AddUser()` (line 126-189) | INSERT hpc_user + INSERT configuration (for each default cluster) | Per user creation | No transaction | **CONTENTION**: Loop with separate INSERT per cluster (lines 167-184) |
| `AddUserIfNotExists()` (line 195-204) | INSERT OR IGNORE | LDAP sync/concurrent login | **Implicit safeguard** | Uses INSERT OR IGNORE for race condition safety |
| `UpdateUser()` (line 206-276) | Multiple UPDATE calls in conditionals (lines 210-273) | Per admin user update | **NO - Separate calls** ✗ | **CONTENTION**: Each role/project change is separate UPDATE with no transaction |
| `AddRole()` (line 336-363) | UPDATE hpc_user SET roles | Per admin request | No transaction | Single update |
| `RemoveRole()` (line 370-404) | UPDATE hpc_user SET roles | Per admin request | No transaction | Single update |
| `AddProject()` (line 412-436) | UPDATE hpc_user SET projects | Per admin request | No transaction | Single update |
| `RemoveProject()` (line 438-476) | UPDATE hpc_user SET projects | Per admin request | No transaction | Single update |
| `DelUser()` (line 278-286) | DELETE hpc_user | Per admin request | No transaction | Single delete |

**Key Findings:**
- `AddUser()` has implicit loop with N separate INSERTs for default metrics (deprecated code at line 162)
- `UpdateUser()` has 4 separate UPDATE paths with no transaction (lines 210, 220, 227, 233, 245, 248, 256, 270)

---

### SECTION 4: TAG MANAGEMENT WRITES

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/tags.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `CreateTag()` (line 255-271) | INSERT tag | Per unique tag | No transaction | Single insert, minimal contention |
| `AddTag()` (line 73-83) | INSERT jobtag + archive.UpdateTags() | Per tag-job association | No transaction | **CONTENTION**: Two writes (DB + archive) without transaction |
| `AddTagDirect()` (line 87-97) | Same as AddTag | From importer/batch | No transaction | Same issue |
| `addJobTag()` (line 398-420) | INSERT jobtag + getTags callback + archive.UpdateTags() | Internal helper | No transaction | **THREE operations**: INSERT, SELECT, archive update |
| `AddTagOrCreate()` (line 424-451) | CreateTag if missing, then AddTag | Per request | No transaction | Nested writes without transaction |
| `RemoveTag()` (line 101-129) | DELETE jobtag + archive.UpdateTags() | Per tag removal | No transaction | **CONTENTION**: Two writes without transaction |
| `RemoveTagByID()` (line 216-244) | DELETE jobtag + DELETE tag + async removeTagFromArchiveJobs | Per tag deletion | **Partial** ✗ | Two DB deletes OK, but async archive update has race condition |
| `RemoveJobTagByRequest()` (line 133-170) | Same as RemoveTag | REST API path | No transaction | Two writes without transaction |
| `ImportTag()` (line 620-640) | INSERT jobtag (after CreateTag checked) | Per import | No transaction | Single insert in loop |

**Key Findings:**
- Tag operations pair DB writes with archive file updates without transactions
- `RemoveTagByID()` spawns goroutine (line 241) for archive update - potential race condition

---

### SECTION 5: USER CONFIG WRITES

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/userConfig.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `UpdateConfig()` (line 110-125) | REPLACE configuration | Per UI config change | No transaction | Single REPLACE, minimal contention |

---

### SECTION 6: ARCHIVER WRITES (BACKGROUND WORKER)

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| Archiving sequence (line 130-160) | UpdateMonitoringStatus (on error) + UpdateFootprint() + UpdateEnergy() + Exec | Per job archiving | **SEPARATE calls** ✗ | **CRITICAL**: UpdateFootprint/UpdateEnergy return builders, archiver executes 2 statements without transaction (lines 147-155) |

**Critical Issue:**
```
stmt := sq.Update("job").Where("job.id = ?", job.ID)
stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta)  // Modifies stmt
if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta)  // Further modifies stmt
// Only NOW is stmt.Exec() called - both updates in single statement
// BUT UpdateMonitoringStatus() above is separate (line 130) = 2 separate writes
```

---

### SECTION 7: IMPORTER WRITES (BULK OPERATIONS)

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/importer/initDB.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `InitDB()` (line 49-154) | Batch 100 jobs per transaction | Startup/full reimport | **Transactional batches** ✓ | GOOD: Groups 100 INSERT + tag operations per transaction |
| Job insert in batch | TransactionAddNamed for job | Per 100 jobs | Within transaction ✓ | OK |
| Tag insert in batch | TransactionAdd for tag/jobtag | Per tag in job | Within transaction ✓ | OK |

**Key Finding:**
- `InitDB()` is well-optimized with batch transactions (100 jobs each)

#### File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/importer/handleImport.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `HandleImportFlag()` (line 38-121) | Per-file: archive.ImportJob + InsertJobDirect + loop ImportTag | Per import file | **NO transaction** ✗ | **ISSUE**: Each imported job is separate (no batching), tags inserted one-by-one |

---

### SECTION 8: REST API WRITES (internal/api)

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `startJob()` (REST POST /api/jobs/) | Start/tag write sequence | Per REST request | Caller handles via NATS/archiver | Delegates to repository |
| `stopJob()` (REST POST /api/jobs/stop/) | Stop + UpdateMonitoringStatus | Per REST request | Separate calls | Job.Stop() + archiver trigger |
| `editMeta()` (PATCH /api/jobs/edit_meta/{id}) | UpdateMetadata() | Per edit request | No transaction | Single DB write + archive |
| `deleteJob()` | DeleteJobByID() | Per admin request | No transaction | Single delete |

#### File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/user.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `updateUser()` (line 76-128) | AddRole/RemoveRole/AddProject/RemoveProject conditionally | Per admin request | **NO transaction** ✗ | Each operation is separate UPDATE call (lines 94, 102, 110, 118) |
| `createUser()` (line 147-193) | AddUser (with deprecated metric config loop) | Per user creation | No transaction | Delegates to repository |

**Key Finding:**
- API handlers directly call repository methods without batching

#### File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `updateNodeStates()` (line 71-150) | BatchUpdateNodeStates() | Per REST /api/nodestats | **YES** ✓ | GOOD: Already using batch transaction wrapper |

#### File: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go`

| Operation | Pattern | Frequency | Transaction Status | Issue |
|-----------|---------|-----------|-------------------|-------|
| `handleStartJob()` (line 180-247) | Start/tag sequence + RepositoryMutex | Per NATS job.start message | Mutex + conditional | Serializes duplicate checks across all job starts |
| `handleStopJob()` (line 252-300+) | Stop + monitoring status | Per NATS job.stop message | No transaction | Separate calls |

**Key Finding:**
- `handleStartJob()` uses coarse `RepositoryMutex` for all job starts (limits concurrency to 1)

---

## SECTION 9: TRANSACTION SUPPORT INFRASTRUCTURE

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/transaction.go`

| Operation | Pattern | Available | Used | Issue |
|-----------|---------|-----------|------|-------|
| `TransactionInit()` | Begin transaction | YES | ✓ InitDB only | Underutilized outside of initialization |
| `TransactionAdd()` | Exec within tx | YES | ✓ InitDB only | Not used in REST/NATS handlers |
| `TransactionAddNamed()` | NamedExec within tx | YES | ✓ InitDB only | Designed for batch but only used in InitDB |
| `Commit()` / `Rollback()` | Standard tx control | YES | ✓ | Available but not used in hot paths |

---

## HIGH-PRIORITY CONTENTION ISSUES

### **CRITICAL (P0):**

1. **Job Start Serialization** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:206`
   - `RepositoryMutex.Lock()` serializes ALL concurrent job starts
   - Limits throughput to single job start at a time
   - Held during duplicate check + tag operations
   - **Impact:** NATS job start concurrency = 1

2. **TransferCachedJobToMain Race Condition** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobCreate.go:143-162`
   - INSERT + DELETE are separate `Exec()` calls
   - No transaction between them
   - If process crashes between INSERT and DELETE, job_cache row orphaned
   - **Impact:** Data loss / inconsistency risk during transfers

3. **Tag Operations Without Transactions** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/tags.go`
   - `AddTag()`: DB INSERT jobtag + archive.UpdateTags() separate
   - `RemoveTag()`: DB DELETE jobtag + archive.UpdateTags() separate
   - `RemoveTagByID()`: spawns async goroutine for archive (race condition)
   - **Impact:** DB-archive inconsistency if either fails mid-operation

4. **Archiver Multiple Separate Updates** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go:130,147-155`
   - UpdateMonitoringStatus (line 130) = separate write
   - UpdateFootprint + UpdateEnergy (lines 147-155) = combined in single statement
   - But if footprint fails, energy never updated
   - **Impact:** Incomplete job archiving data

### **HIGH (P1):**

5. **UpdateUser Without Transaction** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/user.go:206-276`
   - 4 separate UPDATE paths, no transaction
   - If admin modifies role+project, inconsistent state if second fails
   - **Impact:** User permission inconsistencies

6. **AddUser Deprecated Loop** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/user.go:167-184`
   - Loop with separate INSERT per cluster
   - Code marked DEPRECATED but still runs
   - **Impact:** N separate writes per user creation

7. **HandleImportFlag No Batching** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/importer/handleImport.go:38-121`
   - Each imported job is individual (no transaction)
   - Loop of InsertJobDirect + ImportTag calls
   - **Impact:** No batch optimization for bulk imports

8. **DeleteJobsBefore Bulk Without Transaction** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go:399-461`
   - DELETE from job WHERE start_time < ?
   - Potentially thousands of rows, single statement but no transaction
   - If cascades fail, inconsistent state
   - **Impact:** Data integrity during bulk cleanup

9. **UpdateMetadata Split Write** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go:268-306`
   - DB UPDATE + archive.UpdateMetadata() separate
   - No transaction
   - **Impact:** DB-archive inconsistency on failure

10. **REST API User Updates Serialize** - `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/user.go:76-128`
    - Each role/project operation is separate REST handler call
    - updateUser() itself calls single repository method per request
    - **Impact:** User sees 4 separate state changes if adding role+project

---

## SUMMARY TABLE - WRITE OPERATIONS BY CONTENTION RISK

| Component | Location | Writes/Batch | Transaction | Frequency | Risk Level |
|-----------|----------|--------------|-------------|-----------|-----------|
| Job Start (NATS) | nats.go:206 | 1-2 (job+tags) | **Mutex-only** | Per job start | **CRITICAL** |
| Job Start (REST) | jobCreate.go | 1-2 (job+tags) | Mutex-only | Per job start | CRITICAL |
| TransferCachedJobToMain | jobCreate.go:144,156 | 2 (INSERT+DELETE) | **None** | Cache flush | **CRITICAL** |
| Tag Add/Remove | tags.go | 2 (DB+archive) | None | Per tag op | **HIGH** |
| User Update | user.go:206 | 4 (separate UPDATEs) | None | Per admin update | **HIGH** |
| Node State (batch) | node.go:257 | N (batched) | **Transaction** ✓ | Per REST request | LOW |
| InitDB Import | initDB.go | 100 (batched) | **Transaction** ✓ | Startup | LOW |
| Job Archive | archiveWorker.go | 3 (separate) | Partial ✗ | Per job archive | **HIGH** |

---

## RECOMMENDATIONS FOR OPTIMIZATION

1. **Replace Mutex with Transactions** for job start operations
2. **Batch TagOps** into single transaction for add/remove
3. **Transactionalize** UpdateUser role/project changes
4. **Wrap Archiver** updates in single transaction
5. **Add Transactional Batching** to HandleImportFlag
6. **Fix TransferCachedJobToMain** race by moving to transaction
7. **Remove Deprecated User Config Loop** in AddUser

All paths are now documented with specific file paths, line numbers, and actionable issues.

agentId: ab0b0e624428f5b1f (for resuming to continue this agent's work if needed)
<usage>total_tokens: 120521
tool_uses: 32
duration_ms: 81558</usage>

Prompt:
Explore the cc-backend codebase for all database WRITE operations (INSERT, UPDATE, DELETE, Exec, NamedExec) that could cause SQLite write contention. I need to find patterns that are:

1. Called frequently or in loops without transactions
2. Multiple separate writes that could be batched into a single transaction
3. Write operations triggered by external events (REST handlers, scheduled tasks, archiving)

Focus on these directories:
- internal/repository/ (all .go files)
- internal/api/ (all .go files)
- internal/archiver/
- internal/importer/

For each write operation found, note:
- The function name and file path
- Whether it's already in a transaction
- How frequently it's likely called (per-request, per-job, periodic, etc.)
- Whether multiple writes happen in sequence that could be batched

Be very thorough - check every .go file in these directories. Skip test files.


Perfect! Now I have gathered comprehensive information. Let me create a detailed summary of all the findings.

## Summary Report: Periodic/Scheduled Tasks and Background Goroutines in cc-backend

I've completed a comprehensive exploration of the cc-backend codebase to identify all periodic/scheduled tasks and background goroutines that write to SQLite. Here's the complete findings:

---

### **TASK MANAGER (gocron-based scheduler)**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/`

The main scheduler is initialized in `main.go` line 374:
```go
taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
```

#### **1. Commit Job Service (commitJobService.go)**
- **Frequency:** Default 2 minutes (configurable via `commit-job-worker`)
- **DB Operation:** `jobRepo.SyncJobs()` + callback hooks
- **Type:** Duration-based job (gocron.DurationJob)
- **Batching:** Handled via SyncJobs() call
- **Details:** Syncs job cache and triggers job start hooks

#### **2. Update Duration Worker (updateDurationService.go)**
- **Frequency:** Default 5 minutes (configurable via `duration-worker`)
- **DB Operation:** `jobRepo.UpdateDuration()`
- **Type:** Duration-based job
- **Batching:** Single call, likely batches updates internally
- **Details:** Updates job duration fields in DB

#### **3. Footprint Update Worker (updateFootprintService.go)**
- **Frequency:** Default 10 minutes (configurable via `footprint-worker`)
- **DB Operation:** Multiple per-cluster database updates using transactions
  - Uses `jobRepo.TransactionInit()`, `jobRepo.TransactionAdd()`, `jobRepo.TransactionEnd()`
  - Updates job statistics (avg/min/max) for running jobs
- **Batching:** Proper transaction batching per cluster
- **Details:**
  - Iterates through all clusters and running jobs
  - Loads metrics from metric stores
  - Builds UPDATE statements and batches them per cluster
  - Uses transactions to apply all updates atomically

#### **4. Stop Jobs Exceeding Walltime (stopJobsExceedTime.go)**
- **Frequency:** Daily at 3:00 AM (hardcoded)
- **DB Operation:** `jobRepo.StopJobsExceedingWalltimeBy(duration)` + GC
- **Type:** Daily job (gocron.DailyJob)
- **Batching:** Handled internally by repository function
- **Conditional:** Only runs if `config.Keys.StopJobsExceedingWalltime > 0`

#### **5. Retention Delete Service (retentionService.go)**
- **Frequency:** Daily at 3:00 AM (hardcoded)
- **DB Operation:** `jobRepo.FindJobsBetween()` + `jobRepo.DeleteJobsBefore()` + `jobRepo.Optimize()`
- **Type:** Daily job
- **Batching:** Deletes are batched via `DeleteJobsBefore()`
- **Conditional:** Only if retention policy = "delete"
- **Details:** Deletes archived jobs older than configured age

#### **6. Retention Copy Service (retentionService.go)**
- **Frequency:** Daily at 4:00 AM (hardcoded)
- **DB Operation:** `jobRepo.FindJobsBetween()` (no DB writes, only archive operations)
- **Type:** Daily job
- **Conditional:** Only if retention policy = "copy"
- **Details:** Copies jobs to secondary backend without DB deletions

#### **7. Retention Move Service (retentionService.go)**
- **Frequency:** Daily at 5:00 AM (hardcoded)
- **DB Operation:** `jobRepo.FindJobsBetween()` + `jobRepo.DeleteJobsBefore()` + `jobRepo.Optimize()`
- **Type:** Daily job
- **Batching:** Deletes are batched
- **Conditional:** Only if retention policy = "move"
- **Details:** Moves jobs to secondary backend then deletes from DB

#### **8. Compression Service (compressionService.go)**
- **Frequency:** Daily at 5:00 AM (hardcoded, conflicts with move service)
- **DB Operation:** `jobRepo.FindJobsBetween()` (no DB writes, only archive compression)
- **Type:** Daily job
- **Details:** Compresses old archived jobs
- **Always enabled:** Even without explicit retention config

#### **9. LDAP Sync Service (ldapSyncService.go)**
- **Frequency:** Configurable via `LdapConfig.SyncInterval`
- **DB Operation:** `auth.LdapAuth.Sync()` (user table updates, not job-related)
- **Type:** Duration-based job
- **Conditional:** Only if LDAP is configured

#### **10. Node State Retention Delete Service (nodestateRetentionService.go)**
- **Frequency:** Daily at 2:00 AM (hardcoded)
- **DB Operation:** `nodeRepo.DeleteNodeStatesBefore(cutoff)`
- **Type:** Daily job
- **Conditional:** Only if `config.Keys.NodeStateRetention.Policy == "delete"`
- **Details:** Deletes node state records older than configured age (default 24 hours)

#### **11. Node State Retention Move Service (nodestateRetentionService.go)**
- **Frequency:** Daily at 2:30 AM (hardcoded)
- **DB Operation:** `nodeRepo.FindNodeStatesBefore()` + `nodeRepo.DeleteNodeStatesBefore()`
- **Type:** Daily job
- **Conditional:** Only if `config.Keys.NodeStateRetention.Policy == "move"`
- **Details:** Archives node states to Parquet then deletes from DB

---

### **METRIC STORE BACKGROUND WORKERS**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/`

Initialized in `main.go` line 343:
```go
metricstore.Init(mscfg, metrics, &wg)
```

These goroutines run concurrently and handle metric data persistence. **Note: These do NOT directly write to the SQLite job/node database, but they handle metric checkpoint persistence to disk.**

#### **1. Checkpointing Worker (checkpoint.go:94)**
- **Frequency:** 12 hours (default, configurable via `checkpoint-interval`)
- **Operation:** Periodic metric data checkpoints to JSON or binary WAL format
- **Type:** Duration-based ticker
- **Details:**
  - Saves in-memory metric buffers to disk
  - Two formats: JSON (human-readable) or WAL+binary (crash-safe)
  - Uses worker pool with `Keys.NumWorkers` (default: min(CPU/2+1, 10))
  - Rotates WAL files after successful checkpoint

#### **2. WAL Staging Worker (walCheckpoint.go:118)**
- **Frequency:** Continuous (processes messages from channel)
- **Operation:** Writes individual metric entries to write-ahead logs
- **Type:** Channel-based message processor
- **Details:**
  - Processes `WALMessages` channel (from line protocol ingestion)
  - Writes binary WAL records to per-host WAL files
  - Manages file handles for efficient buffering
  - Handles WAL rotation on command

#### **3. Retention Worker (metricstore.go:320)**
- **Frequency:** Half of `Keys.RetentionInMemory` interval
- **Operation:** Frees in-memory metric buffers older than retention period
- **Type:** Duration-based ticker
- **Details:**
  - Calls `Free(ms, t)` to remove old data from memory
  - Cleans buffer pool with `bufferPool.Clean()`
  - Does NOT write to SQLite, only memory management

#### **4. Cleanup Worker (archive.go:23)**
- **Frequency:** `Keys.RetentionInMemory` interval
- **Operation:** Either archives or deletes old checkpoint files
- **Type:** Duration-based ticker
- **Modes:**
  - "archive": Moves old checkpoint files to Parquet format
  - "delete": Deletes old checkpoint files
- **Details:**
  - Uses worker pool for parallel processing
  - Supports both local filesystem and S3 targets
  - Runs daily cleanup job

#### **5. Memory Usage Tracker (metricstore.go:383)**
- **Frequency:** `DefaultMemoryUsageTrackerInterval` (appears to be 5-10 min based on code)
- **Operation:** Monitors memory usage, force-frees buffers if exceeding cap
- **Type:** Duration-based ticker
- **Details:**
  - Checks `runtime.MemStats.Alloc`
  - Force-frees buffers if memory cap exceeded
  - Calls `debug.FreeOSMemory()` to return memory to OS
  - Does NOT write to SQLite

---

### **ARCHIVER BACKGROUND WORKER**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go`

**Initialization:** `main.go` line 373:
```go
archiver.Start(repository.GetJobRepository(), ctx)
```

#### **Archiving Worker (archiveWorker.go:110)**
- **Frequency:** On-demand (triggered by `TriggerArchiving()` calls)
- **DB Operations:**
  - `jobRepo.FetchMetadata(job)` - read
  - `ArchiveJob(job, ctx)` - archive to external storage
  - `jobRepo.UpdateFootprint(stmt, jobMeta)` - **writes**
  - `jobRepo.UpdateEnergy(stmt, jobMeta)` - **writes**
  - `jobRepo.MarkArchived(stmt, status)` - **writes**
  - `jobRepo.Execute(stmt)` - executes composite update
- **Type:** Channel-based consumer (buffered channel, capacity 128)
- **Batching:** Single UPDATE statement with multiple SET clauses
- **Trigger Points:**
  - Called when job stops (via REST API `/api/jobs/stop_job/`)
  - Called when job stops via NATS API
- **Details:**
  - Processes archiving asynchronously
  - Uses `sync.WaitGroup` for graceful shutdown
  - Updates job statistics (footprint, energy, monitoring status) atomically
  - Calls job stop hooks after successful archiving

---

### **REST API ENDPOINTS THAT WRITE TO DATABASE**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/`

#### **Job Start Endpoint** (`job.go:717`)
- **Path:** `POST/PUT /api/jobs/start_job/`
- **DB Operations:**
  - `jobRepo.FindAll()` - check for duplicates (read)
  - `jobRepo.StartDirect()` or `jobRepo.Start()` - **inserts new job**
  - `jobRepo.AddTagOrCreate()` - **adds tags** (if provided)
- **Concurrency:** Uses `RepositoryMutex` to prevent race conditions on duplicate checks
- **Batching:** Individual calls, not explicitly batched
- **Async:** Tag addition is async (after response)

#### **Job Stop Endpoint** (`job.go:805`)
- **Path:** `POST/PUT /api/jobs/stop_job/`
- **DB Operations:**
  - `jobRepo.FindCached()` or `jobRepo.Find()` - read
  - `checkAndHandleStopJob()` - updates job state (line 1000+)
  - Triggers `archiver.TriggerArchiving()` - queues async archiving
- **Details:** Moves job to stopped state, triggers async archiving

#### **Update Node States Endpoint** (`node.go:71`)
- **Path:** `POST/PUT /api/nodestate/`
- **DB Operations:**
  - `nodeRepo.InsertNodeStateBegin()` - **starts transaction**
  - `nodeRepo.InsertNodeState()` - **inserts node states** (batched per subcluster)
  - `nodeRepo.InsertNodeStateEnd()` - **commits transaction**
- **Batching:** Per-subcluster transactions
- **Details:** Updates node allocation/state information

#### **Tag Job Endpoint** (`job.go:545`)
- **Path:** `POST/PATCH /api/jobs/tag_job/{id}`
- **DB Operations:** `jobRepo.AddTagOrCreate()` - **inserts/creates tags**
- **Batching:** No explicit batching

#### **Remove Tag Endpoint** (`job.go:609`)
- **Path:** `DELETE /api/jobs/tag_job/{id}`
- **DB Operations:** Tag removal operations
- **Batching:** No explicit batching

#### **Delete Job Endpoints** (`job.go:852, 897, 950`)
- **Paths:**
  - `DELETE /api/jobs/delete_job/{id}`
  - `DELETE /api/jobs/delete_job/`
  - `DELETE /api/jobs/delete_job_before/{ts}`
- **DB Operations:** `jobRepo.Delete*()` functions
- **Details:** Remove jobs from database

#### **Edit Meta Endpoint** (`job.go`)
- **Path:** `PATCH /api/jobs/edit_meta/{id}`
- **DB Operations:** Updates job metadata
- **Details:** User-facing metadata updates

---

### **NATS API SUBSCRIPTIONS** (Alternative to REST)

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go`

**Initialization:** `server.go:282-287` - conditionally started if `config.Keys.APISubjects` configured

#### **Job Event Subscription** (`nats.go:146`)
- **Subject:** `config.Keys.APISubjects.SubjectJobEvent` (e.g., "cc.job.event")
- **Message Format:** InfluxDB line protocol with JSON payload
- **Operations:** Same as REST endpoints (start_job, stop_job)
- **Concurrency Control:** Uses semaphore (`jobSem` buffer of 8) to limit concurrent processing
- **Details:** Asynchronous job processing via NATS messages

#### **Node State Subscription** (`nats.go`)
- **Subject:** `config.Keys.APISubjects.SubjectNodeState` (e.g., "cc.node.state")
- **Operations:** Same as REST node state updates
- **Concurrency Control:** Uses semaphore (`nodeSem` buffer of 2)

---

### **INITIALIZATION AND SHUTDOWN FLOW**

**Main Startup Sequence:** (`main.go` lines 343-374)
1. `metricstore.Init()` - starts 5 background workers
2. `archiver.Start()` - starts archiving worker goroutine
3. `taskmanager.Start()` - starts gocron scheduler with 11 periodic tasks

**Graceful Shutdown Sequence:** (`server.go:418-441`)
1. Server shutdown (30s timeout)
2. NATS client close
3. `metricstore.Shutdown()` - cancels metric worker contexts
4. `archiver.Shutdown(10s)` - waits for pending archiving jobs with timeout
5. `taskmanager.Shutdown()` - stops gocron scheduler

---

### **TRANSACTION/BATCHING SUMMARY**

| Component | Uses Transactions | Batching Strategy |
|-----------|-------------------|-------------------|
| Footprint Worker | YES (per-cluster) | Builds update statements, executes in transaction |
| Node State Updates | YES (per-subcluster) | `InsertNodeStateBegin/End` wrapper |
| Job Start | NO | Individual insert (has mutex for duplicate prevention) |
| Job Stop | NO | Async archiving triggered separately |
| Retention Delete | Implicit | `DeleteJobsBefore()` handles batching |
| Stop Jobs Exceed | Implicit | `StopJobsExceedingWalltimeBy()` batches |
| Archiver | NO | Single UPDATE statement with multiple SET clauses |
| LDAP Sync | Dependent | Implementation in auth package |
| Node State Retention | Implicit | `DeleteNodeStatesBefore()` batches |

---

### **FILE LOCATIONS SUMMARY**

**Critical Files:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/` - 11 periodic task definitions
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` - 5 background workers
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go` - async archiving
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/rest.go` - REST route registration
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go` - NATS subscriptions
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go` - initialization sequence
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go` - shutdown sequence

agentId: a5a9b60b3b5a2afcf (for resuming to continue this agent's work if needed)
<usage>total_tokens: 74431
tool_uses: 31
duration_ms: 85216</usage>

Prompt:
Explore the cc-backend codebase to find all periodic/scheduled tasks and background goroutines that write to the SQLite database. Look for:

1. Goroutines started with `go func()` that do DB writes
2. Ticker/timer-based loops that write to the DB
3. Archiving operations that write to the DB
4. Retention/cleanup operations
5. Any cron-like scheduled operations

Search in:
- cmd/cc-backend/ (main entry point, initialization)
- internal/archiver/
- internal/taskmanager/ or similar
- internal/repository/
- Any file with "schedule", "ticker", "cron", "periodic", "retention", "cleanup" patterns

For each, note the frequency, what DB operations it performs, and whether those operations are already batched/transacted.

Also look at the REST API file internal/api/rest.go to understand all registered routes and which ones do DB writes.


Good exploration results. Let me now read a few critical files to validate the findings before writing the plan.