internal/repository/job.go:814:func (r *JobRepository) UpdateDuration() error {
internal/repository/job.go-815-	stmnt := sq.Update("job").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-816-		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-817-		Where("job_state = 'running'")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-818-
internal/repository/job.go-819-	_, err := stmnt.RunWith(r.stmtCache).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-820-	if err != nil {
internal/repository/job.go-821-		cclog.Errorf("Error while updating duration for running jobs: %v", err)
internal/repository/job.go-822-		return fmt.Errorf("failed to update duration for running jobs: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-823-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-824-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-825-	return nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-826-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-827-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-828-// FindJobsBetween returns jobs within a specified time range.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-829-// If startTimeBegin is 0, returns all jobs before startTimeEnd.


internal/repository/job.go:399:func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged string) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-400-	var cnt int
internal/repository/job.go-401-	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-402-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-403-	switch omitTagged {
internal/repository/job.go-404-	case "all":
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-405-		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
internal/repository/job.go-406-	case "user":
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-407-		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-408-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-409-
internal/repository/job.go-410-	if err := q.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
internal/repository/job.go-411-		cclog.Errorf("Error counting jobs before %d: %v", startTime, err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-412-		return 0, err
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-413-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-414-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-415-	// Invalidate cache for jobs being deleted (get job IDs first)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-416-	if cnt > 0 {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-417-		var jobIds []int64
internal/repository/job.go-418-		selectQuery := sq.Select("id").From("job").Where("job.start_time < ?", startTime)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-419-


internal/repository/job.go:712:func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
internal/repository/job.go-713-	start := time.Now()
internal/repository/job.go-714-	currentTime := time.Now().Unix()
internal/repository/job.go-715-	res, err := sq.Update("job").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-716-		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-717-		Set("duration", 0).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-718-		Set("job_state", schema.JobStateFailed).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-719-		Where("job.job_state = 'running'").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-720-		Where("job.walltime > 0").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-721-		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-722-		RunWith(r.DB).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-723-	if err != nil {
internal/repository/job.go-724-		cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err)
internal/repository/job.go-725-		return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-726-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-727-
internal/repository/job.go-728-	rowsAffected, err := res.RowsAffected()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-729-	if err != nil {
internal/repository/job.go-730-		cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err)
internal/repository/job.go-731-		return fmt.Errorf("failed to get rows affected count: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-732-	}


Good — `UpdateDuration`, `StopJobsExceedingWalltimeBy`, and `DeleteJobsBefore` are all single bulk SQL statements (no loops), so they're already efficient. The footprint worker already uses transactions properly.

Now I have a complete picture. Let me write the plan.


Write

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically


The file /Users/jan/.claude/plans/woolly-wandering-narwhal.md has been updated successfully.

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically

-# Fix: SQLite Write Contention Causing OOM Crash

+# Additional SQLite Write Contention Reductions

-The server crashes daily around 15:00-16:00 with OOM. Logs show "database is locked" errors and LDAP sync failures (`UNIQUE constraint failed: hpc_user.username`) shortly before the crash.

+Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

-**Root cause:** A burst of concurrent DB writers (NATS job start/stop, node state updates, LDAP sync, scheduled tasks) overwhelm SQLite's 5-second busy_timeout. Failed operations and blocked goroutines accumulate, consuming memory until OOM.

+After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:

+- `UpdateDuration` — single bulk UPDATE, efficient

+- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient

+- `DeleteJobsBefore` — single bulk DELETE, efficient

+- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster

+- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

-The cascade: burst of job events → DB contention → `SQLITE_BUSY` after 5s → error paths spawn retries/new goroutines → goroutine accumulation → OOM.

+The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

-## Fix 1: Increase busy_timeout (contention relief)

+## Fix 6: Wrap TransferCachedJobToMain in a Transaction

-Change `_busy_timeout` from `5000` to `60000` (60 seconds). Also make it configurable:

+Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

-**File:** `internal/repository/config.go` — add `BusyTimeoutMs int` field, default 60000.

+**Change:** Wrap both operations in a transaction:

+func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {

+    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?

+    // DELETE FROM job_cache WHERE id = ?

-**File:** `internal/repository/dbConnection.go:73` — use `repoConfig.BusyTimeoutMs` instead of hardcoded value.

+**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

-**File:** `pkg/archive/sqliteBackend.go:105` — also change hardcoded `busy_timeout=5000` to `busy_timeout=60000`.

-With 60s timeout, SQLite retries internally using its backoff mechanism rather than failing fast. This prevents the "database is locked" error cascade.

-## Fix 2: Batch Node State Updates in a Transaction

+## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

-Currently each node state update is a separate SELECT + INSERT (2 DB round-trips per node). A message with 1000 nodes = 2000+ individual DB operations, each acquiring/releasing the write lock.

+**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag

+**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

-**File:** `internal/repository/node.go` — add `BatchUpdateNodeStates` method:

-- Accept a slice of `(hostname, cluster, nodeState)` tuples

+Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

+**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:

-- For each node: lookup ID (or insert node), then insert state — all within the same transaction

+- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction

+- Single archive.UpdateTags call at the end (outside transaction)

-**File:** `internal/api/nats.go:348-364` (`processNodestateEvent`) — build slice of updates, call `BatchUpdateNodeStates` instead of per-node `UpdateNodeState`.

+- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`

+- `internal/api/job.go` (startJob REST handler) — same change

-**File:** `internal/api/node.go:119-142` (`updateNodeStates`) — same change for the REST path. Note: the REST handler also does health checks per node before DB inserts, so batch only the DB part (the loop at lines 119-142).

+**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

-Keep the existing `UpdateNodeState` method for single-node use cases.

+**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

-The LDAP sync fails when a user is concurrently created by a login. The error is benign but causes `Sync()` to return early (line 215: `return err`), abandoning remaining users.

+**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

-**File:** `internal/repository/user.go` — add `AddUserIfNotExists` method using `INSERT OR IGNORE`:

-func (r *UserRepository) AddUserIfNotExists(user *schema.User) error {

-    rolesJson, _ := json.Marshal(user.Roles)

-    projectsJson, _ := json.Marshal(user.Projects)

-    cols := "username, name, roles, projects, ldap"

-        `INSERT OR IGNORE INTO hpc_user (` + cols + `) VALUES (?, ?, ?, ?, ?)`,

-        user.Username, user.Name, string(rolesJson), string(projectsJson), int(user.AuthSource))

+Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

-**File:** `internal/auth/ldap.go:213` — change `ur.AddUser(user)` to `ur.AddUserIfNotExists(user)`.

+**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:

+- Accept jobID + slice of (type, name, scope)

+- For each tag: resolve/create tag ID + INSERT jobtag

-Note: `AddUserIfNotExists` intentionally skips the deprecated default metrics config insertion that `AddUser` does — LDAP-synced users don't need it, and it adds unnecessary DB writes.

+Update `handleImport.go` to call `BatchImportTags` instead of the loop.

+**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

-## Fix 4: Add Semaphore for NATS Handlers

+## Fix 9: Wrap RemoveTagByID in a Transaction

-Without rate limiting, blocked NATS handlers cause unbounded goroutine accumulation.

-**File:** `internal/api/nats.go` — add semaphores to `NatsAPI` struct:

+Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

-    JobRepository   *repository.JobRepository

+func (r *JobRepository) RemoveTagByID(tagID int64) error {

+    jobIds, err := r.FindJobIdsByTag(tagID)

+    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)

+    // DELETE FROM tag WHERE id = ?  (on tx)

+    go r.removeTagFromArchiveJobs(jobIds)

-Initialize in `NewNatsAPI`: `jobSem: make(chan struct{}, 8)`, `nodeSem: make(chan struct{}, 2)`.

+**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

-Add acquire/release in `handleJobEvent` and `handleNodeState`:

-This caps concurrent NATS processing at 8 job events + 2 node state updates, preventing goroutine pile-up.

-## Fix 5: Wrap SyncJobs in a Transaction

+1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug

+2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path

+3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists

+4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

-Currently does SELECT + INSERT...SELECT + DELETE as 3 separate operations. Wrap in a DB transaction:

-- Do the read-only ID resolution outside the transaction

-This reduces lock acquisitions from 3 to 1 and ensures atomicity.

+| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |

+| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |

+| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |

+| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |

+| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

-1. **Fix 1** (busy_timeout) — quick config change, biggest immediate impact

-2. **Fix 3** (LDAP sync) — eliminates error spam, trivial change

-3. **Fix 2** (batch node states) — biggest contention reduction

-4. **Fix 5** (SyncJobs transaction) — safety + minor contention reduction

-5. **Fix 4** (NATS semaphore) — defense in depth against goroutine accumulation

+These are already efficient and don't need modification:

+- `UpdateDuration` — single bulk UPDATE

+- `StopJobsExceedingWalltimeBy` — single bulk UPDATE

+- `DeleteJobsBefore` — single bulk DELETE

+- Footprint worker — already transactional per cluster

+- Archiver — already chains SETs into single UPDATE via Squirrel builder

+- `UpdateUser` — admin-only, very low frequency, not worth the complexity

+- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

-1. After Fix 1: Check logs for "database is locked" — should be gone or very rare.

-2. After Fix 2: check `Timer updateNodeStates, SQLite Inserts` log line — should be much faster.

-3. After Fix 3: check logs for "UNIQUE constraint failed" — should be gone.

-4. Run `go test ./internal/repository/... ./internal/api/... ./internal/auth/...` after all changes.

-5. Load test: simulate burst of 100+ concurrent job start/stop events + node state update with 500 nodes + LDAP sync running simultaneously. Monitor goroutine count and memory.

+1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`

+2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty

+3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)

+4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically

# Fix: SQLite Write Contention Causing OOM Crash

## Context

The server crashes daily around 15:00-16:00 with OOM. Logs show "database is locked" errors and LDAP sync failures (`UNIQUE constraint failed: hpc_user.username`) shortly before the crash.

**Root cause:** A burst of concurrent DB writers (NATS job start/stop, node state updates, LDAP sync, scheduled tasks) overwhelm SQLite's 5-second busy_timeout. Failed operations and blocked goroutines accumulate, consuming memory until OOM.

The cascade: burst of job events → DB contention → `SQLITE_BUSY` after 5s → error paths spawn retries/new goroutines → goroutine accumulation → OOM.

---

## Fix 1: Increase busy_timeout (contention relief)

**File:** `internal/repository/dbConnection.go:73`

Change `_busy_timeout` from `5000` to `60000` (60 seconds). Also make it configurable:

**File:** `internal/repository/config.go` — add `BusyTimeoutMs int` field, default 60000.

**File:** `internal/repository/dbConnection.go:73` — use `repoConfig.BusyTimeoutMs` instead of hardcoded value.

**File:** `pkg/archive/sqliteBackend.go:105` — also change hardcoded `busy_timeout=5000` to `busy_timeout=60000`.

With 60s timeout, SQLite retries internally using its backoff mechanism rather than failing fast. This prevents the "database is locked" error cascade.

---

## Fix 2: Batch Node State Updates in a Transaction

Currently each node state update is a separate SELECT + INSERT (2 DB round-trips per node). A message with 1000 nodes = 2000+ individual DB operations, each acquiring/releasing the write lock.

**File:** `internal/repository/node.go` — add `BatchUpdateNodeStates` method:
- Accept a slice of `(hostname, cluster, nodeState)` tuples
- Begin transaction
- For each node: lookup ID (or insert node), then insert state — all within the same transaction
- Commit once

**File:** `internal/api/nats.go:348-364` (`processNodestateEvent`) — build slice of updates, call `BatchUpdateNodeStates` instead of per-node `UpdateNodeState`.

**File:** `internal/api/node.go:119-142` (`updateNodeStates`) — same change for the REST path. Note: the REST handler also does health checks per node before DB inserts, so batch only the DB part (the loop at lines 119-142).

Keep the existing `UpdateNodeState` method for single-node use cases.

---

## Fix 3: Fix LDAP Sync Race Condition

The LDAP sync fails when a user is concurrently created by a login. The error is benign but causes `Sync()` to return early (line 215: `return err`), abandoning remaining users.

**File:** `internal/repository/user.go` — add `AddUserIfNotExists` method using `INSERT OR IGNORE`:
```go
func (r *UserRepository) AddUserIfNotExists(user *schema.User) error {
    rolesJson, _ := json.Marshal(user.Roles)
    projectsJson, _ := json.Marshal(user.Projects)
    cols := "username, name, roles, projects, ldap"
    _, err := r.DB.Exec(
        `INSERT OR IGNORE INTO hpc_user (` + cols + `) VALUES (?, ?, ?, ?, ?)`,
        user.Username, user.Name, string(rolesJson), string(projectsJson), int(user.AuthSource))
    return err
}
```

**File:** `internal/auth/ldap.go:213` — change `ur.AddUser(user)` to `ur.AddUserIfNotExists(user)`.

Note: `AddUserIfNotExists` intentionally skips the deprecated default metrics config insertion that `AddUser` does — LDAP-synced users don't need it, and it adds unnecessary DB writes.

---

## Fix 4: Add Semaphore for NATS Handlers

Without rate limiting, blocked NATS handlers cause unbounded goroutine accumulation.

**File:** `internal/api/nats.go` — add semaphores to `NatsAPI` struct:
```go
type NatsAPI struct {
    JobRepository   *repository.JobRepository
    RepositoryMutex sync.Mutex
    jobSem          chan struct{}
    nodeSem         chan struct{}
}
```

Initialize in `NewNatsAPI`: `jobSem: make(chan struct{}, 8)`, `nodeSem: make(chan struct{}, 2)`.

Add acquire/release in `handleJobEvent` and `handleNodeState`:
```go
api.jobSem <- struct{}{}
defer func() { <-api.jobSem }()
```

This caps concurrent NATS processing at 8 job events + 2 node state updates, preventing goroutine pile-up.

---

## Fix 5: Wrap SyncJobs in a Transaction

**File:** `internal/repository/jobCreate.go:72-126` (`SyncJobs`)

Currently does SELECT + INSERT...SELECT + DELETE as 3 separate operations. Wrap in a DB transaction:
- `tx, err := r.DB.Beginx()`
- Run all three operations on `tx`
- `tx.Commit()`
- Do the read-only ID resolution outside the transaction

This reduces lock acquisitions from 3 to 1 and ensures atomicity.

---

## Implementation Order

1. **Fix 1** (busy_timeout) — quick config change, biggest immediate impact
2. **Fix 3** (LDAP sync) — eliminates error spam, trivial change
3. **Fix 2** (batch node states) — biggest contention reduction
4. **Fix 5** (SyncJobs transaction) — safety + minor contention reduction
5. **Fix 4** (NATS semaphore) — defense in depth against goroutine accumulation

---

## Verification

1. After Fix 1: Check logs for "database is locked" — should be gone or very rare.
2. After Fix 2: check `Timer updateNodeStates, SQLite Inserts` log line — should be much faster.
3. After Fix 3: check logs for "UNIQUE constraint failed" — should be gone.
4. Run `go test ./internal/repository/... ./internal/api/... ./internal/auth/...` after all changes.
5. Load test: simulate burst of 100+ concurrent job start/stop events + node state update with 500 nodes + LDAP sync running simultaneously. Monitor goroutine count and memory.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


After a thorough review of all DB write paths in the codebase, I found 4 additional opportunities. Most periodic tasks (UpdateDuration, footprint worker, archiver, bulk deletes) are already well-optimized. The remaining wins are:

1. **Fix 6 — TransferCachedJobToMain** (atomicity bug): INSERT + DELETE are two separate calls without a transaction
2. **Fix 7 — Batch tag operations on job start**: Each tag does ~5 DB round-trips; a job with 5 tags = 30 operations that could be 1 transaction
3. **Fix 8 — Batch tag imports**: Same pattern during `--import`, loops ImportTag per tag
4. **Fix 9 — RemoveTagByID transaction**: Two separate DELETEs (jobtag + tag) without a transaction

Fix 7 is the most impactful for runtime contention since it's on the NATS job start hot path.


ExitPlanMode

Prompt: run go build and go test

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically


Error: The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). To tell you how to proceed, the user said:
Only implement fix 6


Implementing Fix 6 — wrapping `TransferCachedJobToMain` in a transaction.


   128→			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
   129→				job.JobID, job.Cluster, job.StartTime).
   130→			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
   131→			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
   132→				job.JobID, job.Cluster, err)
   133→			continue
   134→		}
   135→		job.ID = &newID
   136→	}
   137→
   138→	return jobs, nil
   139→}
   140→
   141→// TransferCachedJobToMain moves a job from job_cache to the job table.
   142→// Caller must hold r.Mutex. Returns the new job table ID.
   143→func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
   144→	res, err := r.DB.Exec(
   145→		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
   146→		cacheID)
   147→	if err != nil {
   148→		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
   149→	}
   150→
   151→	newID, err := res.LastInsertId()
   152→	if err != nil {
   153→		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
   154→	}
   155→
   156→	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
   157→	if err != nil {

			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
				job.JobID, job.Cluster, job.StartTime).
			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
				job.JobID, job.Cluster, err)
			continue
		}
		job.ID = &newID
	}

	return jobs, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {


Edit

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	return newID, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	tx, err := r.DB.Beginx()
	if err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)
	}
	defer tx.Rollback()

	res, err := tx.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	if err := tx.Commit(); err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)
	}

	return newID, nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobCreate.go has been updated successfully.

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	tx, err := r.DB.Beginx()
	if err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)
	}
	defer tx.Rollback()

	res, err := tx.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	if err := tx.Commit(); err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)
	}

	return newID, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"encoding/json"
	"fmt"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
)

const NamedJobCacheInsert string = `INSERT INTO job_cache (
	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
) VALUES (
	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
);`

const NamedJobInsert string = `INSERT INTO job (
	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
) VALUES (
	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
);`

// InsertJobDirect inserts a job directly into the job table (not job_cache).
// Use this when the returned ID will be used for operations on the job table
// (e.g., adding tags), or for imported jobs that are already completed.
func (r *JobRepository) InsertJobDirect(job *schema.Job) (int64, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	res, err := r.DB.NamedExec(NamedJobInsert, job)
	if err != nil {
		cclog.Warn("Error while NamedJobInsert (direct)")
		return 0, err
	}
	id, err := res.LastInsertId()
	if err != nil {
		cclog.Warn("Error while getting last insert ID (direct)")
		return 0, err
	}

	return id, nil
}

func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	res, err := r.DB.NamedExec(NamedJobCacheInsert, job)
	if err != nil {
		cclog.Warn("Error while NamedJobInsert")
		return 0, err
	}
	id, err := res.LastInsertId()
	if err != nil {
		cclog.Warn("Error while getting last insert ID")
		return 0, err
	}

	return id, nil
}

func (r *JobRepository) SyncJobs() ([]*schema.Job, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	query := sq.Select(jobCacheColumns...).From("job_cache")

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running query %v", err)
		return nil, err
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, 50)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warn("Error while scanning rows")
			return nil, err
		}
		jobs = append(jobs, job)
	}

	// Transfer cached jobs to main table and clear cache in a single transaction.
	tx, err := r.DB.Beginx()
	if err != nil {
		cclog.Errorf("SyncJobs: begin transaction: %v", err)
		return nil, err
	}
	defer tx.Rollback()

	// Use INSERT OR IGNORE to skip jobs already transferred by the stop path
	_, err = tx.Exec(
		"INSERT OR IGNORE INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache")
	if err != nil {
		cclog.Errorf("Error while Job sync: %v", err)
		return nil, err
	}

	_, err = tx.Exec("DELETE FROM job_cache")
	if err != nil {
		cclog.Errorf("Error while Job cache clean: %v", err)
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		cclog.Errorf("SyncJobs: commit transaction: %v", err)
		return nil, err
	}

	// Resolve correct job.id from the job table. The IDs read from job_cache
	// are from a different auto-increment sequence and must not be used to
	// query the job table.
	for _, job := range jobs {
		var newID int64
		if err := sq.Select("job.id").From("job").
			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
				job.JobID, job.Cluster, job.StartTime).
			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
				job.JobID, job.Cluster, err)
			continue
		}
		job.ID = &newID
	}

	return jobs, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	return newID, nil
}

// Start inserts a new job in the table, returning the unique job ID.
// Statistics are not transfered!
func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
	job.RawFootprint, err = json.Marshal(job.Footprint)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
	}

	job.RawResources, err = json.Marshal(job.Resources)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
	}

	job.RawMetaData, err = json.Marshal(job.MetaData)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
	}

	return r.InsertJob(job)
}

// StartDirect inserts a new job directly into the job table (not job_cache).
// Use this when the returned ID will immediately be used for job table
// operations such as adding tags.
func (r *JobRepository) StartDirect(job *schema.Job) (id int64, err error) {
	job.RawFootprint, err = json.Marshal(job.Footprint)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
	}

	job.RawResources, err = json.Marshal(job.Resources)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
	}

	job.RawMetaData, err = json.Marshal(job.MetaData)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
	}

	return r.InsertJobDirect(job)
}

// Stop updates the job with the database id jobId using the provided arguments.
func (r *JobRepository) Stop(
	jobID int64,
	duration int32,
	state schema.JobState,
	monitoringStatus int32,
) (err error) {
	// Invalidate cache entries as job state is changing
	r.cache.Del(fmt.Sprintf("metadata:%d", jobID))
	r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID))

	stmt := sq.Update("job").
		Set("job_state", state).
		Set("duration", duration).
		Set("monitoring_status", monitoringStatus).
		Where("job.id = ?", jobID)

	_, err = stmt.RunWith(r.stmtCache).Exec()
	return err
}

+    return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)

-  _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)

+  _, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)

+    return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)


Bash

Build all packages


(Bash completed with no output)


go test ./internal/repository/... ./internal/api/... 2>&1

Run tests for affected packages


ok  	github.com/ClusterCockpit/cc-backend/internal/repository	0.941s
ok  	github.com/ClusterCockpit/cc-backend/internal/api	7.189s


Done. `TransferCachedJobToMain` now wraps both the INSERT and DELETE in a single transaction (`tx.Beginx()` / `tx.Commit()` with `defer tx.Rollback()`). This ensures atomicity — if the DELETE fails or the process crashes, the INSERT is rolled back too — and reduces lock acquisitions from 2 to 1.


Update the Database config section in the README to reflect the new setting


     1→# NOTE
     2→
     3→While we do our best to keep the master branch in a usable state, there is no guarantee the master branch works.
     4→Please do not use it for production!
     5→
     6→Please have a look at the [Release
     7→Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
     8→for breaking changes!
     9→
    10→# ClusterCockpit REST and GraphQL API backend
    11→
    12→[![Build](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml)
    13→
    14→This is a Golang backend implementation for a REST and GraphQL API according to
    15→the [ClusterCockpit
    16→specifications](https://github.com/ClusterCockpit/cc-specifications). It also
    17→includes a web interface for ClusterCockpit. This implementation replaces the
    18→previous PHP Symfony based ClusterCockpit web interface. The reasons for
    19→switching from PHP Symfony to a Golang based solution are explained
    20→[here](https://github.com/ClusterCockpit/ClusterCockpit/wiki/Why-we-switched-from-PHP-Symfony-to-a-Golang-based-solution).
    21→
    22→## Overview
    23→
    24→This is a Golang web backend for the ClusterCockpit job-specific performance
    25→monitoring framework. It provides a REST API and an optional NATS-based messaging
    26→API for integrating ClusterCockpit with an HPC cluster batch system and external
    27→analysis scripts. Data exchange between the web front-end and the back-end is
    28→based on a GraphQL API. The web frontend is also served by the backend using
    29→[Svelte](https://svelte.dev/) components. Layout and styling are based on
    30→[Bootstrap 5](https://getbootstrap.com/) using
    31→[Bootstrap Icons](https://icons.getbootstrap.com/).
    32→
    33→The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database.
    34→While there are metric data backends for the InfluxDB and Prometheus time series
    35→databases, the only tested and supported setup is to use cc-metric-store as the
    36→metric data backend. Documentation on how to integrate ClusterCockpit with other
    37→time series databases will be added in the future.
    38→
    39→For real-time integration with HPC systems, the backend can subscribe to
    40→[NATS](https://nats.io/) subjects to receive job start/stop events and node
    41→state updates, providing an alternative to REST API polling.
    42→
    43→Completed batch jobs are stored in a file-based job archive according to
    44→[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
    45→The backend supports authentication via local accounts, an external LDAP
    46→directory, and JWT tokens. Authorization for APIs is implemented with
    47→[JWT](https://jwt.io/) tokens created with public/private key encryption.
    48→
    49→You find a detailed documentation on the [ClusterCockpit
    50→Webpage](https://clustercockpit.org).
    51→
    52→## Build requirements
    53→
    54→ClusterCockpit requires a current version of the golang toolchain and node.js.
    55→You can check `go.mod` to see what is the current minimal golang version needed.
    56→Homebrew and Archlinux usually have current golang versions. For other Linux
    57→distros this often means that you have to install the golang compiler yourself.
    58→Fortunately, this is easy with golang. Since much of the functionality is based
    59→on the Go standard library, it is crucial for security and performance to use a
    60→current version of golang. In addition, an old golang toolchain may limit the supported
    61→versions of third-party packages.
    62→
    63→## How to try ClusterCockpit with a demo setup
    64→
    65→We provide a shell script that downloads demo data and automatically starts the
    66→cc-backend. You will need `wget`, `go`, `node`, `npm` in your path to
    67→start the demo. The demo downloads 32MB of data (223MB on disk).
    68→
    69→```sh
    70→git clone https://github.com/ClusterCockpit/cc-backend.git
    71→cd ./cc-backend
    72→./startDemo.sh
    73→```
    74→
    75→You can also try the demo using the latest release binary.
    76→Create a folder and put the release binary `cc-backend` into this folder.
    77→Execute the following steps:
    78→
    79→```shell
    80→./cc-backend -init
    81→vim config.json (Add a second cluster entry and name the clusters alex and fritz)
    82→wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
    83→tar xf job-archive-demo.tar
    84→./cc-backend -init-db -add-user demo:admin:demo -loglevel info
    85→./cc-backend -server -dev -loglevel info
    86→```
    87→
    88→You can access the web interface at [http://localhost:8080](http://localhost:8080).
    89→Credentials for login are `demo:demo`.
    90→Please note that some views do not work without a metric backend (e.g., the
    91→Analysis, Systems and Status views).
    92→
    93→## How to build and run
    94→
    95→There is a Makefile to automate the build of cc-backend. The Makefile supports
    96→the following targets:
    97→
    98→- `make`: Initialize `var` directory and build svelte frontend and backend
    99→  binary. Note that there is no proper prerequisite handling. Any change of
   100→  frontend source files will result in a complete rebuild.
   101→- `make clean`: Clean go build cache and remove binary.
   102→- `make test`: Run the tests that are also run in the GitHub workflow setup.
   103→
   104→A common workflow for setting up cc-backend from scratch is:
   105→
   106→```sh
   107→git clone https://github.com/ClusterCockpit/cc-backend.git
   108→
   109→# Build binary
   110→cd ./cc-backend/
   111→make
   112→
   113→# EDIT THE .env FILE BEFORE YOU DEPLOY (Change the secrets)!
   114→# If authentication is disabled, it can be empty.
   115→cp configs/env-template.txt  .env
   116→vim .env
   117→
   118→cp configs/config.json .
   119→vim config.json
   120→
   121→#Optional: Link an existing job archive:
   122→ln -s <your-existing-job-archive> ./var/job-archive
   123→
   124→# This will first initialize the job.db database by traversing all
   125→# `meta.json` files in the job-archive and add a new user.
   126→./cc-backend -init-db -add-user <your-username>:admin:<your-password>
   127→
   128→# Start a HTTP server (HTTPS can be enabled in the configuration, the default port is 8080).
   129→# The --dev flag enables GraphQL Playground (http://localhost:8080/playground) and Swagger UI (http://localhost:8080/swagger).
   130→./cc-backend -server  -dev
   131→
   132→# Show other options:
   133→./cc-backend -help
   134→```
   135→
   136→## Database Configuration
   137→
   138→cc-backend uses SQLite as its database. For large installations, SQLite memory
   139→usage can be tuned via the optional `db-config` section in config.json under
   140→`main`:
   141→
   142→```json
   143→{
   144→  "main": {
   145→    "db": "./var/job.db",
   146→    "db-config": {
   147→      "cache-size-mb": 2048,
   148→      "soft-heap-limit-mb": 16384,
   149→      "max-open-connections": 4,
   150→      "max-idle-connections": 4,
   151→      "max-idle-time-minutes": 10
   152→    }
   153→  }
   154→}
   155→```
   156→
   157→All fields are optional. If `db-config` is omitted entirely, built-in defaults
   158→are used.
   159→
   160→### Options
   161→
   162→| Option                  | Default | Description                                                                                |
   163→| ----------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
   164→| `cache-size-mb`         | 2048    | SQLite page cache size per connection in MB. Maps to `PRAGMA cache_size`. Total cache memory is up to `cache-size-mb × max-open-connections`.                                           |
   165→| `soft-heap-limit-mb`    | 16384   | Process-wide SQLite soft heap limit in MB. SQLite will try to release cache pages to stay under this limit. Queries won't fail if exceeded, but cache eviction becomes more aggressive. |
   166→| `max-open-connections`  | 4       | Maximum number of open database connections.                                                                                |
   167→| `max-idle-connections`  | 4       | Maximum number of idle database connections kept in the pool.                                                                                |
   168→| `max-idle-time-minutes` | 10      | Maximum time in minutes a connection can sit idle before being closed.                                                                                |
   169→
   170→### Sizing Guidelines
   171→
   172→SQLite's `cache_size` is a **per-connection** setting — each connection
   173→maintains its own independent page cache. With multiple connections, the total
   174→memory available for caching is the sum across all connections.
   175→
   176→In practice, different connections tend to cache **different pages** (e.g., one
   177→handles a job listing query while another runs a statistics aggregation), so
   178→their caches naturally spread across the database. The formula
   179→`DB_size / max-open-connections` gives enough per-connection cache that the
   180→combined caches can cover the entire database.
   181→
   182→However, this is a best-case estimate. Connections running similar queries will
   183→cache the same pages redundantly. In the worst case (all connections caching
   184→identical pages), only `cache-size-mb` worth of unique data is cached rather
   185→than `cache-size-mb × max-open-connections`. For workloads with diverse
   186→concurrent queries, cache overlap is typically low.
   187→
   188→**Rules of thumb:**
   189→
   190→- **cache-size-mb**: Set to `DB_size_in_MB / max-open-connections` to allow the
   191→  entire database to be cached in memory. For example, an 80GB database with 8
   192→  connections needs at least 10240 MB (10GB) per connection. If your workload
   193→  has many similar concurrent queries, consider setting it higher to account for
   194→  cache overlap between connections.
   195→
   196→- **soft-heap-limit-mb**: Should be >= `cache-size-mb × max-open-connections` to
   197→  avoid cache thrashing. This is the total SQLite memory budget for the process.
   198→- On small installations the defaults work well. On servers with large databases
   199→  (tens of GB) and plenty of RAM, increasing these values significantly improves
   200→  query performance by reducing disk I/O.
   201→
   202→### Example: Large Server (512GB RAM, 80GB database)
   203→
   204→```json
   205→{
   206→  "main": {
   207→    "db-config": {
   208→      "cache-size-mb": 16384,
   209→      "soft-heap-limit-mb": 131072,
   210→      "max-open-connections": 8,
   211→      "max-idle-time-minutes": 30
   212→    }
   213→  }
   214→}
   215→```
   216→
   217→This allows the entire 80GB database to be cached (8 × 16GB = 128GB page cache)
   218→with a 128GB soft heap limit, using about 25% of available RAM.
   219→
   220→The effective configuration is logged at startup for verification.
   221→
   222→## Project file structure
   223→
   224→- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/master/.github)
   225→  GitHub Actions workflows and dependabot configuration for CI/CD.
   226→- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
   227→  contains the API schema files for the REST and GraphQL APIs. The REST API is
   228→  documented in the OpenAPI 3.0 format in
   229→  [./api/swagger.yaml](./api/swagger.yaml). The GraphQL schema is in
   230→  [./api/schema.graphqls](./api/schema.graphqls).
   231→- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
   232→  contains the main application entry point and CLI implementation.
   233→- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
   234→  contains documentation about configuration and command line options and required
   235→  environment variables. Sample configuration files are provided.
   236→- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
   237→  contains an example of setting up systemd for production use.
   238→- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
   239→  contains library source code that is not intended for use by others.
   240→  - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/api)
   241→    REST API handlers and NATS integration
   242→  - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/archiver)
   243→    Job archiving functionality
   244→  - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/auth)
   245→    Authentication (local, LDAP, OIDC) and JWT token handling
   246→  - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/config)
   247→    Configuration management and validation
   248→  - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/graph)
   249→    GraphQL schema and resolvers
   250→  - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer)
   251→    Job data import and database initialization
   252→  - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdispatch)
   253→    Dispatches metric data loading to appropriate backends
   254→  - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository)
   255→    Database repository layer for jobs and metadata
   256→  - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/routerConfig)
   257→    HTTP router configuration and middleware
   258→  - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/tagger)
   259→    Job classification and application detection
   260→  - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager)
   261→    Background task management and scheduled jobs
   262→  - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstoreclient)
   263→    Client for cc-metric-store queries
   264→- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
   265→  contains Go packages that can be used by other projects.
   266→  - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive)
   267→    Job archive backend implementations (filesystem, S3, SQLite)
   268→  - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/metricstore)
   269→    In-memory metric data store with checkpointing and metric loading
   270→- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
   271→  Additional command line helper tools.
   272→  - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
   273→    Commands for getting infos about an existing job archive, importing jobs
   274→    between archive backends, and converting archives between JSON and Parquet formats.
   275→  - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration)
   276→    Tool for migrating job archives between formats.
   277→  - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
   278→    Tool to convert external pubkey for use in `cc-backend`.
   279→  - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
   280→    contains a small application to generate a compatible JWT keypair. You find
   281→    documentation on how to use it
   282→    [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
   283→- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
   284→  Server-side templates and frontend-related files:
   285→  - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
   286→    Svelte components and static assets for the frontend UI
   287→  - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
   288→    Server-side Go templates, including monitoring views
   289→- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
   290→  Configures the behaviour and generation of
   291→  [gqlgen](https://github.com/99designs/gqlgen).
   292→- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
   293→  is a shell script that sets up demo data, and builds and starts `cc-backend`.
   294→

# NOTE

While we do our best to keep the master branch in a usable state, there is no guarantee the master branch works.
Please do not use it for production!

Please have a look at the [Release
Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
for breaking changes!

# ClusterCockpit REST and GraphQL API backend

[![Build](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml)

This is a Golang backend implementation for a REST and GraphQL API according to
the [ClusterCockpit
specifications](https://github.com/ClusterCockpit/cc-specifications). It also
includes a web interface for ClusterCockpit. This implementation replaces the
previous PHP Symfony based ClusterCockpit web interface. The reasons for
switching from PHP Symfony to a Golang based solution are explained
[here](https://github.com/ClusterCockpit/ClusterCockpit/wiki/Why-we-switched-from-PHP-Symfony-to-a-Golang-based-solution).

## Overview

This is a Golang web backend for the ClusterCockpit job-specific performance
monitoring framework. It provides a REST API and an optional NATS-based messaging
API for integrating ClusterCockpit with an HPC cluster batch system and external
analysis scripts. Data exchange between the web front-end and the back-end is
based on a GraphQL API. The web frontend is also served by the backend using
[Svelte](https://svelte.dev/) components. Layout and styling are based on
[Bootstrap 5](https://getbootstrap.com/) using
[Bootstrap Icons](https://icons.getbootstrap.com/).

The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database.
While there are metric data backends for the InfluxDB and Prometheus time series
databases, the only tested and supported setup is to use cc-metric-store as the
metric data backend. Documentation on how to integrate ClusterCockpit with other
time series databases will be added in the future.

For real-time integration with HPC systems, the backend can subscribe to
[NATS](https://nats.io/) subjects to receive job start/stop events and node
state updates, providing an alternative to REST API polling.

Completed batch jobs are stored in a file-based job archive according to
[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
The backend supports authentication via local accounts, an external LDAP
directory, and JWT tokens. Authorization for APIs is implemented with
[JWT](https://jwt.io/) tokens created with public/private key encryption.

You find a detailed documentation on the [ClusterCockpit
Webpage](https://clustercockpit.org).

## Build requirements

ClusterCockpit requires a current version of the golang toolchain and node.js.
You can check `go.mod` to see what is the current minimal golang version needed.
Homebrew and Archlinux usually have current golang versions. For other Linux
distros this often means that you have to install the golang compiler yourself.
Fortunately, this is easy with golang. Since much of the functionality is based
on the Go standard library, it is crucial for security and performance to use a
current version of golang. In addition, an old golang toolchain may limit the supported
versions of third-party packages.

## How to try ClusterCockpit with a demo setup

We provide a shell script that downloads demo data and automatically starts the
cc-backend. You will need `wget`, `go`, `node`, `npm` in your path to
start the demo. The demo downloads 32MB of data (223MB on disk).

```sh
git clone https://github.com/ClusterCockpit/cc-backend.git
cd ./cc-backend
./startDemo.sh
```

You can also try the demo using the latest release binary.
Create a folder and put the release binary `cc-backend` into this folder.
Execute the following steps:

```shell
./cc-backend -init
vim config.json (Add a second cluster entry and name the clusters alex and fritz)
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
tar xf job-archive-demo.tar
./cc-backend -init-db -add-user demo:admin:demo -loglevel info
./cc-backend -server -dev -loglevel info
```

You can access the web interface at [http://localhost:8080](http://localhost:8080).
Credentials for login are `demo:demo`.
Please note that some views do not work without a metric backend (e.g., the
Analysis, Systems and Status views).

## How to build and run

There is a Makefile to automate the build of cc-backend. The Makefile supports
the following targets:

- `make`: Initialize `var` directory and build svelte frontend and backend
  binary. Note that there is no proper prerequisite handling. Any change of
  frontend source files will result in a complete rebuild.
- `make clean`: Clean go build cache and remove binary.
- `make test`: Run the tests that are also run in the GitHub workflow setup.

A common workflow for setting up cc-backend from scratch is:

```sh
git clone https://github.com/ClusterCockpit/cc-backend.git

# Build binary
cd ./cc-backend/
make

# EDIT THE .env FILE BEFORE YOU DEPLOY (Change the secrets)!
# If authentication is disabled, it can be empty.
cp configs/env-template.txt  .env
vim .env

cp configs/config.json .
vim config.json

#Optional: Link an existing job archive:
ln -s <your-existing-job-archive> ./var/job-archive

# This will first initialize the job.db database by traversing all
# `meta.json` files in the job-archive and add a new user.
./cc-backend -init-db -add-user <your-username>:admin:<your-password>

# Start a HTTP server (HTTPS can be enabled in the configuration, the default port is 8080).
# The --dev flag enables GraphQL Playground (http://localhost:8080/playground) and Swagger UI (http://localhost:8080/swagger).
./cc-backend -server  -dev

# Show other options:
./cc-backend -help
```

## Database Configuration

cc-backend uses SQLite as its database. For large installations, SQLite memory
usage can be tuned via the optional `db-config` section in config.json under
`main`:

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 4,
      "max-idle-connections": 4,
      "max-idle-time-minutes": 10
    }
  }
}
```

All fields are optional. If `db-config` is omitted entirely, built-in defaults
are used.

### Options

| Option                  | Default | Description                                                                                |
| ----------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `cache-size-mb`         | 2048    | SQLite page cache size per connection in MB. Maps to `PRAGMA cache_size`. Total cache memory is up to `cache-size-mb × max-open-connections`.                                           |
| `soft-heap-limit-mb`    | 16384   | Process-wide SQLite soft heap limit in MB. SQLite will try to release cache pages to stay under this limit. Queries won't fail if exceeded, but cache eviction becomes more aggressive. |
| `max-open-connections`  | 4       | Maximum number of open database connections.                                                                                |
| `max-idle-connections`  | 4       | Maximum number of idle database connections kept in the pool.                                                                                |
| `max-idle-time-minutes` | 10      | Maximum time in minutes a connection can sit idle before being closed.                                                                                |

### Sizing Guidelines

SQLite's `cache_size` is a **per-connection** setting — each connection
maintains its own independent page cache. With multiple connections, the total
memory available for caching is the sum across all connections.

In practice, different connections tend to cache **different pages** (e.g., one
handles a job listing query while another runs a statistics aggregation), so
their caches naturally spread across the database. The formula
`DB_size / max-open-connections` gives enough per-connection cache that the
combined caches can cover the entire database.

However, this is a best-case estimate. Connections running similar queries will
cache the same pages redundantly. In the worst case (all connections caching
identical pages), only `cache-size-mb` worth of unique data is cached rather
than `cache-size-mb × max-open-connections`. For workloads with diverse
concurrent queries, cache overlap is typically low.

**Rules of thumb:**

- **cache-size-mb**: Set to `DB_size_in_MB / max-open-connections` to allow the
  entire database to be cached in memory. For example, an 80GB database with 8
  connections needs at least 10240 MB (10GB) per connection. If your workload
  has many similar concurrent queries, consider setting it higher to account for
  cache overlap between connections.

- **soft-heap-limit-mb**: Should be >= `cache-size-mb × max-open-connections` to
  avoid cache thrashing. This is the total SQLite memory budget for the process.
- On small installations the defaults work well. On servers with large databases
  (tens of GB) and plenty of RAM, increasing these values significantly improves
  query performance by reducing disk I/O.

### Example: Large Server (512GB RAM, 80GB database)

```json
{
  "main": {
    "db-config": {
      "cache-size-mb": 16384,
      "soft-heap-limit-mb": 131072,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This allows the entire 80GB database to be cached (8 × 16GB = 128GB page cache)
with a 128GB soft heap limit, using about 25% of available RAM.

The effective configuration is logged at startup for verification.

## Project file structure

- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/master/.github)
  GitHub Actions workflows and dependabot configuration for CI/CD.
- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
  contains the API schema files for the REST and GraphQL APIs. The REST API is
  documented in the OpenAPI 3.0 format in
  [./api/swagger.yaml](./api/swagger.yaml). The GraphQL schema is in
  [./api/schema.graphqls](./api/schema.graphqls).
- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
  contains the main application entry point and CLI implementation.
- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
  contains documentation about configuration and command line options and required
  environment variables. Sample configuration files are provided.
- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
  contains an example of setting up systemd for production use.
- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
  contains library source code that is not intended for use by others.
  - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/api)
    REST API handlers and NATS integration
  - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/archiver)
    Job archiving functionality
  - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/auth)
    Authentication (local, LDAP, OIDC) and JWT token handling
  - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/config)
    Configuration management and validation
  - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/graph)
    GraphQL schema and resolvers
  - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer)
    Job data import and database initialization
  - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdispatch)
    Dispatches metric data loading to appropriate backends
  - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository)
    Database repository layer for jobs and metadata
  - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/routerConfig)
    HTTP router configuration and middleware
  - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/tagger)
    Job classification and application detection
  - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager)
    Background task management and scheduled jobs
  - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstoreclient)
    Client for cc-metric-store queries
- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
  contains Go packages that can be used by other projects.
  - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive)
    Job archive backend implementations (filesystem, S3, SQLite)
  - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/metricstore)
    In-memory metric data store with checkpointing and metric loading
- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
  Additional command line helper tools.
  - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
    Commands for getting infos about an existing job archive, importing jobs
    between archive backends, and converting archives between JSON and Parquet formats.
  - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration)
    Tool for migrating job archives between formats.
  - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
    Tool to convert external pubkey for use in `cc-backend`.
  - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
    contains a small application to generate a compatible JWT keypair. You find
    documentation on how to use it
    [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
  Server-side templates and frontend-related files:
  - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
    Svelte components and static assets for the frontend UI
  - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
    Server-side Go templates, including monitoring views
- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
  Configures the behaviour and generation of
  [gqlgen](https://github.com/99designs/gqlgen).
- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
  is a shell script that sets up demo data, and builds and starts `cc-backend`.


I can see the Database Configuration section at lines 136-221. I need to add the `busy-timeout-ms` option.


Edit

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 4,
      "max-idle-connections": 4,
      "max-idle-time-minutes": 10
    }
  }
}
```

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 4,
      "max-idle-connections": 4,
      "max-idle-time-minutes": 10,
      "busy-timeout-ms": 60000
    }
  }
}
```