internal/repository/job.go:814:func (r *JobRepository) UpdateDuration() error {
internal/repository/job.go-815-	stmnt := sq.Update("job").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-816-		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-817-		Where("job_state = 'running'")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-818-
internal/repository/job.go-819-	_, err := stmnt.RunWith(r.stmtCache).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-820-	if err != nil {
internal/repository/job.go-821-		cclog.Errorf("Error while updating duration for running jobs: %v", err)
internal/repository/job.go-822-		return fmt.Errorf("failed to update duration for running jobs: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-823-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-824-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-825-	return nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-826-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-827-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-828-// FindJobsBetween returns jobs within a specified time range.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-829-// If startTimeBegin is 0, returns all jobs before startTimeEnd.


internal/repository/job.go:399:func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged string) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-400-	var cnt int
internal/repository/job.go-401-	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-402-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-403-	switch omitTagged {
internal/repository/job.go-404-	case "all":
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-405-		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
internal/repository/job.go-406-	case "user":
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-407-		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-408-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-409-
internal/repository/job.go-410-	if err := q.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
internal/repository/job.go-411-		cclog.Errorf("Error counting jobs before %d: %v", startTime, err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-412-		return 0, err
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-413-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-414-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-415-	// Invalidate cache for jobs being deleted (get job IDs first)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-416-	if cnt > 0 {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-417-		var jobIds []int64
internal/repository/job.go-418-		selectQuery := sq.Select("id").From("job").Where("job.start_time < ?", startTime)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-419-


internal/repository/job.go:712:func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
internal/repository/job.go-713-	start := time.Now()
internal/repository/job.go-714-	currentTime := time.Now().Unix()
internal/repository/job.go-715-	res, err := sq.Update("job").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-716-		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-717-		Set("duration", 0).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-718-		Set("job_state", schema.JobStateFailed).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-719-		Where("job.job_state = 'running'").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-720-		Where("job.walltime > 0").
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-721-		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-722-		RunWith(r.DB).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-723-	if err != nil {
internal/repository/job.go-724-		cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err)
internal/repository/job.go-725-		return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-726-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-727-
internal/repository/job.go-728-	rowsAffected, err := res.RowsAffected()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-729-	if err != nil {
internal/repository/job.go-730-		cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err)
internal/repository/job.go-731-		return fmt.Errorf("failed to get rows affected count: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-732-	}


Good — `UpdateDuration`, `StopJobsExceedingWalltimeBy`, and `DeleteJobsBefore` are all single bulk SQL statements (no loops), so they're already efficient. The footprint worker already uses transactions properly.

Now I have a complete picture. Let me write the plan.


Write

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically


The file /Users/jan/.claude/plans/woolly-wandering-narwhal.md has been updated successfully.

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically

-# Fix: SQLite Write Contention Causing OOM Crash

+# Additional SQLite Write Contention Reductions

-The server crashes daily around 15:00-16:00 with OOM. Logs show "database is locked" errors and LDAP sync failures (`UNIQUE constraint failed: hpc_user.username`) shortly before the crash.

+Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

-**Root cause:** A burst of concurrent DB writers (NATS job start/stop, node state updates, LDAP sync, scheduled tasks) overwhelm SQLite's 5-second busy_timeout. Failed operations and blocked goroutines accumulate, consuming memory until OOM.

+After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:

+- `UpdateDuration` — single bulk UPDATE, efficient

+- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient

+- `DeleteJobsBefore` — single bulk DELETE, efficient

+- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster

+- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

-The cascade: burst of job events → DB contention → `SQLITE_BUSY` after 5s → error paths spawn retries/new goroutines → goroutine accumulation → OOM.

+The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

-## Fix 1: Increase busy_timeout (contention relief)

+## Fix 6: Wrap TransferCachedJobToMain in a Transaction

-Change `_busy_timeout` from `5000` to `60000` (60 seconds). Also make it configurable:

+Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

-**File:** `internal/repository/config.go` — add `BusyTimeoutMs int` field, default 60000.

+**Change:** Wrap both operations in a transaction:

+func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {

+    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?

+    // DELETE FROM job_cache WHERE id = ?

-**File:** `internal/repository/dbConnection.go:73` — use `repoConfig.BusyTimeoutMs` instead of hardcoded value.

+**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

-**File:** `pkg/archive/sqliteBackend.go:105` — also change hardcoded `busy_timeout=5000` to `busy_timeout=60000`.

-With 60s timeout, SQLite retries internally using its backoff mechanism rather than failing fast. This prevents the "database is locked" error cascade.

-## Fix 2: Batch Node State Updates in a Transaction

+## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

-Currently each node state update is a separate SELECT + INSERT (2 DB round-trips per node). A message with 1000 nodes = 2000+ individual DB operations, each acquiring/releasing the write lock.

+**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag

+**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

-**File:** `internal/repository/node.go` — add `BatchUpdateNodeStates` method:

-- Accept a slice of `(hostname, cluster, nodeState)` tuples

+Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

+**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:

-- For each node: lookup ID (or insert node), then insert state — all within the same transaction

+- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction

+- Single archive.UpdateTags call at the end (outside transaction)

-**File:** `internal/api/nats.go:348-364` (`processNodestateEvent`) — build slice of updates, call `BatchUpdateNodeStates` instead of per-node `UpdateNodeState`.

+- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`

+- `internal/api/job.go` (startJob REST handler) — same change

-**File:** `internal/api/node.go:119-142` (`updateNodeStates`) — same change for the REST path. Note: the REST handler also does health checks per node before DB inserts, so batch only the DB part (the loop at lines 119-142).

+**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

-Keep the existing `UpdateNodeState` method for single-node use cases.

+**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

-The LDAP sync fails when a user is concurrently created by a login. The error is benign but causes `Sync()` to return early (line 215: `return err`), abandoning remaining users.

+**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

-**File:** `internal/repository/user.go` — add `AddUserIfNotExists` method using `INSERT OR IGNORE`:

-func (r *UserRepository) AddUserIfNotExists(user *schema.User) error {

-    rolesJson, _ := json.Marshal(user.Roles)

-    projectsJson, _ := json.Marshal(user.Projects)

-    cols := "username, name, roles, projects, ldap"

-        `INSERT OR IGNORE INTO hpc_user (` + cols + `) VALUES (?, ?, ?, ?, ?)`,

-        user.Username, user.Name, string(rolesJson), string(projectsJson), int(user.AuthSource))

+Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

-**File:** `internal/auth/ldap.go:213` — change `ur.AddUser(user)` to `ur.AddUserIfNotExists(user)`.

+**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:

+- Accept jobID + slice of (type, name, scope)

+- For each tag: resolve/create tag ID + INSERT jobtag

-Note: `AddUserIfNotExists` intentionally skips the deprecated default metrics config insertion that `AddUser` does — LDAP-synced users don't need it, and it adds unnecessary DB writes.

+Update `handleImport.go` to call `BatchImportTags` instead of the loop.

+**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

-## Fix 4: Add Semaphore for NATS Handlers

+## Fix 9: Wrap RemoveTagByID in a Transaction

-Without rate limiting, blocked NATS handlers cause unbounded goroutine accumulation.

-**File:** `internal/api/nats.go` — add semaphores to `NatsAPI` struct:

+Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

-    JobRepository   *repository.JobRepository

+func (r *JobRepository) RemoveTagByID(tagID int64) error {

+    jobIds, err := r.FindJobIdsByTag(tagID)

+    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)

+    // DELETE FROM tag WHERE id = ?  (on tx)

+    go r.removeTagFromArchiveJobs(jobIds)

-Initialize in `NewNatsAPI`: `jobSem: make(chan struct{}, 8)`, `nodeSem: make(chan struct{}, 2)`.

+**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

-Add acquire/release in `handleJobEvent` and `handleNodeState`:

-This caps concurrent NATS processing at 8 job events + 2 node state updates, preventing goroutine pile-up.

-## Fix 5: Wrap SyncJobs in a Transaction

+1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug

+2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path

+3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists

+4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

-Currently does SELECT + INSERT...SELECT + DELETE as 3 separate operations. Wrap in a DB transaction:

-- Do the read-only ID resolution outside the transaction

-This reduces lock acquisitions from 3 to 1 and ensures atomicity.

+| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |

+| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |

+| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |

+| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |

+| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

-1. **Fix 1** (busy_timeout) — quick config change, biggest immediate impact

-2. **Fix 3** (LDAP sync) — eliminates error spam, trivial change

-3. **Fix 2** (batch node states) — biggest contention reduction

-4. **Fix 5** (SyncJobs transaction) — safety + minor contention reduction

-5. **Fix 4** (NATS semaphore) — defense in depth against goroutine accumulation

+These are already efficient and don't need modification:

+- `UpdateDuration` — single bulk UPDATE

+- `StopJobsExceedingWalltimeBy` — single bulk UPDATE

+- `DeleteJobsBefore` — single bulk DELETE

+- Footprint worker — already transactional per cluster

+- Archiver — already chains SETs into single UPDATE via Squirrel builder

+- `UpdateUser` — admin-only, very low frequency, not worth the complexity

+- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

-1. After Fix 1: Check logs for "database is locked" — should be gone or very rare.

-2. After Fix 2: check `Timer updateNodeStates, SQLite Inserts` log line — should be much faster.

-3. After Fix 3: check logs for "UNIQUE constraint failed" — should be gone.

-4. Run `go test ./internal/repository/... ./internal/api/... ./internal/auth/...` after all changes.

-5. Load test: simulate burst of 100+ concurrent job start/stop events + node state update with 500 nodes + LDAP sync running simultaneously. Monitor goroutine count and memory.

+1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`

+2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty

+3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)

+4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically

# Fix: SQLite Write Contention Causing OOM Crash

## Context

The server crashes daily around 15:00-16:00 with OOM. Logs show "database is locked" errors and LDAP sync failures (`UNIQUE constraint failed: hpc_user.username`) shortly before the crash.

**Root cause:** A burst of concurrent DB writers (NATS job start/stop, node state updates, LDAP sync, scheduled tasks) overwhelm SQLite's 5-second busy_timeout. Failed operations and blocked goroutines accumulate, consuming memory until OOM.

The cascade: burst of job events → DB contention → `SQLITE_BUSY` after 5s → error paths spawn retries/new goroutines → goroutine accumulation → OOM.

---

## Fix 1: Increase busy_timeout (contention relief)

**File:** `internal/repository/dbConnection.go:73`

Change `_busy_timeout` from `5000` to `60000` (60 seconds). Also make it configurable:

**File:** `internal/repository/config.go` — add `BusyTimeoutMs int` field, default 60000.

**File:** `internal/repository/dbConnection.go:73` — use `repoConfig.BusyTimeoutMs` instead of hardcoded value.

**File:** `pkg/archive/sqliteBackend.go:105` — also change hardcoded `busy_timeout=5000` to `busy_timeout=60000`.

With 60s timeout, SQLite retries internally using its backoff mechanism rather than failing fast. This prevents the "database is locked" error cascade.

---

## Fix 2: Batch Node State Updates in a Transaction

Currently each node state update is a separate SELECT + INSERT (2 DB round-trips per node). A message with 1000 nodes = 2000+ individual DB operations, each acquiring/releasing the write lock.

**File:** `internal/repository/node.go` — add `BatchUpdateNodeStates` method:
- Accept a slice of `(hostname, cluster, nodeState)` tuples
- Begin transaction
- For each node: lookup ID (or insert node), then insert state — all within the same transaction
- Commit once

**File:** `internal/api/nats.go:348-364` (`processNodestateEvent`) — build slice of updates, call `BatchUpdateNodeStates` instead of per-node `UpdateNodeState`.

**File:** `internal/api/node.go:119-142` (`updateNodeStates`) — same change for the REST path. Note: the REST handler also does health checks per node before DB inserts, so batch only the DB part (the loop at lines 119-142).

Keep the existing `UpdateNodeState` method for single-node use cases.

---

## Fix 3: Fix LDAP Sync Race Condition

The LDAP sync fails when a user is concurrently created by a login. The error is benign but causes `Sync()` to return early (line 215: `return err`), abandoning remaining users.

**File:** `internal/repository/user.go` — add `AddUserIfNotExists` method using `INSERT OR IGNORE`:
```go
func (r *UserRepository) AddUserIfNotExists(user *schema.User) error {
    rolesJson, _ := json.Marshal(user.Roles)
    projectsJson, _ := json.Marshal(user.Projects)
    cols := "username, name, roles, projects, ldap"
    _, err := r.DB.Exec(
        `INSERT OR IGNORE INTO hpc_user (` + cols + `) VALUES (?, ?, ?, ?, ?)`,
        user.Username, user.Name, string(rolesJson), string(projectsJson), int(user.AuthSource))
    return err
}
```

**File:** `internal/auth/ldap.go:213` — change `ur.AddUser(user)` to `ur.AddUserIfNotExists(user)`.

Note: `AddUserIfNotExists` intentionally skips the deprecated default metrics config insertion that `AddUser` does — LDAP-synced users don't need it, and it adds unnecessary DB writes.

---

## Fix 4: Add Semaphore for NATS Handlers

Without rate limiting, blocked NATS handlers cause unbounded goroutine accumulation.

**File:** `internal/api/nats.go` — add semaphores to `NatsAPI` struct:
```go
type NatsAPI struct {
    JobRepository   *repository.JobRepository
    RepositoryMutex sync.Mutex
    jobSem          chan struct{}
    nodeSem         chan struct{}
}
```

Initialize in `NewNatsAPI`: `jobSem: make(chan struct{}, 8)`, `nodeSem: make(chan struct{}, 2)`.

Add acquire/release in `handleJobEvent` and `handleNodeState`:
```go
api.jobSem <- struct{}{}
defer func() { <-api.jobSem }()
```

This caps concurrent NATS processing at 8 job events + 2 node state updates, preventing goroutine pile-up.

---

## Fix 5: Wrap SyncJobs in a Transaction

**File:** `internal/repository/jobCreate.go:72-126` (`SyncJobs`)

Currently does SELECT + INSERT...SELECT + DELETE as 3 separate operations. Wrap in a DB transaction:
- `tx, err := r.DB.Beginx()`
- Run all three operations on `tx`
- `tx.Commit()`
- Do the read-only ID resolution outside the transaction

This reduces lock acquisitions from 3 to 1 and ensures atomicity.

---

## Implementation Order

1. **Fix 1** (busy_timeout) — quick config change, biggest immediate impact
2. **Fix 3** (LDAP sync) — eliminates error spam, trivial change
3. **Fix 2** (batch node states) — biggest contention reduction
4. **Fix 5** (SyncJobs transaction) — safety + minor contention reduction
5. **Fix 4** (NATS semaphore) — defense in depth against goroutine accumulation

---

## Verification

1. After Fix 1: Check logs for "database is locked" — should be gone or very rare.
2. After Fix 2: check `Timer updateNodeStates, SQLite Inserts` log line — should be much faster.
3. After Fix 3: check logs for "UNIQUE constraint failed" — should be gone.
4. Run `go test ./internal/repository/... ./internal/api/... ./internal/auth/...` after all changes.
5. Load test: simulate burst of 100+ concurrent job start/stop events + node state update with 500 nodes + LDAP sync running simultaneously. Monitor goroutine count and memory.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


After a thorough review of all DB write paths in the codebase, I found 4 additional opportunities. Most periodic tasks (UpdateDuration, footprint worker, archiver, bulk deletes) are already well-optimized. The remaining wins are:

1. **Fix 6 — TransferCachedJobToMain** (atomicity bug): INSERT + DELETE are two separate calls without a transaction
2. **Fix 7 — Batch tag operations on job start**: Each tag does ~5 DB round-trips; a job with 5 tags = 30 operations that could be 1 transaction
3. **Fix 8 — Batch tag imports**: Same pattern during `--import`, loops ImportTag per tag
4. **Fix 9 — RemoveTagByID transaction**: Two separate DELETEs (jobtag + tag) without a transaction

Fix 7 is the most impactful for runtime contention since it's on the NATS job start hot path.


ExitPlanMode

Prompt: run go build and go test

# Additional SQLite Write Contention Reductions

## Context

Fixes 1-5 (busy_timeout, batch node states, LDAP sync, NATS semaphores, SyncJobs transaction) are already implemented. This plan identifies **remaining** opportunities to reduce DB write pressure through transactions and batching.

After reviewing all write paths in the codebase, most periodic tasks are already well-optimized:
- `UpdateDuration` — single bulk UPDATE, efficient
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE, efficient
- `DeleteJobsBefore` — single bulk DELETE, efficient
- Footprint worker — already uses `TransactionInit/TransactionAdd/TransactionEnd` per cluster
- Archiver — chains UpdateFootprint + UpdateEnergy + MarkArchived into single `sq.Update` statement, efficient

The remaining opportunities are in **tag operations**, **TransferCachedJobToMain**, and **job imports**.

---

## Fix 6: Wrap TransferCachedJobToMain in a Transaction

**File:** `internal/repository/jobCreate.go:130-149`

Currently does INSERT + DELETE as two separate `r.DB.Exec()` calls. If the process crashes between them, the job_cache row is orphaned (duplicate on next SyncJobs).

**Change:** Wrap both operations in a transaction:
```go
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
    tx, err := r.DB.Beginx()
    // INSERT INTO job ... SELECT ... FROM job_cache WHERE id = ?
    // newID from LastInsertId
    // DELETE FROM job_cache WHERE id = ?
    // tx.Commit()
}
```

**Impact:** Atomicity + 1 lock acquisition instead of 2. Called per job stop from cache path.

---

## Fix 7: Batch Tag Operations on Job Start (NATS + REST)

**File:** `internal/api/nats.go:228-233` — loops `AddTagOrCreate` per tag
**File:** `internal/api/job.go` — same pattern in `startJob` REST handler

Currently each tag does: SELECT tag → INSERT tag (if new) → SELECT job → INSERT jobtag → SELECT tags → archive.UpdateTags. For a job with 5 tags = 30 individual DB operations.

**Change:** Add `BatchAddTagsDirect` to `internal/repository/tags.go`:
- Accept jobID + slice of tags
- Begin transaction
- For each tag: resolve/create tag ID, INSERT jobtag — all within same transaction
- Commit
- Single archive.UpdateTags call at the end (outside transaction)

Then update callers:
- `internal/api/nats.go:228-233` — replace loop with `BatchAddTagsDirect`
- `internal/api/job.go` (startJob REST handler) — same change

**Keep** existing `AddTagOrCreate` for single-tag use cases (GraphQL, REST tag endpoint).

**Impact:** N tags = 1 transaction + 1 archive update instead of N * 5 separate operations. Most impactful when scheduler sends jobs with multiple tags.

---

## Fix 8: Batch Tag Imports

**File:** `internal/importer/handleImport.go:111-116` — loops `ImportTag` per tag per job

Currently each tag does: SELECT tag → INSERT tag (if new) → INSERT jobtag. For 100 jobs with 3 tags each = 600+ individual operations.

**Change:** Add `BatchImportTags` to `internal/repository/tags.go`:
- Accept jobID + slice of (type, name, scope)
- Begin transaction
- For each tag: resolve/create tag ID + INSERT jobtag
- Commit

Update `handleImport.go` to call `BatchImportTags` instead of the loop.

**Impact:** Reduces lock acquisitions from 2*N to 1 per job's tags during import. Import is a startup-only operation but can be long-running for large archives.

---

## Fix 9: Wrap RemoveTagByID in a Transaction

**File:** `internal/repository/tags.go:216-244`

Currently does DELETE FROM jobtag + DELETE FROM tag as two separate operations. If the second fails, jobtag rows are gone but the orphaned tag row remains.

**Change:**
```go
func (r *JobRepository) RemoveTagByID(tagID int64) error {
    jobIds, err := r.FindJobIdsByTag(tagID)
    // ...
    tx, err := r.DB.Beginx()
    defer tx.Rollback()
    // DELETE FROM jobtag WHERE tag_id = ?  (on tx)
    // DELETE FROM tag WHERE id = ?  (on tx)
    tx.Commit()
    go r.removeTagFromArchiveJobs(jobIds)
}
```

**Impact:** Atomicity for tag deletion. Low frequency (admin operation) but prevents orphaned data.

---

## Implementation Order

1. **Fix 6** (TransferCachedJobToMain) — trivial, fixes a real atomicity bug
2. **Fix 7** (batch tags on job start) — biggest remaining contention reduction for NATS path
3. **Fix 8** (batch tag imports) — only affects import, but easy once Fix 7 pattern exists
4. **Fix 9** (RemoveTagByID transaction) — minor, correctness improvement

---

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/jobCreate.go` | Fix 6: wrap TransferCachedJobToMain in transaction |
| `internal/repository/tags.go` | Fix 7: add `BatchAddTagsDirect`; Fix 8: add `BatchImportTags`; Fix 9: transaction in RemoveTagByID |
| `internal/api/nats.go` | Fix 7: use BatchAddTagsDirect in handleStartJob |
| `internal/api/job.go` | Fix 7: use BatchAddTagsDirect in startJob REST handler |
| `internal/importer/handleImport.go` | Fix 8: use BatchImportTags |

---

## What NOT to Change

These are already efficient and don't need modification:
- `UpdateDuration` — single bulk UPDATE
- `StopJobsExceedingWalltimeBy` — single bulk UPDATE
- `DeleteJobsBefore` — single bulk DELETE
- Footprint worker — already transactional per cluster
- Archiver — already chains SETs into single UPDATE via Squirrel builder
- `UpdateUser` — admin-only, very low frequency, not worth the complexity
- `AddUser` deprecated metrics loop — marked DEPRECATED, will be removed; not worth optimizing

---

## Verification

1. `go test ./internal/repository/... ./internal/api/... ./internal/importer/...`
2. After Fix 6: verify `TransferCachedJobToMain` atomicity — stop a job that's in cache, check job table has correct ID and cache is empty
3. After Fix 7: start a job with multiple tags via NATS — verify all tags appear, check that only 1 transaction was used (enable debug logging)
4. After Fix 9: delete a tag via GraphQL — verify both tag and jobtag rows removed atomically


Error: The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). To tell you how to proceed, the user said:
Only implement fix 6


Implementing Fix 6 — wrapping `TransferCachedJobToMain` in a transaction.


   128→			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
   129→				job.JobID, job.Cluster, job.StartTime).
   130→			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
   131→			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
   132→				job.JobID, job.Cluster, err)
   133→			continue
   134→		}
   135→		job.ID = &newID
   136→	}
   137→
   138→	return jobs, nil
   139→}
   140→
   141→// TransferCachedJobToMain moves a job from job_cache to the job table.
   142→// Caller must hold r.Mutex. Returns the new job table ID.
   143→func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
   144→	res, err := r.DB.Exec(
   145→		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
   146→		cacheID)
   147→	if err != nil {
   148→		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
   149→	}
   150→
   151→	newID, err := res.LastInsertId()
   152→	if err != nil {
   153→		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
   154→	}
   155→
   156→	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
   157→	if err != nil {

			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
				job.JobID, job.Cluster, job.StartTime).
			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
				job.JobID, job.Cluster, err)
			continue
		}
		job.ID = &newID
	}

	return jobs, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {


Edit

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	return newID, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	tx, err := r.DB.Beginx()
	if err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)
	}
	defer tx.Rollback()

	res, err := tx.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	if err := tx.Commit(); err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)
	}

	return newID, nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobCreate.go has been updated successfully.

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	tx, err := r.DB.Beginx()
	if err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)
	}
	defer tx.Rollback()

	res, err := tx.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	if err := tx.Commit(); err != nil {
		return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)
	}

	return newID, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"encoding/json"
	"fmt"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
)

const NamedJobCacheInsert string = `INSERT INTO job_cache (
	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
) VALUES (
	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
);`

const NamedJobInsert string = `INSERT INTO job (
	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
) VALUES (
	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
);`

// InsertJobDirect inserts a job directly into the job table (not job_cache).
// Use this when the returned ID will be used for operations on the job table
// (e.g., adding tags), or for imported jobs that are already completed.
func (r *JobRepository) InsertJobDirect(job *schema.Job) (int64, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	res, err := r.DB.NamedExec(NamedJobInsert, job)
	if err != nil {
		cclog.Warn("Error while NamedJobInsert (direct)")
		return 0, err
	}
	id, err := res.LastInsertId()
	if err != nil {
		cclog.Warn("Error while getting last insert ID (direct)")
		return 0, err
	}

	return id, nil
}

func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	res, err := r.DB.NamedExec(NamedJobCacheInsert, job)
	if err != nil {
		cclog.Warn("Error while NamedJobInsert")
		return 0, err
	}
	id, err := res.LastInsertId()
	if err != nil {
		cclog.Warn("Error while getting last insert ID")
		return 0, err
	}

	return id, nil
}

func (r *JobRepository) SyncJobs() ([]*schema.Job, error) {
	r.Mutex.Lock()
	defer r.Mutex.Unlock()

	query := sq.Select(jobCacheColumns...).From("job_cache")

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running query %v", err)
		return nil, err
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, 50)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warn("Error while scanning rows")
			return nil, err
		}
		jobs = append(jobs, job)
	}

	// Transfer cached jobs to main table and clear cache in a single transaction.
	tx, err := r.DB.Beginx()
	if err != nil {
		cclog.Errorf("SyncJobs: begin transaction: %v", err)
		return nil, err
	}
	defer tx.Rollback()

	// Use INSERT OR IGNORE to skip jobs already transferred by the stop path
	_, err = tx.Exec(
		"INSERT OR IGNORE INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache")
	if err != nil {
		cclog.Errorf("Error while Job sync: %v", err)
		return nil, err
	}

	_, err = tx.Exec("DELETE FROM job_cache")
	if err != nil {
		cclog.Errorf("Error while Job cache clean: %v", err)
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		cclog.Errorf("SyncJobs: commit transaction: %v", err)
		return nil, err
	}

	// Resolve correct job.id from the job table. The IDs read from job_cache
	// are from a different auto-increment sequence and must not be used to
	// query the job table.
	for _, job := range jobs {
		var newID int64
		if err := sq.Select("job.id").From("job").
			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
				job.JobID, job.Cluster, job.StartTime).
			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
				job.JobID, job.Cluster, err)
			continue
		}
		job.ID = &newID
	}

	return jobs, nil
}

// TransferCachedJobToMain moves a job from job_cache to the job table.
// Caller must hold r.Mutex. Returns the new job table ID.
func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
	res, err := r.DB.Exec(
		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
		cacheID)
	if err != nil {
		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
	}

	newID, err := res.LastInsertId()
	if err != nil {
		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
	}

	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
	if err != nil {
		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
	}

	return newID, nil
}

// Start inserts a new job in the table, returning the unique job ID.
// Statistics are not transfered!
func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
	job.RawFootprint, err = json.Marshal(job.Footprint)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
	}

	job.RawResources, err = json.Marshal(job.Resources)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
	}

	job.RawMetaData, err = json.Marshal(job.MetaData)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
	}

	return r.InsertJob(job)
}

// StartDirect inserts a new job directly into the job table (not job_cache).
// Use this when the returned ID will immediately be used for job table
// operations such as adding tags.
func (r *JobRepository) StartDirect(job *schema.Job) (id int64, err error) {
	job.RawFootprint, err = json.Marshal(job.Footprint)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
	}

	job.RawResources, err = json.Marshal(job.Resources)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
	}

	job.RawMetaData, err = json.Marshal(job.MetaData)
	if err != nil {
		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
	}

	return r.InsertJobDirect(job)
}

// Stop updates the job with the database id jobId using the provided arguments.
func (r *JobRepository) Stop(
	jobID int64,
	duration int32,
	state schema.JobState,
	monitoringStatus int32,
) (err error) {
	// Invalidate cache entries as job state is changing
	r.cache.Del(fmt.Sprintf("metadata:%d", jobID))
	r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID))

	stmt := sq.Update("job").
		Set("job_state", state).
		Set("duration", duration).
		Set("monitoring_status", monitoringStatus).
		Where("job.id = ?", jobID)

	_, err = stmt.RunWith(r.stmtCache).Exec()
	return err
}

+    return 0, fmt.Errorf("TransferCachedJobToMain: begin transaction: %w", err)

-  _, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)

+  _, err = tx.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)

+    return 0, fmt.Errorf("TransferCachedJobToMain: commit: %w", err)


Bash

Build all packages


(Bash completed with no output)


go test ./internal/repository/... ./internal/api/... 2>&1

Run tests for affected packages


ok  	github.com/ClusterCockpit/cc-backend/internal/repository	0.941s
ok  	github.com/ClusterCockpit/cc-backend/internal/api	7.189s


Done. `TransferCachedJobToMain` now wraps both the INSERT and DELETE in a single transaction (`tx.Beginx()` / `tx.Commit()` with `defer tx.Rollback()`). This ensures atomicity — if the DELETE fails or the process crashes, the INSERT is rolled back too — and reduces lock acquisitions from 2 to 1.