ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2026-03-15 12:27:30 +01:00
Code Issues Packages Releases Activity

fix: Prevent memory explosion in sqlite. And make db options configurable

Browse Source 
Entire-Checkpoint: e368e6d8abf3
This commit is contained in:
Jan Eitzinger
2026-03-11 06:14:05 +01:00
parent 301e590580
commit 09fa239e8b
9 changed files with 135 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
.entire/logs/entire.log
View File

@@ -45,3 +45,44 @@
{"time":"2026-03-11T05:43:59.204149+01:00","level":"INFO","msg":"turn-end","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} {"time":"2026-03-11T05:43:59.204149+01:00","level":"INFO","msg":"turn-end","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T05:43:59.565762+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/eba3995-e3b0c4"} {"time":"2026-03-11T05:43:59.565762+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/eba3995-e3b0c4"}
{"time":"2026-03-11T05:43:59.565768+01:00","level":"INFO","msg":"checkpoint saved","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":3,"new_files":3,"deleted_files":0,"shadow_branch":"entire/eba3995-e3b0c4","branch_created":false} {"time":"2026-03-11T05:43:59.565768+01:00","level":"INFO","msg":"checkpoint saved","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":3,"new_files":3,"deleted_files":0,"shadow_branch":"entire/eba3995-e3b0c4","branch_created":false}
{"time":"2026-03-11T05:46:03.833387+01:00","level":"INFO","msg":"prepare-commit-msg: trailer added","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","source":"message","checkpoint_id":"af7afc9a29ff"}
{"time":"2026-03-11T05:46:04.634284+01:00","level":"INFO","msg":"attribution calculated","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"attribution","agent_lines":385,"human_added":166,"human_modified":0,"human_removed":0,"total_committed":551,"agent_percentage":69.87295825771325,"accumulated_user_added":0,"accumulated_user_removed":0,"files_touched":3}
{"time":"2026-03-11T05:46:04.683895+01:00","level":"INFO","msg":"session condensed","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","checkpoint_id":"af7afc9a29ff","checkpoints_condensed":1,"transcript_lines":40}
{"time":"2026-03-11T05:46:04.685034+01:00","level":"INFO","msg":"post-commit: carried forward remaining files","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","remaining_files":1}
{"time":"2026-03-11T05:46:04.698054+01:00","level":"INFO","msg":"shadow branch deleted","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","shadow_branch":"entire/eba3995-e3b0c4"}
{"time":"2026-03-11T05:51:11.625132+01:00","level":"INFO","msg":"turn-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T05:51:11.968404+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnStart","from":"idle","to":"active"}
{"time":"2026-03-11T05:51:30.210036+01:00","level":"INFO","msg":"subagent started","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SubagentStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","transcript":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T05:52:16.656618+01:00","level":"INFO","msg":"subagent completed","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SubagentEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","agent_id":"a1dbe6dd7557b1be4"}
{"time":"2026-03-11T05:52:16.871366+01:00","level":"INFO","msg":"committed task checkpoint to shadow branch","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"}
{"time":"2026-03-11T05:52:16.871372+01:00","level":"INFO","msg":"task checkpoint saved","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"task","checkpoint_uuid":"","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","subagent_type":"Explore","modified_files":3,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false}
{"time":"2026-03-11T05:56:19.392152+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T05:56:19.67035+01:00","level":"INFO","msg":"no files modified during session, skipping checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code"}
{"time":"2026-03-11T05:56:19.670441+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnEnd","from":"active","to":"idle"}
{"time":"2026-03-11T05:56:47.002811+01:00","level":"INFO","msg":"turn-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T05:56:47.228777+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnStart","from":"idle","to":"active"}
{"time":"2026-03-11T06:01:36.364992+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"}
{"time":"2026-03-11T06:01:36.703304+01:00","level":"INFO","msg":"no files modified during session, skipping checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code"}
{"time":"2026-03-11T06:01:36.703385+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnEnd","from":"active","to":"idle"}
{"time":"2026-03-11T06:01:36.773309+01:00","level":"INFO","msg":"session-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SessionEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243"}
{"time":"2026-03-11T06:01:36.773353+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"SessionStop","from":"idle","to":"ended"}
{"time":"2026-03-11T06:01:36.838257+01:00","level":"INFO","msg":"session-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SessionStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"}
{"time":"2026-03-11T06:03:55.812894+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"}
{"time":"2026-03-11T06:03:56.341848+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"}
{"time":"2026-03-11T06:03:56.341854+01:00","level":"INFO","msg":"checkpoint saved","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":5,"new_files":2,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false}
{"time":"2026-03-11T06:05:35.439593+01:00","level":"INFO","msg":"turn-start","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"}
{"time":"2026-03-11T06:05:35.667094+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"TurnStart","from":"idle","to":"active"}
{"time":"2026-03-11T06:05:49.787804+01:00","level":"INFO","msg":"subagent started","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SubagentStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","transcript":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"}
{"time":"2026-03-11T06:06:31.188423+01:00","level":"INFO","msg":"subagent completed","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SubagentEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","agent_id":"afa5b3492136fa9f7"}
{"time":"2026-03-11T06:06:31.426892+01:00","level":"INFO","msg":"committed task checkpoint to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"}
{"time":"2026-03-11T06:06:31.426899+01:00","level":"INFO","msg":"task checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"task","checkpoint_uuid":"","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","subagent_type":"Explore","modified_files":5,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false}
{"time":"2026-03-11T06:09:33.192814+01:00","level":"INFO","msg":"turn-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"}
{"time":"2026-03-11T06:09:33.784083+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"}
{"time":"2026-03-11T06:09:33.78409+01:00","level":"INFO","msg":"checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":2,"modified_files":5,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false}
{"time":"2026-03-11T06:09:33.784142+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"TurnEnd","from":"active","to":"idle"}
{"time":"2026-03-11T06:09:33.857698+01:00","level":"INFO","msg":"session-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SessionEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48"}
{"time":"2026-03-11T06:09:33.857744+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"SessionStop","from":"idle","to":"ended"}
{"time":"2026-03-11T06:09:33.922855+01:00","level":"INFO","msg":"session-start","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SessionStart","session_id":"50b2b10a-1be0-441f-aafb-3c5828f0fcc9","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/50b2b10a-1be0-441f-aafb-3c5828f0fcc9.jsonl"}
{"time":"2026-03-11T06:11:05.125819+01:00","level":"INFO","msg":"turn-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"50b2b10a-1be0-441f-aafb-3c5828f0fcc9","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/50b2b10a-1be0-441f-aafb-3c5828f0fcc9.jsonl"}
{"time":"2026-03-11T06:11:05.512735+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"}
{"time":"2026-03-11T06:11:05.512742+01:00","level":"INFO","msg":"checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":8,"new_files":2,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false}

20
cmd/cc-backend/main.go
View File

@@ -108,6 +108,26 @@ func initConfiguration() error {
} }
func initDatabase() error { func initDatabase() error {
if config.Keys.DbConfig != nil {
cfg := repository.DefaultConfig()
dc := config.Keys.DbConfig
if dc.CacheSizeMB > 0 {
cfg.DbCacheSizeMB = dc.CacheSizeMB
}
if dc.SoftHeapLimitMB > 0 {
cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
}
if dc.MaxOpenConnections > 0 {
cfg.MaxOpenConnections = dc.MaxOpenConnections
}
if dc.MaxIdleConnections > 0 {
cfg.MaxIdleConnections = dc.MaxIdleConnections
}
if dc.ConnectionMaxIdleTimeMins > 0 {
cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
}
repository.SetConfig(cfg)
}
repository.Connect(config.Keys.DB) repository.Connect(config.Keys.DB)
return nil return nil
} }

11
internal/config/config.go
View File

@@ -77,6 +77,17 @@ type ProgramConfig struct {
// Node state retention configuration // Node state retention configuration
NodeStateRetention *NodeStateRetention `json:"nodestate-retention"` NodeStateRetention *NodeStateRetention `json:"nodestate-retention"`
// Database tuning configuration
DbConfig *DbConfig `json:"db-config"`
}
type DbConfig struct {
CacheSizeMB int `json:"cache-size-mb"`
SoftHeapLimitMB int `json:"soft-heap-limit-mb"`
MaxOpenConnections int `json:"max-open-connections"`
MaxIdleConnections int `json:"max-idle-connections"`
ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
} }
type NodeStateRetention struct { type NodeStateRetention struct {

26
internal/config/schema.go
View File

@@ -177,6 +177,32 @@ var configSchema = `
} }
}, },
"required": ["policy"] "required": ["policy"]
},
"db-config": {
"description": "SQLite database tuning configuration.",
"type": "object",
"properties": {
"cache-size-mb": {
"description": "SQLite page cache size per connection in MB (default: 2048).",
"type": "integer"
},
"soft-heap-limit-mb": {
"description": "Process-wide SQLite soft heap limit in MB (default: 16384).",
"type": "integer"
},
"max-open-connections": {
"description": "Maximum number of open database connections (default: 4).",
"type": "integer"
},
"max-idle-connections": {
"description": "Maximum number of idle database connections (default: 4).",
"type": "integer"
},
"max-idle-time-minutes": {
"description": "Maximum idle time for a connection in minutes (default: 10).",
"type": "integer"
}
}
} }
} }
}` }`

20
internal/repository/config.go
View File

@@ -27,13 +27,25 @@ type RepositoryConfig struct {
ConnectionMaxLifetime time.Duration ConnectionMaxLifetime time.Duration
// ConnectionMaxIdleTime is the maximum amount of time a connection may be idle. // ConnectionMaxIdleTime is the maximum amount of time a connection may be idle.
// Default: 1 hour // Default: 10 minutes
ConnectionMaxIdleTime time.Duration ConnectionMaxIdleTime time.Duration
// MinRunningJobDuration is the minimum duration in seconds for a job to be // MinRunningJobDuration is the minimum duration in seconds for a job to be
// considered in "running jobs" queries. This filters out very short jobs. // considered in "running jobs" queries. This filters out very short jobs.
// Default: 600 seconds (10 minutes) // Default: 600 seconds (10 minutes)
MinRunningJobDuration int MinRunningJobDuration int
// DbCacheSizeMB is the SQLite page cache size per connection in MB.
// Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
// and DbCacheSizeMB=2048, total page cache is up to 8GB.
// Default: 2048 (2GB)
DbCacheSizeMB int
// DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
// SQLite will try to release cache pages to stay under this limit.
// It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
// Default: 16384 (16GB)
DbSoftHeapLimitMB int
} }
// DefaultConfig returns the default repository configuration. // DefaultConfig returns the default repository configuration.
@@ -44,8 +56,10 @@ func DefaultConfig() *RepositoryConfig {
MaxOpenConnections: 4, MaxOpenConnections: 4,
MaxIdleConnections: 4, MaxIdleConnections: 4,
ConnectionMaxLifetime: time.Hour, ConnectionMaxLifetime: time.Hour,
ConnectionMaxIdleTime: time.Hour, ConnectionMaxIdleTime: 10 * time.Minute,
MinRunningJobDuration: 600, // 10 minutes MinRunningJobDuration: 600, // 10 minutes
DbCacheSizeMB: 2048, // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
} }
} }

11
internal/repository/dbConnection.go
View File

@@ -36,9 +36,10 @@ type DatabaseOptions struct {
ConnectionMaxIdleTime time.Duration ConnectionMaxIdleTime time.Duration
} }
func setupSqlite(db *sql.DB) error { func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
pragmas := []string{ pragmas := []string{
"temp_store = memory", "temp_store = memory",
fmt.Sprintf("soft_heap_limit = %d", int64(cfg.DbSoftHeapLimitMB)*1024*1024),
} }
for _, pragma := range pragmas { for _, pragma := range pragmas {
@@ -79,7 +80,8 @@ func Connect(db string) {
connectionURLParams.Add("_journal_mode", "WAL") connectionURLParams.Add("_journal_mode", "WAL")
connectionURLParams.Add("_busy_timeout", "5000") connectionURLParams.Add("_busy_timeout", "5000")
connectionURLParams.Add("_synchronous", "NORMAL") connectionURLParams.Add("_synchronous", "NORMAL")
connectionURLParams.Add("_cache_size", "1000000000") cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024 // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
connectionURLParams.Add("_foreign_keys", "true") connectionURLParams.Add("_foreign_keys", "true")
opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode()) opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode())
@@ -94,11 +96,14 @@ func Connect(db string) {
cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error()) cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error())
} }
err = setupSqlite(dbHandle.DB) err = setupSqlite(dbHandle.DB, repoConfig)
if err != nil { if err != nil {
cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error()) cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error())
} }
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
dbHandle.SetMaxOpenConns(opts.MaxOpenConnections) dbHandle.SetMaxOpenConns(opts.MaxOpenConnections)
dbHandle.SetMaxIdleConns(opts.MaxIdleConnections) dbHandle.SetMaxIdleConns(opts.MaxIdleConnections)
dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime) dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime)

4
internal/repository/jobFind.go
View File

@@ -171,7 +171,7 @@ func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job,
return nil, qerr return nil, qerr
} }
return scanJob(q.RunWith(r.stmtCache).QueryRow()) return scanJob(q.RunWith(r.stmtCache).QueryRowContext(ctx))
} }
// FindByIDWithUser executes a SQL query to find a specific batch job. // FindByIDWithUser executes a SQL query to find a specific batch job.
@@ -217,7 +217,7 @@ func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime
return nil, qerr return nil, qerr
} }
return scanJob(q.RunWith(r.stmtCache).QueryRow()) return scanJob(q.RunWith(r.stmtCache).QueryRowContext(ctx))
} }
// IsJobOwner checks if the specified user owns the batch job identified by jobID, // IsJobOwner checks if the specified user owns the batch job identified by jobID,

4
internal/repository/jobQuery.go
View File

@@ -84,7 +84,7 @@ func (r *JobRepository) QueryJobs(
query = BuildWhereClause(f, query) query = BuildWhereClause(f, query)
} }
rows, err := query.RunWith(r.stmtCache).Query() rows, err := query.RunWith(r.stmtCache).QueryContext(ctx)
if err != nil { if err != nil {
queryString, queryVars, _ := query.ToSql() queryString, queryVars, _ := query.ToSql()
return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err) return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
@@ -126,7 +126,7 @@ func (r *JobRepository) CountJobs(
} }
var count int var count int
if err := query.RunWith(r.DB).Scan(&count); err != nil { if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&count); err != nil {
return 0, fmt.Errorf("failed to count jobs: %w", err) return 0, fmt.Errorf("failed to count jobs: %w", err)
} }

16
internal/repository/stats.go
View File

@@ -230,7 +230,7 @@ func (r *JobRepository) JobsStatsGrouped(
query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit) query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
} }
rows, err := query.RunWith(r.DB).Query() rows, err := query.RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Warn("Error while querying DB for job statistics") cclog.Warn("Error while querying DB for job statistics")
return nil, err return nil, err
@@ -355,7 +355,7 @@ func (r *JobRepository) JobsStats(
return nil, err return nil, err
} }
row := query.RunWith(r.DB).QueryRow() row := query.RunWith(r.DB).QueryRowContext(ctx)
stats := make([]*model.JobsStatistics, 0, 1) stats := make([]*model.JobsStatistics, 0, 1)
var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64 var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
@@ -440,7 +440,7 @@ func (r *JobRepository) JobCountGrouped(
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := query.RunWith(r.DB).Query() rows, err := query.RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Warn("Error while querying DB for job statistics") cclog.Warn("Error while querying DB for job statistics")
return nil, err return nil, err
@@ -501,7 +501,7 @@ func (r *JobRepository) AddJobCountGrouped(
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := query.RunWith(r.DB).Query() rows, err := query.RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Warn("Error while querying DB for job statistics") cclog.Warn("Error while querying DB for job statistics")
return nil, err return nil, err
@@ -566,7 +566,7 @@ func (r *JobRepository) AddJobCount(
return nil, err return nil, err
} }
var cnt sql.NullInt64 var cnt sql.NullInt64
if err := query.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil { if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&cnt); err != nil {
cclog.Warn("Error while querying DB for job count") cclog.Warn("Error while querying DB for job count")
return nil, err return nil, err
} }
@@ -755,7 +755,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
query = BuildWhereClause(f, query) query = BuildWhereClause(f, query)
} }
rows, err := query.GroupBy("value").RunWith(r.DB).Query() rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Error("Error while running query") cclog.Error("Error while running query")
return nil, err return nil, err
@@ -829,7 +829,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
query = BuildWhereClause(f, query) query = BuildWhereClause(f, query)
} }
rows, err := query.GroupBy("value").RunWith(r.DB).Query() rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Error("Error while running query") cclog.Error("Error while running query")
return nil, err return nil, err
@@ -959,7 +959,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
mainQuery = mainQuery.GroupBy("bin").OrderBy("bin") mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
rows, err := mainQuery.RunWith(r.DB).Query() rows, err := mainQuery.RunWith(r.DB).QueryContext(ctx)
if err != nil { if err != nil {
cclog.Errorf("Error while running mainQuery: %s", err) cclog.Errorf("Error while running mainQuery: %s", err)
return nil, err return nil, err