From 09fa239e8bb69bbde7215f8b1c5bcbea23551cb0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 11 Mar 2026 06:14:05 +0100 Subject: [PATCH] fix: Prevent memory explosion in sqlite. And make db options configurable Entire-Checkpoint: e368e6d8abf3 --- .entire/logs/entire.log | 41 +++++++++++++++++++++++++++++ cmd/cc-backend/main.go | 20 ++++++++++++++ internal/config/config.go | 11 ++++++++ internal/config/schema.go | 26 ++++++++++++++++++ internal/repository/config.go | 20 +++++++++++--- internal/repository/dbConnection.go | 11 +++++--- internal/repository/jobFind.go | 4 +-- internal/repository/jobQuery.go | 4 +-- internal/repository/stats.go | 16 +++++------ 9 files changed, 135 insertions(+), 18 deletions(-) diff --git a/.entire/logs/entire.log b/.entire/logs/entire.log index fe99ba74..8495493f 100644 --- a/.entire/logs/entire.log +++ b/.entire/logs/entire.log @@ -45,3 +45,44 @@ {"time":"2026-03-11T05:43:59.204149+01:00","level":"INFO","msg":"turn-end","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} {"time":"2026-03-11T05:43:59.565762+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/eba3995-e3b0c4"} {"time":"2026-03-11T05:43:59.565768+01:00","level":"INFO","msg":"checkpoint saved","session_id":"42401d2e-7d1c-4c0e-abe6-356cb2d48747","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":3,"new_files":3,"deleted_files":0,"shadow_branch":"entire/eba3995-e3b0c4","branch_created":false} +{"time":"2026-03-11T05:46:03.833387+01:00","level":"INFO","msg":"prepare-commit-msg: trailer added","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","source":"message","checkpoint_id":"af7afc9a29ff"} +{"time":"2026-03-11T05:46:04.634284+01:00","level":"INFO","msg":"attribution calculated","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"attribution","agent_lines":385,"human_added":166,"human_modified":0,"human_removed":0,"total_committed":551,"agent_percentage":69.87295825771325,"accumulated_user_added":0,"accumulated_user_removed":0,"files_touched":3} +{"time":"2026-03-11T05:46:04.683895+01:00","level":"INFO","msg":"session condensed","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","checkpoint_id":"af7afc9a29ff","checkpoints_condensed":1,"transcript_lines":40} +{"time":"2026-03-11T05:46:04.685034+01:00","level":"INFO","msg":"post-commit: carried forward remaining files","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","remaining_files":1} +{"time":"2026-03-11T05:46:04.698054+01:00","level":"INFO","msg":"shadow branch deleted","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","strategy":"manual-commit","shadow_branch":"entire/eba3995-e3b0c4"} +{"time":"2026-03-11T05:51:11.625132+01:00","level":"INFO","msg":"turn-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} +{"time":"2026-03-11T05:51:11.968404+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnStart","from":"idle","to":"active"} +{"time":"2026-03-11T05:51:30.210036+01:00","level":"INFO","msg":"subagent started","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SubagentStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","transcript":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} +{"time":"2026-03-11T05:52:16.656618+01:00","level":"INFO","msg":"subagent completed","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SubagentEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","agent_id":"a1dbe6dd7557b1be4"} +{"time":"2026-03-11T05:52:16.871366+01:00","level":"INFO","msg":"committed task checkpoint to shadow branch","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"} +{"time":"2026-03-11T05:52:16.871372+01:00","level":"INFO","msg":"task checkpoint saved","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"task","checkpoint_uuid":"","tool_use_id":"toolu_01L3pfY2BpNHq7yjJw6HRp5r","subagent_type":"Explore","modified_files":3,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false} +{"time":"2026-03-11T05:56:19.392152+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} +{"time":"2026-03-11T05:56:19.67035+01:00","level":"INFO","msg":"no files modified during session, skipping checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code"} +{"time":"2026-03-11T05:56:19.670441+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnEnd","from":"active","to":"idle"} +{"time":"2026-03-11T05:56:47.002811+01:00","level":"INFO","msg":"turn-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} +{"time":"2026-03-11T05:56:47.228777+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnStart","from":"idle","to":"active"} +{"time":"2026-03-11T06:01:36.364992+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/c31c699a-f492-48f7-bcf0-35d3ceeac243.jsonl"} +{"time":"2026-03-11T06:01:36.703304+01:00","level":"INFO","msg":"no files modified during session, skipping checkpoint","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code"} +{"time":"2026-03-11T06:01:36.703385+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"TurnEnd","from":"active","to":"idle"} +{"time":"2026-03-11T06:01:36.773309+01:00","level":"INFO","msg":"session-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SessionEnd","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243"} +{"time":"2026-03-11T06:01:36.773353+01:00","level":"INFO","msg":"phase transition","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"session","agent":"claude-code","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","event":"SessionStop","from":"idle","to":"ended"} +{"time":"2026-03-11T06:01:36.838257+01:00","level":"INFO","msg":"session-start","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"SessionStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"} +{"time":"2026-03-11T06:03:55.812894+01:00","level":"INFO","msg":"turn-end","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"} +{"time":"2026-03-11T06:03:56.341848+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"} +{"time":"2026-03-11T06:03:56.341854+01:00","level":"INFO","msg":"checkpoint saved","session_id":"c31c699a-f492-48f7-bcf0-35d3ceeac243","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":5,"new_files":2,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false} +{"time":"2026-03-11T06:05:35.439593+01:00","level":"INFO","msg":"turn-start","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"} +{"time":"2026-03-11T06:05:35.667094+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"TurnStart","from":"idle","to":"active"} +{"time":"2026-03-11T06:05:49.787804+01:00","level":"INFO","msg":"subagent started","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SubagentStart","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","transcript":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"} +{"time":"2026-03-11T06:06:31.188423+01:00","level":"INFO","msg":"subagent completed","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SubagentEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","agent_id":"afa5b3492136fa9f7"} +{"time":"2026-03-11T06:06:31.426892+01:00","level":"INFO","msg":"committed task checkpoint to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"} +{"time":"2026-03-11T06:06:31.426899+01:00","level":"INFO","msg":"task checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"task","checkpoint_uuid":"","tool_use_id":"toolu_01Fg8BspZD8Gf4W2rKpqcLxE","subagent_type":"Explore","modified_files":5,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false} +{"time":"2026-03-11T06:09:33.192814+01:00","level":"INFO","msg":"turn-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48.jsonl"} +{"time":"2026-03-11T06:09:33.784083+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"} +{"time":"2026-03-11T06:09:33.78409+01:00","level":"INFO","msg":"checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":2,"modified_files":5,"new_files":0,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false} +{"time":"2026-03-11T06:09:33.784142+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"TurnEnd","from":"active","to":"idle"} +{"time":"2026-03-11T06:09:33.857698+01:00","level":"INFO","msg":"session-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SessionEnd","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48"} +{"time":"2026-03-11T06:09:33.857744+01:00","level":"INFO","msg":"phase transition","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"session","agent":"claude-code","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","event":"SessionStop","from":"idle","to":"ended"} +{"time":"2026-03-11T06:09:33.922855+01:00","level":"INFO","msg":"session-start","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"SessionStart","session_id":"50b2b10a-1be0-441f-aafb-3c5828f0fcc9","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/50b2b10a-1be0-441f-aafb-3c5828f0fcc9.jsonl"} +{"time":"2026-03-11T06:11:05.125819+01:00","level":"INFO","msg":"turn-end","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"lifecycle","agent":"claude-code","event":"TurnEnd","session_id":"50b2b10a-1be0-441f-aafb-3c5828f0fcc9","session_ref":"/Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/50b2b10a-1be0-441f-aafb-3c5828f0fcc9.jsonl"} +{"time":"2026-03-11T06:11:05.512735+01:00","level":"INFO","msg":"committed changes to shadow branch","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","shadow_branch":"entire/301e590-e3b0c4"} +{"time":"2026-03-11T06:11:05.512742+01:00","level":"INFO","msg":"checkpoint saved","session_id":"520afa6a-6a70-437b-96c1-35c40ed3ec48","component":"checkpoint","agent":"claude-code","strategy":"manual-commit","checkpoint_type":"session","checkpoint_count":1,"modified_files":8,"new_files":2,"deleted_files":0,"shadow_branch":"entire/301e590-e3b0c4","branch_created":false} diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 57c8d65b..5e6eea9a 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -108,6 +108,26 @@ func initConfiguration() error { } func initDatabase() error { + if config.Keys.DbConfig != nil { + cfg := repository.DefaultConfig() + dc := config.Keys.DbConfig + if dc.CacheSizeMB > 0 { + cfg.DbCacheSizeMB = dc.CacheSizeMB + } + if dc.SoftHeapLimitMB > 0 { + cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB + } + if dc.MaxOpenConnections > 0 { + cfg.MaxOpenConnections = dc.MaxOpenConnections + } + if dc.MaxIdleConnections > 0 { + cfg.MaxIdleConnections = dc.MaxIdleConnections + } + if dc.ConnectionMaxIdleTimeMins > 0 { + cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute + } + repository.SetConfig(cfg) + } repository.Connect(config.Keys.DB) return nil } diff --git a/internal/config/config.go b/internal/config/config.go index f635b7e4..e04d0a72 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,17 @@ type ProgramConfig struct { // Node state retention configuration NodeStateRetention *NodeStateRetention `json:"nodestate-retention"` + + // Database tuning configuration + DbConfig *DbConfig `json:"db-config"` +} + +type DbConfig struct { + CacheSizeMB int `json:"cache-size-mb"` + SoftHeapLimitMB int `json:"soft-heap-limit-mb"` + MaxOpenConnections int `json:"max-open-connections"` + MaxIdleConnections int `json:"max-idle-connections"` + ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"` } type NodeStateRetention struct { diff --git a/internal/config/schema.go b/internal/config/schema.go index 5e02732d..64d9bcf8 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -177,6 +177,32 @@ var configSchema = ` } }, "required": ["policy"] + }, + "db-config": { + "description": "SQLite database tuning configuration.", + "type": "object", + "properties": { + "cache-size-mb": { + "description": "SQLite page cache size per connection in MB (default: 2048).", + "type": "integer" + }, + "soft-heap-limit-mb": { + "description": "Process-wide SQLite soft heap limit in MB (default: 16384).", + "type": "integer" + }, + "max-open-connections": { + "description": "Maximum number of open database connections (default: 4).", + "type": "integer" + }, + "max-idle-connections": { + "description": "Maximum number of idle database connections (default: 4).", + "type": "integer" + }, + "max-idle-time-minutes": { + "description": "Maximum idle time for a connection in minutes (default: 10).", + "type": "integer" + } + } } } }` diff --git a/internal/repository/config.go b/internal/repository/config.go index 114702f3..c574308f 100644 --- a/internal/repository/config.go +++ b/internal/repository/config.go @@ -27,13 +27,25 @@ type RepositoryConfig struct { ConnectionMaxLifetime time.Duration // ConnectionMaxIdleTime is the maximum amount of time a connection may be idle. - // Default: 1 hour + // Default: 10 minutes ConnectionMaxIdleTime time.Duration // MinRunningJobDuration is the minimum duration in seconds for a job to be // considered in "running jobs" queries. This filters out very short jobs. // Default: 600 seconds (10 minutes) MinRunningJobDuration int + + // DbCacheSizeMB is the SQLite page cache size per connection in MB. + // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4 + // and DbCacheSizeMB=2048, total page cache is up to 8GB. + // Default: 2048 (2GB) + DbCacheSizeMB int + + // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB. + // SQLite will try to release cache pages to stay under this limit. + // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive. + // Default: 16384 (16GB) + DbSoftHeapLimitMB int } // DefaultConfig returns the default repository configuration. @@ -44,8 +56,10 @@ func DefaultConfig() *RepositoryConfig { MaxOpenConnections: 4, MaxIdleConnections: 4, ConnectionMaxLifetime: time.Hour, - ConnectionMaxIdleTime: time.Hour, - MinRunningJobDuration: 600, // 10 minutes + ConnectionMaxIdleTime: 10 * time.Minute, + MinRunningJobDuration: 600, // 10 minutes + DbCacheSizeMB: 2048, // 2GB per connection + DbSoftHeapLimitMB: 16384, // 16GB process-wide } } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 25e6c470..cda7981a 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -36,9 +36,10 @@ type DatabaseOptions struct { ConnectionMaxIdleTime time.Duration } -func setupSqlite(db *sql.DB) error { +func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error { pragmas := []string{ "temp_store = memory", + fmt.Sprintf("soft_heap_limit = %d", int64(cfg.DbSoftHeapLimitMB)*1024*1024), } for _, pragma := range pragmas { @@ -79,7 +80,8 @@ func Connect(db string) { connectionURLParams.Add("_journal_mode", "WAL") connectionURLParams.Add("_busy_timeout", "5000") connectionURLParams.Add("_synchronous", "NORMAL") - connectionURLParams.Add("_cache_size", "1000000000") + cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024 // Convert MB to KiB + connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB)) connectionURLParams.Add("_foreign_keys", "true") opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode()) @@ -94,11 +96,14 @@ func Connect(db string) { cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error()) } - err = setupSqlite(dbHandle.DB) + err = setupSqlite(dbHandle.DB, repoConfig) if err != nil { cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error()) } + cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d", + repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections) + dbHandle.SetMaxOpenConns(opts.MaxOpenConnections) dbHandle.SetMaxIdleConns(opts.MaxIdleConnections) dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 13dd4418..45310296 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -171,7 +171,7 @@ func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, return nil, qerr } - return scanJob(q.RunWith(r.stmtCache).QueryRow()) + return scanJob(q.RunWith(r.stmtCache).QueryRowContext(ctx)) } // FindByIDWithUser executes a SQL query to find a specific batch job. @@ -217,7 +217,7 @@ func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime return nil, qerr } - return scanJob(q.RunWith(r.stmtCache).QueryRow()) + return scanJob(q.RunWith(r.stmtCache).QueryRowContext(ctx)) } // IsJobOwner checks if the specified user owns the batch job identified by jobID, diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 437801aa..0c457ec2 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -84,7 +84,7 @@ func (r *JobRepository) QueryJobs( query = BuildWhereClause(f, query) } - rows, err := query.RunWith(r.stmtCache).Query() + rows, err := query.RunWith(r.stmtCache).QueryContext(ctx) if err != nil { queryString, queryVars, _ := query.ToSql() return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err) @@ -126,7 +126,7 @@ func (r *JobRepository) CountJobs( } var count int - if err := query.RunWith(r.DB).Scan(&count); err != nil { + if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&count); err != nil { return 0, fmt.Errorf("failed to count jobs: %w", err) } diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 4a61f62d..16ee3549 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -230,7 +230,7 @@ func (r *JobRepository) JobsStatsGrouped( query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit) } - rows, err := query.RunWith(r.DB).Query() + rows, err := query.RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Warn("Error while querying DB for job statistics") return nil, err @@ -355,7 +355,7 @@ func (r *JobRepository) JobsStats( return nil, err } - row := query.RunWith(r.DB).QueryRow() + row := query.RunWith(r.DB).QueryRowContext(ctx) stats := make([]*model.JobsStatistics, 0, 1) var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64 @@ -440,7 +440,7 @@ func (r *JobRepository) JobCountGrouped( if err != nil { return nil, err } - rows, err := query.RunWith(r.DB).Query() + rows, err := query.RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Warn("Error while querying DB for job statistics") return nil, err @@ -501,7 +501,7 @@ func (r *JobRepository) AddJobCountGrouped( if err != nil { return nil, err } - rows, err := query.RunWith(r.DB).Query() + rows, err := query.RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Warn("Error while querying DB for job statistics") return nil, err @@ -566,7 +566,7 @@ func (r *JobRepository) AddJobCount( return nil, err } var cnt sql.NullInt64 - if err := query.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil { + if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&cnt); err != nil { cclog.Warn("Error while querying DB for job count") return nil, err } @@ -755,7 +755,7 @@ func (r *JobRepository) jobsStatisticsHistogram( query = BuildWhereClause(f, query) } - rows, err := query.GroupBy("value").RunWith(r.DB).Query() + rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Error("Error while running query") return nil, err @@ -829,7 +829,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram( query = BuildWhereClause(f, query) } - rows, err := query.GroupBy("value").RunWith(r.DB).Query() + rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Error("Error while running query") return nil, err @@ -959,7 +959,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( mainQuery = mainQuery.GroupBy("bin").OrderBy("bin") - rows, err := mainQuery.RunWith(r.DB).Query() + rows, err := mainQuery.RunWith(r.DB).QueryContext(ctx) if err != nil { cclog.Errorf("Error while running mainQuery: %s", err) return nil, err