// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package config implements the program configuration data structures, validation and parsing
package config

import (
	"bytes"
	"encoding/json"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
)

type ProgramConfig struct {
	// Address where the http (or https) server will listen on (for example: 'localhost:80').
	Addr string `json:"addr"`

	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
	APIAllowedIPs []string `json:"api-allowed-ips"`

	APISubjects *NATSConfig `json:"api-subjects"`

	// Drop root permissions once .env was read and the port was taken.
	User  string `json:"user"`
	Group string `json:"group"`

	// Disable authentication (for everything: API, Web-UI, ...)
	DisableAuthentication bool `json:"disable-authentication"`

	// If `embed-static-files` is true (default), the frontend files are directly
	// embeded into the go binary and expected to be in web/frontend. Only if
	// it is false the files in `static-files` are served instead.
	EmbedStaticFiles bool   `json:"embed-static-files"`
	StaticFiles      string `json:"static-files"`

	// Path to SQLite database file
	DB string `json:"db"`

	EnableJobTaggers bool `json:"enable-job-taggers"`

	// Validate json input against schema
	Validate bool `json:"validate"`

	// If 0 or empty, the session does not expire!
	SessionMaxAge string `json:"session-max-age"`

	// If both those options are not empty, use HTTPS using those certificates.
	HTTPSCertFile string `json:"https-cert-file"`
	HTTPSKeyFile  string `json:"https-key-file"`

	// If not the empty string and `addr` does not end in ":80",
	// redirect every request incoming at port 80 to that url.
	RedirectHTTPTo string `json:"redirect-http-to"`

	// Where to store MachineState files
	MachineStateDir string `json:"machine-state-dir"`

	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`

	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`

	// Energy Mix CO2 Emission Constant [g/kWh]
	// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
	EmissionConstant int `json:"emission-constant"`

	// If exists, will enable dynamic zoom in frontend metric plots using the configured values
	EnableResampling *ResampleConfig `json:"resampling"`

	// Systemd unit name for log viewer (default: "clustercockpit")
	SystemdUnit string `json:"systemd-unit"`

	// Node state retention configuration
	NodeStateRetention *NodeStateRetention `json:"nodestate-retention"`
}

type NodeStateRetention struct {
	Policy             string `json:"policy"`      // "delete" or "move"
	Age                int    `json:"age"`         // hours, default 24
	TargetKind         string `json:"target-kind"` // "file" or "s3"
	TargetPath         string `json:"target-path"`
	TargetEndpoint     string `json:"target-endpoint"`
	TargetBucket       string `json:"target-bucket"`
	TargetAccessKey    string `json:"target-access-key"`
	TargetSecretKey    string `json:"target-secret-key"`
	TargetRegion       string `json:"target-region"`
	TargetUsePathStyle bool   `json:"target-use-path-style"`
	MaxFileSizeMB      int    `json:"max-file-size-mb"`
}

type ResampleConfig struct {
	// Minimum number of points to trigger resampling of data
	MinimumPoints int `json:"minimum-points"`
	// Array of resampling target resolutions, in seconds; Example: [600,300,60]
	Resolutions []int `json:"resolutions"`
	// Trigger next zoom level at less than this many visible datapoints
	Trigger int `json:"trigger"`
}

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
}

type IntRange struct {
	From int `json:"from"`
	To   int `json:"to"`
}

type TimeRange struct {
	From  *time.Time `json:"from"`
	To    *time.Time `json:"to"`
	Range string     `json:"range,omitempty"`
}

type FilterRanges struct {
	Duration  *IntRange  `json:"duration"`
	NumNodes  *IntRange  `json:"num-nodes"`
	StartTime *TimeRange `json:"start-time"`
}

var Keys ProgramConfig = ProgramConfig{
	Addr:                      "localhost:8080",
	EmbedStaticFiles:          true,
	DB:                        "./var/job.db",
	SessionMaxAge:             "168h",
	StopJobsExceedingWalltime: 0,
	ShortRunningJobsDuration:  5 * 60,
}

func Init(mainConfig json.RawMessage) {
	Validate(configSchema, mainConfig)
	dec := json.NewDecoder(bytes.NewReader(mainConfig))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&Keys); err != nil {
		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
	}

	if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 {
		resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints)
	}
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package config
     7→
     8→var configSchema = `
     9→{
    10→  "type": "object",
    11→  "properties": {
    12→    "addr": {
    13→      "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').",
    14→      "type": "string"
    15→    },
    16→    "api-allowed-ips": {
    17→      "description": "Addresses from which secured API endpoints can be reached",
    18→      "type": "array",
    19→      "items": {
    20→        "type": "string"
    21→      }
    22→    },
    23→    "user": {
    24→      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
    25→      "type": "string"
    26→    },
    27→    "group": {
    28→      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
    29→      "type": "string"
    30→    },
    31→    "disable-authentication": {
    32→      "description": "Disable authentication (for everything: API, Web-UI, ...).",
    33→      "type": "boolean"
    34→    },
    35→    "embed-static-files": {
    36→      "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.",
    37→      "type": "boolean"
    38→    },
    39→    "static-files": {
    40→      "description": "Folder where static assets can be found, if embed-static-files is false.",
    41→      "type": "string"
    42→    },
    43→    "db": {
    44→      "description": "Path to SQLite database file (e.g., './var/job.db')",
    45→      "type": "string"
    46→    },
    47→    "enable-job-taggers": {
    48→      "description": "Turn on automatic application and jobclass taggers",
    49→      "type": "boolean"
    50→    },
    51→    "validate": {
    52→      "description": "Validate all input json documents against json schema.",
    53→      "type": "boolean"
    54→    },
    55→    "session-max-age": {
    56→      "description": "Specifies for how long a session shall be valid  as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!",
    57→      "type": "string"
    58→    },
    59→    "https-cert-file": {
    60→      "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.",
    61→      "type": "string"
    62→    },
    63→    "https-key-file": {
    64→      "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.",
    65→      "type": "string"
    66→    },
    67→    "redirect-http-to": {
    68→      "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.",
    69→      "type": "string"
    70→    },
    71→    "stop-jobs-exceeding-walltime": {
    72→      "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.",
    73→      "type": "integer"
    74→    },
    75→    "short-running-jobs-duration": {
    76→      "description": "Do not show running jobs shorter than X seconds.",
    77→      "type": "integer"
    78→    },
    79→    "emission-constant": {
    80→      "description": "Energy mix CO2 emission constant [g/kWh]. If set, displays estimated CO2 emission for jobs.",
    81→      "type": "integer"
    82→    },
    83→    "machine-state-dir": {
    84→      "description": "Where to store MachineState files.",
    85→      "type": "string"
    86→    },
    87→    "systemd-unit": {
    88→      "description": "Systemd unit name for log viewer (default: 'clustercockpit').",
    89→      "type": "string"
    90→    },
    91→    "resampling": {
    92→      "description": "Enable dynamic zoom in frontend metric plots.",
    93→      "type": "object",
    94→      "properties": {
    95→        "minimum-points": {
    96→          "description": "Minimum points to trigger resampling of time-series data.",
    97→          "type": "integer"
    98→        },
    99→        "trigger": {
   100→          "description": "Trigger next zoom level at less than this many visible datapoints.",
   101→          "type": "integer"
   102→        },
   103→        "resolutions": {
   104→          "description": "Array of resampling target resolutions, in seconds.",
   105→          "type": "array",
   106→          "items": {
   107→            "type": "integer"
   108→          }
   109→        }
   110→      },
   111→      "required": ["trigger", "resolutions"]
   112→    },
   113→    "api-subjects": {
   114→      "description": "NATS subjects configuration for subscribing to job and node events.",
   115→      "type": "object",
   116→      "properties": {
   117→        "subject-job-event": {
   118→          "description": "NATS subject for job events (start_job, stop_job)",
   119→          "type": "string"
   120→        },
   121→        "subject-node-state": {
   122→          "description": "NATS subject for node state updates",
   123→          "type": "string"
   124→        }
   125→      },
   126→      "required": ["subject-job-event", "subject-node-state"]
   127→    },
   128→    "nodestate-retention": {
   129→      "description": "Node state retention configuration for cleaning up old node_state rows.",
   130→      "type": "object",
   131→      "properties": {
   132→        "policy": {
   133→          "description": "Retention policy: 'delete' to remove old rows, 'move' to archive to Parquet then delete.",
   134→          "type": "string",
   135→          "enum": ["delete", "move"]
   136→        },
   137→        "age": {
   138→          "description": "Retention age in hours (default: 24).",
   139→          "type": "integer"
   140→        },
   141→        "target-kind": {
   142→          "description": "Target kind for parquet archiving: 'file' or 's3'.",
   143→          "type": "string",
   144→          "enum": ["file", "s3"]
   145→        },
   146→        "target-path": {
   147→          "description": "Filesystem path for parquet file target.",
   148→          "type": "string"
   149→        },
   150→        "target-endpoint": {
   151→          "description": "S3 endpoint URL.",
   152→          "type": "string"
   153→        },
   154→        "target-bucket": {
   155→          "description": "S3 bucket name.",
   156→          "type": "string"
   157→        },
   158→        "target-access-key": {
   159→          "description": "S3 access key.",
   160→          "type": "string"
   161→        },
   162→        "target-secret-key": {
   163→          "description": "S3 secret key.",
   164→          "type": "string"
   165→        },
   166→        "target-region": {
   167→          "description": "S3 region.",
   168→          "type": "string"
   169→        },
   170→        "target-use-path-style": {
   171→          "description": "Use path-style S3 addressing.",
   172→          "type": "boolean"
   173→        },
   174→        "max-file-size-mb": {
   175→          "description": "Maximum parquet file size in MB (default: 128).",
   176→          "type": "integer"
   177→        }
   178→      },
   179→      "required": ["policy"]
   180→    }
   181→  }
   182→}`
   183→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package config

var configSchema = `
{
  "type": "object",
  "properties": {
    "addr": {
      "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').",
      "type": "string"
    },
    "api-allowed-ips": {
      "description": "Addresses from which secured API endpoints can be reached",
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "user": {
      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
      "type": "string"
    },
    "group": {
      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
      "type": "string"
    },
    "disable-authentication": {
      "description": "Disable authentication (for everything: API, Web-UI, ...).",
      "type": "boolean"
    },
    "embed-static-files": {
      "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.",
      "type": "boolean"
    },
    "static-files": {
      "description": "Folder where static assets can be found, if embed-static-files is false.",
      "type": "string"
    },
    "db": {
      "description": "Path to SQLite database file (e.g., './var/job.db')",
      "type": "string"
    },
    "enable-job-taggers": {
      "description": "Turn on automatic application and jobclass taggers",
      "type": "boolean"
    },
    "validate": {
      "description": "Validate all input json documents against json schema.",
      "type": "boolean"
    },
    "session-max-age": {
      "description": "Specifies for how long a session shall be valid  as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!",
      "type": "string"
    },
    "https-cert-file": {
      "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.",
      "type": "string"
    },
    "https-key-file": {
      "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.",
      "type": "string"
    },
    "redirect-http-to": {
      "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.",
      "type": "string"
    },
    "stop-jobs-exceeding-walltime": {
      "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.",
      "type": "integer"
    },
    "short-running-jobs-duration": {
      "description": "Do not show running jobs shorter than X seconds.",
      "type": "integer"
    },
    "emission-constant": {
      "description": "Energy mix CO2 emission constant [g/kWh]. If set, displays estimated CO2 emission for jobs.",
      "type": "integer"
    },
    "machine-state-dir": {
      "description": "Where to store MachineState files.",
      "type": "string"
    },
    "systemd-unit": {
      "description": "Systemd unit name for log viewer (default: 'clustercockpit').",
      "type": "string"
    },
    "resampling": {
      "description": "Enable dynamic zoom in frontend metric plots.",
      "type": "object",
      "properties": {
        "minimum-points": {
          "description": "Minimum points to trigger resampling of time-series data.",
          "type": "integer"
        },
        "trigger": {
          "description": "Trigger next zoom level at less than this many visible datapoints.",
          "type": "integer"
        },
        "resolutions": {
          "description": "Array of resampling target resolutions, in seconds.",
          "type": "array",
          "items": {
            "type": "integer"
          }
        }
      },
      "required": ["trigger", "resolutions"]
    },
    "api-subjects": {
      "description": "NATS subjects configuration for subscribing to job and node events.",
      "type": "object",
      "properties": {
        "subject-job-event": {
          "description": "NATS subject for job events (start_job, stop_job)",
          "type": "string"
        },
        "subject-node-state": {
          "description": "NATS subject for node state updates",
          "type": "string"
        }
      },
      "required": ["subject-job-event", "subject-node-state"]
    },
    "nodestate-retention": {
      "description": "Node state retention configuration for cleaning up old node_state rows.",
      "type": "object",
      "properties": {
        "policy": {
          "description": "Retention policy: 'delete' to remove old rows, 'move' to archive to Parquet then delete.",
          "type": "string",
          "enum": ["delete", "move"]
        },
        "age": {
          "description": "Retention age in hours (default: 24).",
          "type": "integer"
        },
        "target-kind": {
          "description": "Target kind for parquet archiving: 'file' or 's3'.",
          "type": "string",
          "enum": ["file", "s3"]
        },
        "target-path": {
          "description": "Filesystem path for parquet file target.",
          "type": "string"
        },
        "target-endpoint": {
          "description": "S3 endpoint URL.",
          "type": "string"
        },
        "target-bucket": {
          "description": "S3 bucket name.",
          "type": "string"
        },
        "target-access-key": {
          "description": "S3 access key.",
          "type": "string"
        },
        "target-secret-key": {
          "description": "S3 secret key.",
          "type": "string"
        },
        "target-region": {
          "description": "S3 region.",
          "type": "string"
        },
        "target-use-path-style": {
          "description": "Use path-style S3 addressing.",
          "type": "boolean"
        },
        "max-file-size-mb": {
          "description": "Maximum parquet file size in MB (default: 128).",
          "type": "integer"
        }
      },
      "required": ["policy"]
    }
  }
}`


Now let me check how `initDatabase` works in main.go.


108-}
109-
110:func initDatabase() error {
111:	repository.Connect(config.Keys.DB)
112-	return nil
113-}
114-
115-func handleDatabaseCommands() error {
116-	if flagMigrateDB {
--
486-
487-	// Initialize database
488:	if err := initDatabase(); err != nil {
489-		return err
490-	}
491-
492-	// Handle user commands (add, delete, sync, JWT)
493-	if err := handleUserCommands(); err != nil {


I have all the information I need. Here's my assessment:

**Yes, it makes sense** to make these configurable, especially since:
1. `RepositoryConfig` + `SetConfig()` already exist but are **never wired up** — there's no way to override defaults from config.json today
2. A 512GB server could comfortably use 2-4GB per connection cache (8-16GB total) and a 16GB soft heap limit
3. The pattern already exists in the codebase (`MaxOpenConnections`, `MaxIdleConnections` are in `RepositoryConfig` but also can't be tuned from config.json)

The scope would be:
1. Add `DbCacheSizeMB` and `DbSoftHeapLimitMB` fields to `RepositoryConfig`
2. Add a `"db-config"` section to `ProgramConfig` and the JSON schema
3. Wire `SetConfig()` into `initDatabase()` in main.go
4. Use the config values in `Connect()` and `setupSqlite()`

Let me update the plan.


The production db is 80GB. So the suggested values for cache and heap limit potentially limit the performance


Edit

# Fix SQLite Memory Not Released After Query Timeout

## Context

On the production 20M-row database, when a query runs into a timeout (due to full-table scan with wrong index), the memory allocated by SQLite is **not released afterwards**. The process stays bloated until restarted. This is caused by three compounding issues in the current SQLite configuration.

## Root Cause Analysis

### 1. `_cache_size=1000000000` is effectively unlimited (~4TB)

**File:** `internal/repository/dbConnection.go:82`

```go
connectionURLParams.Add("_cache_size", "1000000000")
```

SQLite's `cache_size` PRAGMA interprets **positive values as page count** (default page size = 4KB). So 1,000,000,000 pages × 4KB = ~4TB. In practice, this means "never evict cached pages." After a full-table scan of 20M rows, every page touched stays in SQLite's page cache. With 4 connections (`MaxOpenConns=4`), each can independently cache gigabytes.

For comparison, the SQLite archive backend in `pkg/archive/sqliteBackend.go` uses `PRAGMA cache_size=-64000` (64MB — negative = KiB).

### 2. No query context/timeout — queries run indefinitely

**File:** `internal/repository/jobQuery.go:87`

```go
rows, err := query.RunWith(r.stmtCache).Query()  // No context!
```

The `ctx` parameter is available but never passed to the database layer. Squirrel supports `.QueryContext(ctx)` but it's not used. If the HTTP request times out or the client disconnects, the query keeps running and scanning pages into cache.

### 3. No SQLite memory limit — no `soft_heap_limit` or `shrink_memory`

SQLite has built-in memory management PRAGMAs that are not configured:
- **`soft_heap_limit`** — asks SQLite to keep heap usage below N bytes (best-effort, releases cache pages to stay under limit)
- **`hard_heap_limit`** — hard cap, queries fail with SQLITE_NOMEM if exceeded
- **`shrink_memory`** — immediately releases all unused memory back to the OS

None of these are set, so SQLite allocates freely and never releases.

### 4. `temp_store = memory` amplifies the problem

**File:** `internal/repository/dbConnection.go:41`

Temporary B-tree sorts (exactly what happens during ORDER BY on a full-table scan) are stored in RAM. With 20M rows and no sort optimization, this can be gigabytes of temporary memory on top of the page cache.

### 5. Connections live for 1 hour after use

`ConnMaxIdleTime = 1 hour` means a connection that just did a massive full-table scan sits idle in the pool for up to an hour, holding all its cached pages.

## Proposed Changes

### Fix 1: Set reasonable `cache_size` (high impact, low risk)

**File:** `internal/repository/dbConnection.go:82`

Change from `1000000000` (1B pages ≈ 4TB) to `-200000` (200MB in KiB notation, per connection):

```go
connectionURLParams.Add("_cache_size", "-200000")  // 200MB per connection
```

With 4 max connections: 4 × 200MB = 800MB max page cache. This is generous enough for normal queries but prevents runaway memory after full-table scans.

### Fix 2: Add `soft_heap_limit` (high impact, low risk)

**File:** `internal/repository/dbConnection.go`, in `setupSqlite()`:

```go
"soft_heap_limit = 1073741824",  // 1GB soft limit across all connections
```

This is a **process-wide** limit (not per-connection). SQLite will try to release cache pages to stay under 1GB total. It's a soft limit — it won't abort queries, just evicts cache more aggressively.

### Fix 3: Pass context to database queries (medium impact, medium effort)

Change `.Query()` to `.QueryContext(ctx)`, `.QueryRow()` to `.QueryRowContext(ctx)`, and `.Scan()` to `.ScanContext(ctx)` for all query methods that already receive a `ctx` parameter. This allows HTTP request cancellation to stop the SQLite query.

**Note:** The `stmtCache` from squirrel supports `QueryContext`/`QueryRowContext`. Only methods that already have `ctx` are changed — no signature changes needed.

**Call sites to update** (methods that have `ctx` and call `.Query()`/`.QueryRow()`/`.Scan()`):

| File | Method | Line | Change |
|------|--------|------|--------|
| `jobQuery.go` | `QueryJobs` | 87 | `.Query()` → `.QueryContext(ctx)` |
| `jobQuery.go` | `CountJobs` | 129 | `.Scan()` → `.ScanContext(ctx)` |
| `stats.go` | `JobsStatsGrouped` | 233 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `JobsStats` | 358 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `stats.go` | `JobCountGrouped` | 443 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `AddJobCountGrouped` | 504 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `AddJobCount` | 569 | `.Scan()` → `.ScanContext(ctx)` |
| `stats.go` | `jobsStatisticsHistogram` | 758 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `jobsDurationStatisticsHistogram` | 832 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `jobsMetricStatisticsHistogram` | 962 | `.Query()` → `.QueryContext(ctx)` |
| `jobFind.go` | `FindByID` | 174 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `jobFind.go` | `FindByJobID` | 220 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `job.go` | `CountGroupedJobs` | 410 | `.Scan()` → `.ScanContext(ctx)` (needs ctx added to signature) |
| `job.go` | `GetJobList` | 751 | `.Query()` → `.QueryContext(ctx)` (needs ctx added to signature) |

Methods without `ctx` in their signature (e.g., `GetJobList`, `CountGroupedJobs`) can either have `ctx` added or be left for a follow-up. The priority is the methods already accepting `ctx`.

### Fix 4: Reduce `ConnMaxIdleTime`

**File:** `internal/repository/config.go`

Reduce `ConnectionMaxIdleTime` from 1 hour to 10 minutes. Idle connections with cached pages are released sooner.

## Scope — All Four Fixes

All fixes are complementary, independent, and low-risk. No disadvantages to applying all together.

## Files to Modify

- `internal/repository/dbConnection.go` — cache_size, soft_heap_limit (Fixes 1 & 2)
- `internal/repository/jobQuery.go` — QueryContext (Fix 3)
- `internal/repository/job.go` — QueryContext for other query methods (Fix 3)
- `internal/repository/stats.go` — QueryContext for stats queries (Fix 3)
- `internal/repository/config.go` — ConnMaxIdleTime (Fix 4, optional)

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/...` — tests pass
3. Monitor RSS memory before/after on production:
   - Trigger a heavy query, observe memory spike
   - Verify memory drops back down within seconds (not hours)
4. `PRAGMA cache_size;` in sqlite3 CLI on running DB — should show `-200000`
5. `PRAGMA soft_heap_limit;` — should show `1073741824`

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     200,   // 200MB per connection
DbSoftHeapLimitMB: 1024,  // 1GB process-wide
```

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values


The file /Users/jan/.claude/plans/refactored-wishing-journal.md has been updated successfully.

# Fix SQLite Memory Not Released After Query Timeout

## Context

On the production 20M-row database, when a query runs into a timeout (due to full-table scan with wrong index), the memory allocated by SQLite is **not released afterwards**. The process stays bloated until restarted. This is caused by three compounding issues in the current SQLite configuration.

## Root Cause Analysis

### 1. `_cache_size=1000000000` is effectively unlimited (~4TB)

**File:** `internal/repository/dbConnection.go:82`

```go
connectionURLParams.Add("_cache_size", "1000000000")
```

SQLite's `cache_size` PRAGMA interprets **positive values as page count** (default page size = 4KB). So 1,000,000,000 pages × 4KB = ~4TB. In practice, this means "never evict cached pages." After a full-table scan of 20M rows, every page touched stays in SQLite's page cache. With 4 connections (`MaxOpenConns=4`), each can independently cache gigabytes.

For comparison, the SQLite archive backend in `pkg/archive/sqliteBackend.go` uses `PRAGMA cache_size=-64000` (64MB — negative = KiB).

### 2. No query context/timeout — queries run indefinitely

**File:** `internal/repository/jobQuery.go:87`

```go
rows, err := query.RunWith(r.stmtCache).Query()  // No context!
```

The `ctx` parameter is available but never passed to the database layer. Squirrel supports `.QueryContext(ctx)` but it's not used. If the HTTP request times out or the client disconnects, the query keeps running and scanning pages into cache.

### 3. No SQLite memory limit — no `soft_heap_limit` or `shrink_memory`

SQLite has built-in memory management PRAGMAs that are not configured:
- **`soft_heap_limit`** — asks SQLite to keep heap usage below N bytes (best-effort, releases cache pages to stay under limit)
- **`hard_heap_limit`** — hard cap, queries fail with SQLITE_NOMEM if exceeded
- **`shrink_memory`** — immediately releases all unused memory back to the OS

None of these are set, so SQLite allocates freely and never releases.

### 4. `temp_store = memory` amplifies the problem

**File:** `internal/repository/dbConnection.go:41`

Temporary B-tree sorts (exactly what happens during ORDER BY on a full-table scan) are stored in RAM. With 20M rows and no sort optimization, this can be gigabytes of temporary memory on top of the page cache.

### 5. Connections live for 1 hour after use

`ConnMaxIdleTime = 1 hour` means a connection that just did a massive full-table scan sits idle in the pool for up to an hour, holding all its cached pages.

## Proposed Changes

### Fix 1: Set reasonable `cache_size` (high impact, low risk)

**File:** `internal/repository/dbConnection.go:82`

Change from `1000000000` (1B pages ≈ 4TB) to `-200000` (200MB in KiB notation, per connection):

```go
connectionURLParams.Add("_cache_size", "-200000")  // 200MB per connection
```

With 4 max connections: 4 × 200MB = 800MB max page cache. This is generous enough for normal queries but prevents runaway memory after full-table scans.

### Fix 2: Add `soft_heap_limit` (high impact, low risk)

**File:** `internal/repository/dbConnection.go`, in `setupSqlite()`:

```go
"soft_heap_limit = 1073741824",  // 1GB soft limit across all connections
```

This is a **process-wide** limit (not per-connection). SQLite will try to release cache pages to stay under 1GB total. It's a soft limit — it won't abort queries, just evicts cache more aggressively.

### Fix 3: Pass context to database queries (medium impact, medium effort)

Change `.Query()` to `.QueryContext(ctx)`, `.QueryRow()` to `.QueryRowContext(ctx)`, and `.Scan()` to `.ScanContext(ctx)` for all query methods that already receive a `ctx` parameter. This allows HTTP request cancellation to stop the SQLite query.

**Note:** The `stmtCache` from squirrel supports `QueryContext`/`QueryRowContext`. Only methods that already have `ctx` are changed — no signature changes needed.

**Call sites to update** (methods that have `ctx` and call `.Query()`/`.QueryRow()`/`.Scan()`):

| File | Method | Line | Change |
|------|--------|------|--------|
| `jobQuery.go` | `QueryJobs` | 87 | `.Query()` → `.QueryContext(ctx)` |
| `jobQuery.go` | `CountJobs` | 129 | `.Scan()` → `.ScanContext(ctx)` |
| `stats.go` | `JobsStatsGrouped` | 233 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `JobsStats` | 358 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `stats.go` | `JobCountGrouped` | 443 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `AddJobCountGrouped` | 504 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `AddJobCount` | 569 | `.Scan()` → `.ScanContext(ctx)` |
| `stats.go` | `jobsStatisticsHistogram` | 758 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `jobsDurationStatisticsHistogram` | 832 | `.Query()` → `.QueryContext(ctx)` |
| `stats.go` | `jobsMetricStatisticsHistogram` | 962 | `.Query()` → `.QueryContext(ctx)` |
| `jobFind.go` | `FindByID` | 174 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `jobFind.go` | `FindByJobID` | 220 | `.QueryRow()` → `.QueryRowContext(ctx)` |
| `job.go` | `CountGroupedJobs` | 410 | `.Scan()` → `.ScanContext(ctx)` (needs ctx added to signature) |
| `job.go` | `GetJobList` | 751 | `.Query()` → `.QueryContext(ctx)` (needs ctx added to signature) |

Methods without `ctx` in their signature (e.g., `GetJobList`, `CountGroupedJobs`) can either have `ctx` added or be left for a follow-up. The priority is the methods already accepting `ctx`.

### Fix 4: Reduce `ConnMaxIdleTime`

**File:** `internal/repository/config.go`

Reduce `ConnectionMaxIdleTime` from 1 hour to 10 minutes. Idle connections with cached pages are released sooner.

## Scope — All Four Fixes

All fixes are complementary, independent, and low-risk. No disadvantages to applying all together.

## Files to Modify

- `internal/repository/dbConnection.go` — cache_size, soft_heap_limit (Fixes 1 & 2)
- `internal/repository/jobQuery.go` — QueryContext (Fix 3)
- `internal/repository/job.go` — QueryContext for other query methods (Fix 3)
- `internal/repository/stats.go` — QueryContext for stats queries (Fix 3)
- `internal/repository/config.go` — ConnMaxIdleTime (Fix 4, optional)

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/...` — tests pass
3. Monitor RSS memory before/after on production:
   - Trigger a heavy query, observe memory spike
   - Verify memory drops back down within seconds (not hours)
4. `PRAGMA cache_size;` in sqlite3 CLI on running DB — should show `-200000`
5. `PRAGMA soft_heap_limit;` — should show `1073741824`

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     200,   // 200MB per connection
DbSoftHeapLimitMB: 1024,  // 1GB process-wide
```

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values

-# Fix SQLite Memory Not Released After Query Timeout

+# Make SQLite Memory Limits Configurable via config.json

-On the production 20M-row database, when a query runs into a timeout (due to full-table scan with wrong index), the memory allocated by SQLite is **not released afterwards**. The process stays bloated until restarted. This is caused by three compounding issues in the current SQLite configuration.

+Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

-### 1. `_cache_size=1000000000` is effectively unlimited (~4TB)

+- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)

+- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)

+- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)

+- `MaxOpenConnections = 4` (hardcoded default in `config.go`)

+- Context propagation to all query call sites (already done)

+`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

-SQLite's `cache_size` PRAGMA interprets **positive values as page count** (default page size = 4KB). So 1,000,000,000 pages × 4KB = ~4TB. In practice, this means "never evict cached pages." After a full-table scan of 20M rows, every page touched stays in SQLite's page cache. With 4 connections (`MaxOpenConns=4`), each can independently cache gigabytes.

-For comparison, the SQLite archive backend in `pkg/archive/sqliteBackend.go` uses `PRAGMA cache_size=-64000` (64MB — negative = KiB).

+### 1. Add SQLite memory fields to `RepositoryConfig`

-### 2. No query context/timeout — queries run indefinitely

+Add two new fields with sensible defaults:

-rows, err := query.RunWith(r.stmtCache).Query()  // No context!

-The `ctx` parameter is available but never passed to the database layer. Squirrel supports `.QueryContext(ctx)` but it's not used. If the HTTP request times out or the client disconnects, the query keeps running and scanning pages into cache.

+    // DbCacheSizeMB is the SQLite page cache size per connection in MB.

+    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4

+    // and DbCacheSizeMB=200, total page cache is up to 800MB.

-### 3. No SQLite memory limit — no `soft_heap_limit` or `shrink_memory`

+    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.

+    // SQLite will try to release cache pages to stay under this limit.

+    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.

-SQLite has built-in memory management PRAGMAs that are not configured:

-- **`soft_heap_limit`** — asks SQLite to keep heap usage below N bytes (best-effort, releases cache pages to stay under limit)

-- **`hard_heap_limit`** — hard cap, queries fail with SQLITE_NOMEM if exceeded

-- **`shrink_memory`** — immediately releases all unused memory back to the OS

+DbCacheSizeMB:     200,   // 200MB per connection

+DbSoftHeapLimitMB: 1024,  // 1GB process-wide

-None of these are set, so SQLite allocates freely and never releases.

+### 2. Use config values in `Connect()` and `setupSqlite()`

-### 4. `temp_store = memory` amplifies the problem

+In `Connect()`, replace the hardcoded cache_size:

+cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB

-Temporary B-tree sorts (exactly what happens during ORDER BY on a full-table scan) are stored in RAM. With 20M rows and no sort optimization, this can be gigabytes of temporary memory on top of the page cache.

+Change `setupSqlite()` to accept the config and use it for soft_heap_limit:

+func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {

+        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),

-### 5. Connections live for 1 hour after use

-`ConnMaxIdleTime = 1 hour` means a connection that just did a massive full-table scan sits idle in the pool for up to an hour, holding all its cached pages.

-### Fix 1: Set reasonable `cache_size` (high impact, low risk)

-Change from `1000000000` (1B pages ≈ 4TB) to `-200000` (200MB in KiB notation, per connection):

-connectionURLParams.Add("_cache_size", "-200000")  // 200MB per connection

+err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)

-With 4 max connections: 4 × 200MB = 800MB max page cache. This is generous enough for normal queries but prevents runaway memory after full-table scans.

+### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

-### Fix 2: Add `soft_heap_limit` (high impact, low risk)

-**File:** `internal/repository/dbConnection.go`, in `setupSqlite()`:

+Add a new struct and field to `ProgramConfig`:

-"soft_heap_limit = 1073741824",  // 1GB soft limit across all connections

+    CacheSizeMB       int `json:"cache-size-mb"`

+    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`

+    MaxOpenConnections int `json:"max-open-connections"`

+    MaxIdleConnections int `json:"max-idle-connections"`

+    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`

+    DbConfig *DbConfig `json:"db-config"`

-This is a **process-wide** limit (not per-connection). SQLite will try to release cache pages to stay under 1GB total. It's a soft limit — it won't abort queries, just evicts cache more aggressively.

-### Fix 3: Pass context to database queries (medium impact, medium effort)

-Change `.Query()` to `.QueryContext(ctx)`, `.QueryRow()` to `.QueryRowContext(ctx)`, and `.Scan()` to `.ScanContext(ctx)` for all query methods that already receive a `ctx` parameter. This allows HTTP request cancellation to stop the SQLite query.

+### 4. Wire `SetConfig()` in `initDatabase()`

-**Note:** The `stmtCache` from squirrel supports `QueryContext`/`QueryRowContext`. Only methods that already have `ctx` are changed — no signature changes needed.

-**Call sites to update** (methods that have `ctx` and call `.Query()`/`.QueryRow()`/`.Scan()`):

+        cfg := repository.DefaultConfig()

+            cfg.DbCacheSizeMB = dc.CacheSizeMB

+            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB

+            cfg.MaxOpenConnections = dc.MaxOpenConnections

+            cfg.MaxIdleConnections = dc.MaxIdleConnections

+        if dc.ConnectionMaxIdleTimeMins > 0 {

+            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute

-| `jobQuery.go` | `QueryJobs` | 87 | `.Query()` → `.QueryContext(ctx)` |

-| `jobQuery.go` | `CountJobs` | 129 | `.Scan()` → `.ScanContext(ctx)` |

-| `stats.go` | `JobsStatsGrouped` | 233 | `.Query()` → `.QueryContext(ctx)` |

-| `stats.go` | `JobsStats` | 358 | `.QueryRow()` → `.QueryRowContext(ctx)` |

-| `stats.go` | `JobCountGrouped` | 443 | `.Query()` → `.QueryContext(ctx)` |

-| `stats.go` | `AddJobCountGrouped` | 504 | `.Query()` → `.QueryContext(ctx)` |

-| `stats.go` | `AddJobCount` | 569 | `.Scan()` → `.ScanContext(ctx)` |

-| `stats.go` | `jobsStatisticsHistogram` | 758 | `.Query()` → `.QueryContext(ctx)` |

-| `stats.go` | `jobsDurationStatisticsHistogram` | 832 | `.Query()` → `.QueryContext(ctx)` |

-| `stats.go` | `jobsMetricStatisticsHistogram` | 962 | `.Query()` → `.QueryContext(ctx)` |

-| `jobFind.go` | `FindByID` | 174 | `.QueryRow()` → `.QueryRowContext(ctx)` |

-| `jobFind.go` | `FindByJobID` | 220 | `.QueryRow()` → `.QueryRowContext(ctx)` |

-| `job.go` | `CountGroupedJobs` | 410 | `.Scan()` → `.ScanContext(ctx)` (needs ctx added to signature) |

-| `job.go` | `GetJobList` | 751 | `.Query()` → `.QueryContext(ctx)` (needs ctx added to signature) |

-Methods without `ctx` in their signature (e.g., `GetJobList`, `CountGroupedJobs`) can either have `ctx` added or be left for a follow-up. The priority is the methods already accepting `ctx`.

+After setting PRAGMAs, log the effective values so operators can verify:

+cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",

+    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)

+## Example config.json (for 512GB server)

-Reduce `ConnectionMaxIdleTime` from 1 hour to 10 minutes. Idle connections with cached pages are released sooner.

+This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

-All fixes are complementary, independent, and low-risk. No disadvantages to applying all together.

-- `internal/repository/dbConnection.go` — cache_size, soft_heap_limit (Fixes 1 & 2)

-- `internal/repository/jobQuery.go` — QueryContext (Fix 3)

-- `internal/repository/job.go` — QueryContext for other query methods (Fix 3)

-- `internal/repository/stats.go` — QueryContext for stats queries (Fix 3)

-- `internal/repository/config.go` — ConnMaxIdleTime (Fix 4, optional)

+| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |

+| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |

+| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |

+| `internal/config/schema.go` | Add `"db-config"` JSON schema section |

+| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

-2. `go test ./internal/repository/...` — tests pass

-3. Monitor RSS memory before/after on production:

-   - Trigger a heavy query, observe memory spike

-   - Verify memory drops back down within seconds (not hours)

-4. `PRAGMA cache_size;` in sqlite3 CLI on running DB — should show `-200000`

-5. `PRAGMA soft_heap_limit;` — should show `1073741824`

+2. `go test ./internal/repository/... ./internal/config/...` — tests pass

+3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible

+4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI

+5. Check startup log shows effective values


Good point — with an 80GB database on a 512GB server, you'd want the cache to hold a significant portion of the DB. Let me adjust the defaults and example values.


Update `DefaultConfig()`:
```go
DbCacheSizeMB:     200,   // 200MB per connection
DbSoftHeapLimitMB: 1024,  // 1GB process-wide
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     2048,  // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
```

**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.


Update `DefaultConfig()`:
```go
DbCacheSizeMB:     2048,  // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
```

**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     200,   // 200MB per connection
DbSoftHeapLimitMB: 1024,  // 1GB process-wide
```

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values

-DbCacheSizeMB:     200,   // 200MB per connection

-DbSoftHeapLimitMB: 1024,  // 1GB process-wide

+DbCacheSizeMB:     2048,  // 2GB per connection

+DbSoftHeapLimitMB: 16384, // 16GB process-wide

+**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.


Also update the example config and the already-hardcoded values discussion:


## Example config.json (for 512GB server)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

## Example config.json (for 512GB server with 80GB database)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 16384,
      "soft-heap-limit-mb": 131072,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 16GB cache = 128GB max page cache, with a 128GB soft heap limit. The entire 80GB database can be cached in memory. On a 512GB server that's ~25% of RAM.

**Sizing guidance (for documentation):**
- `cache-size-mb`: Set to `DB_size / max-open-connections` to allow the entire DB to be cached. E.g., 80GB DB with 8 connections → 10GB per connection minimum.
- `soft-heap-limit-mb`: Set to total desired SQLite memory budget. Should be ≥ `cache-size-mb × max-open-connections` to avoid cache thrashing.


## Example config.json (for 512GB server with 80GB database)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 16384,
      "soft-heap-limit-mb": 131072,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 16GB cache = 128GB max page cache, with a 128GB soft heap limit. The entire 80GB database can be cached in memory. On a 512GB server that's ~25% of RAM.

**Sizing guidance (for documentation):**
- `cache-size-mb`: Set to `DB_size / max-open-connections` to allow the entire DB to be cached. E.g., 80GB DB with 8 connections → 10GB per connection minimum.
- `soft-heap-limit-mb`: Set to total desired SQLite memory budget. Should be ≥ `cache-size-mb × max-open-connections` to avoid cache thrashing.

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     2048,  // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
```

**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 2048,
      "soft-heap-limit-mb": 16384,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values

-## Example config.json (for 512GB server)

+## Example config.json (for 512GB server with 80GB database)

-This would give: 8 connections × 2GB cache = 16GB max page cache, with a 16GB soft heap limit. On a 512GB server that's ~3% of RAM — plenty of headroom.

+This would give: 8 connections × 16GB cache = 128GB max page cache, with a 128GB soft heap limit. The entire 80GB database can be cached in memory. On a 512GB server that's ~25% of RAM.

+**Sizing guidance (for documentation):**

+- `cache-size-mb`: Set to `DB_size / max-open-connections` to allow the entire DB to be cached. E.g., 80GB DB with 8 connections → 10GB per connection minimum.

+- `soft-heap-limit-mb`: Set to total desired SQLite memory budget. Should be ≥ `cache-size-mb × max-open-connections` to avoid cache thrashing.


Now update the "Current State" section to note these hardcoded values need to be replaced:


## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Current State (already implemented on this branch)

- `_cache_size = -200000` (200MB per connection, hardcoded) — **too low for 80GB DB, will be made configurable**
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded) — **too low, will be made configurable**
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default)
- `MaxOpenConnections = 4` (hardcoded default)
- Context propagation to all query call sites (already done)


## Current State (already implemented on this branch)

- `_cache_size = -200000` (200MB per connection, hardcoded) — **too low for 80GB DB, will be made configurable**
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded) — **too low, will be made configurable**
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default)
- `MaxOpenConnections = 4` (hardcoded default)
- Context propagation to all query call sites (already done)

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented)

- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)
- `MaxOpenConnections = 4` (hardcoded default in `config.go`)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     2048,  // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
```

**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server with 80GB database)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 16384,
      "soft-heap-limit-mb": 131072,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 16GB cache = 128GB max page cache, with a 128GB soft heap limit. The entire 80GB database can be cached in memory. On a 512GB server that's ~25% of RAM.

**Sizing guidance (for documentation):**
- `cache-size-mb`: Set to `DB_size / max-open-connections` to allow the entire DB to be cached. E.g., 80GB DB with 8 connections → 10GB per connection minimum.
- `soft-heap-limit-mb`: Set to total desired SQLite memory budget. Should be ≥ `cache-size-mb × max-open-connections` to avoid cache thrashing.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values

+## Current State (already implemented on this branch)

-- `_cache_size = -200000` (200MB per connection, hardcoded in `dbConnection.go:82`)

-- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded in `setupSqlite()`)

-- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default in `config.go`)

-- `MaxOpenConnections = 4` (hardcoded default in `config.go`)

+- `_cache_size = -200000` (200MB per connection, hardcoded) — **too low for 80GB DB, will be made configurable**

+- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded) — **too low, will be made configurable**

+- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default)

+- `MaxOpenConnections = 4` (hardcoded default)


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


ExitPlanMode

# Make SQLite Memory Limits Configurable via config.json

## Context

Fixes 1-4 for the SQLite memory leak are already implemented on this branch. The hardcoded defaults (200MB cache per connection, 1GB soft heap limit) are conservative. On the production server with 512GB RAM, these could be tuned higher for better query performance. Additionally, `RepositoryConfig` and `SetConfig()` exist but are **never wired up** — there's currently no way to override any repository defaults from config.json.

## Current State (already implemented on this branch)

- `_cache_size = -200000` (200MB per connection, hardcoded) — **too low for 80GB DB, will be made configurable**
- `soft_heap_limit = 1073741824` (1GB process-wide, hardcoded) — **too low, will be made configurable**
- `ConnectionMaxIdleTime = 10 * time.Minute` (hardcoded default)
- `MaxOpenConnections = 4` (hardcoded default)
- Context propagation to all query call sites (already done)

## Problem

`repository.SetConfig()` exists but is never called from `main.go`. The `initDatabase()` function (line 110) just calls `repository.Connect(config.Keys.DB)` directly. There's no `"db-config"` section in `ProgramConfig` or the JSON schema.

## Proposed Changes

### 1. Add SQLite memory fields to `RepositoryConfig`

**File:** `internal/repository/config.go`

Add two new fields with sensible defaults:

```go
type RepositoryConfig struct {
    // ... existing fields ...

    // DbCacheSizeMB is the SQLite page cache size per connection in MB.
    // Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    // and DbCacheSizeMB=200, total page cache is up to 800MB.
    // Default: 200 (MB)
    DbCacheSizeMB int

    // DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    // SQLite will try to release cache pages to stay under this limit.
    // It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    // Default: 1024 (1GB)
    DbSoftHeapLimitMB int
}
```

Update `DefaultConfig()`:
```go
DbCacheSizeMB:     2048,  // 2GB per connection
DbSoftHeapLimitMB: 16384, // 16GB process-wide
```

**Rationale for defaults:** With an 80GB production database on a 512GB server, we want the cache to hold a significant portion of the DB. At 4 connections × 2GB = 8GB default page cache, plus 16GB soft heap limit. The previous 200MB/1GB hardcoded values were too conservative and would hurt query performance by forcing excessive cache eviction. These defaults use ~5% of a 512GB server — still safe for smaller machines, while enabling good performance on production.

### 2. Use config values in `Connect()` and `setupSqlite()`

**File:** `internal/repository/dbConnection.go`

In `Connect()`, replace the hardcoded cache_size:
```go
cacheSizeKiB := repoConfig.DbCacheSizeMB * 1024  // Convert MB to KiB
connectionURLParams.Add("_cache_size", fmt.Sprintf("-%d", cacheSizeKiB))
```

Change `setupSqlite()` to accept the config and use it for soft_heap_limit:
```go
func setupSqlite(db *sql.DB, cfg *RepositoryConfig) error {
    pragmas := []string{
        "temp_store = memory",
        fmt.Sprintf("soft_heap_limit = %d", cfg.DbSoftHeapLimitMB*1024*1024),
    }
    // ...
}
```

Update the call site in `Connect()`:
```go
err = setupSqlite(dbHandle.DB, &opts)  // was: setupSqlite(dbHandle.DB)
```

### 3. Add `"db-config"` section to `ProgramConfig` and JSON schema

**File:** `internal/config/config.go`

Add a new struct and field to `ProgramConfig`:

```go
type DbConfig struct {
    CacheSizeMB       int `json:"cache-size-mb"`
    SoftHeapLimitMB   int `json:"soft-heap-limit-mb"`
    MaxOpenConnections int `json:"max-open-connections"`
    MaxIdleConnections int `json:"max-idle-connections"`
    ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
}

type ProgramConfig struct {
    // ... existing fields ...
    DbConfig *DbConfig `json:"db-config"`
}
```

**File:** `internal/config/schema.go`

Add the schema section for validation.

### 4. Wire `SetConfig()` in `initDatabase()`

**File:** `cmd/cc-backend/main.go`

```go
func initDatabase() error {
    if config.Keys.DbConfig != nil {
        cfg := repository.DefaultConfig()
        dc := config.Keys.DbConfig
        if dc.CacheSizeMB > 0 {
            cfg.DbCacheSizeMB = dc.CacheSizeMB
        }
        if dc.SoftHeapLimitMB > 0 {
            cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
        }
        if dc.MaxOpenConnections > 0 {
            cfg.MaxOpenConnections = dc.MaxOpenConnections
        }
        if dc.MaxIdleConnections > 0 {
            cfg.MaxIdleConnections = dc.MaxIdleConnections
        }
        if dc.ConnectionMaxIdleTimeMins > 0 {
            cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
        }
        repository.SetConfig(cfg)
    }
    repository.Connect(config.Keys.DB)
    return nil
}
```

### 5. Log effective values on startup

**File:** `internal/repository/dbConnection.go`

After setting PRAGMAs, log the effective values so operators can verify:
```go
cclog.Infof("SQLite config: cache_size=%dMB/conn, soft_heap_limit=%dMB, max_conns=%d",
    repoConfig.DbCacheSizeMB, repoConfig.DbSoftHeapLimitMB, repoConfig.MaxOpenConnections)
```

## Example config.json (for 512GB server with 80GB database)

```json
{
  "main": {
    "db": "./var/job.db",
    "db-config": {
      "cache-size-mb": 16384,
      "soft-heap-limit-mb": 131072,
      "max-open-connections": 8,
      "max-idle-time-minutes": 30
    }
  }
}
```

This would give: 8 connections × 16GB cache = 128GB max page cache, with a 128GB soft heap limit. The entire 80GB database can be cached in memory. On a 512GB server that's ~25% of RAM.

**Sizing guidance (for documentation):**
- `cache-size-mb`: Set to `DB_size / max-open-connections` to allow the entire DB to be cached. E.g., 80GB DB with 8 connections → 10GB per connection minimum.
- `soft-heap-limit-mb`: Set to total desired SQLite memory budget. Should be ≥ `cache-size-mb × max-open-connections` to avoid cache thrashing.

## Files to Modify

| File | Changes |
|------|---------|
| `internal/repository/config.go` | Add `DbCacheSizeMB`, `DbSoftHeapLimitMB` fields + defaults |
| `internal/repository/dbConnection.go` | Use config values instead of hardcoded; pass config to `setupSqlite`; add startup log |
| `internal/config/config.go` | Add `DbConfig` struct and field to `ProgramConfig` |
| `internal/config/schema.go` | Add `"db-config"` JSON schema section |
| `cmd/cc-backend/main.go` | Wire `SetConfig()` in `initDatabase()` |

## Verification

1. `go build ./...` — compiles
2. `go test ./internal/repository/... ./internal/config/...` — tests pass
3. Without `db-config` in config.json: defaults apply (200MB cache, 1GB heap) — backwards compatible
4. With `db-config`: verify with `PRAGMA cache_size;` and `PRAGMA soft_heap_limit;` in sqlite3 CLI
5. Check startup log shows effective values


Error: The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.


Understood — waiting for your input on how to proceed.