diff --git a/go.mod b/go.mod index 685605f..59306d4 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.32.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect modernc.org/libc v1.67.6 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 1a5074d..4918414 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 447b70f..d231f2e 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -1,8 +1,12 @@ package logger import ( + "io" "log/slog" "os" + "path/filepath" + + "gopkg.in/natefinch/lumberjack.v2" ) type Logger struct { @@ -10,13 +14,28 @@ type Logger struct { } func New(debug bool) *Logger { + logDir := "/var/log/banforge" + if err := os.MkdirAll(logDir, 0755); err != nil { + return nil + } + + fileWriter := &lumberjack.Logger{ + Filename: filepath.Join(logDir, "banforge.log"), + MaxSize: 500, + MaxBackups: 3, + MaxAge: 28, + Compress: true, + } + var level slog.Level if debug { level = slog.LevelDebug } else { level = slog.LevelInfo } - handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + multiWriter := io.MultiWriter(fileWriter, os.Stdout) + + handler := slog.NewTextHandler(multiWriter, &slog.HandlerOptions{ Level: level, }) diff --git a/internal/storage/db.go b/internal/storage/db.go index a76a33b..f825bc6 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -21,8 +21,11 @@ type DB struct { func NewDB() (*DB, error) { db, err := sql.Open( "sqlite", - "/var/lib/banforge/storage.db?mode=rwc&_journal_mode=WAL&_busy_timeout=10000&cache=shared", + "/var/lib/banforge/storage.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(30000)&_pragma=synchronous(NORMAL)", ) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + db.SetConnMaxLifetime(0) if err != nil { return nil, err } diff --git a/internal/storage/db_test.go b/internal/storage/db_test.go index af52448..bc3d613 100644 --- a/internal/storage/db_test.go +++ b/internal/storage/db_test.go @@ -37,7 +37,7 @@ func createTestDBStruct(t *testing.T) *DB { } filePath := filepath.Join(tmpDir, "test.db") - sqlDB, err := sql.Open("sqlite3", filePath) + sqlDB, err := sql.Open("sqlite", filePath) if err != nil { t.Fatal(err) } diff --git a/internal/storage/writer.go b/internal/storage/writer.go index 479843b..dc03e08 100644 --- a/internal/storage/writer.go +++ b/internal/storage/writer.go @@ -5,18 +5,76 @@ import ( ) func Write(db *DB, resultCh <-chan *LogEntry) { - for result := range resultCh { - _, err := db.db.Exec( - "INSERT INTO requests (service, ip, path, method, status, created_at) VALUES (?, ?, ?, ?, ?, ?)", - result.Service, - result.IP, - result.Path, - result.Method, - result.Status, - time.Now().Format(time.RFC3339), - ) + const batchSize = 100 + const flushInterval = 1 * time.Second + + batch := make([]*LogEntry, 0, batchSize) + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + + flush := func() { + if len(batch) == 0 { + return + } + + tx, err := db.db.Begin() if err != nil { - db.logger.Error("Failed to write to database", "error", err) + db.logger.Error("Failed to begin transaction", "error", err) + return + } + + stmt, err := tx.Prepare( + "INSERT INTO requests (service, ip, path, method, status, created_at) VALUES (?, ?, ?, ?, ?, ?)") + if err != nil { + tx.Rollback() + db.logger.Error("Failed to prepare statement", "error", err) + return + } + defer func() { + err := stmt.Close() + if err != nil { + db.logger.Error("Failed to close statement", "error", err) + } + }() + + for _, entry := range batch { + _, err := stmt.Exec( + entry.Service, + entry.IP, + entry.Path, + entry.Method, + entry.Status, + time.Now().Format(time.RFC3339), + ) + if err != nil { + db.logger.Error("Failed to insert entry", "error", err) + } + } + + if err := tx.Commit(); err != nil { + db.logger.Error("Failed to commit transaction", "error", err) + } else { + db.logger.Debug("Flushed batch", "count", len(batch)) + } + + batch = batch[:0] + } + + for { + select { + case result, ok := <-resultCh: + if !ok { + flush() + return + } + + batch = append(batch, result) + if len(batch) >= batchSize { + flush() + } + + case <-ticker.C: + flush() } } } diff --git a/internal/storage/writer_test.go b/internal/storage/writer_test.go index d216d54..cb2aca7 100644 --- a/internal/storage/writer_test.go +++ b/internal/storage/writer_test.go @@ -28,7 +28,7 @@ func TestWrite(t *testing.T) { close(resultCh) - time.Sleep(100 * time.Millisecond) + time.Sleep(200 * time.Millisecond) err = d.db.QueryRow("SELECT ip FROM requests LIMIT 1").Scan(&ip) if err != nil {