chore: add clickhouse as db for logs on agent and search

backend/internal/config/types.go

@@ -9,4 +9,5 @@ type Databases struct {
 	Clickhouse_host     string `yaml:"clickhouse_host"`
 	Clickhouse_user     string `yaml:"clickhouse_user"`
 	Clickhouse_password string `yaml:"clickhouse_password"`
+	Clickhouse_database string `yaml:"clickhouse_database"`
 }

backend/internal/handlers/logs.go

@@ -0,0 +1,229 @@
+package handlers
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository"
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage"
+	"github.com/gin-gonic/gin"
+)
+
+type LogHandlers struct {
+	LogRepo *repository.LogRepository
+}
+
+func NewLogHandlers(logRepo *repository.LogRepository) *LogHandlers {
+	return &LogHandlers{LogRepo: logRepo}
+}
+
+type InsertLogRequest struct {
+	Timestamp time.Time `json:"timestamp"`
+	Level     string    `json:"level" binding:"required"`
+	Service   string    `json:"service" binding:"required"`
+	Agent     string    `json:"agent" binding:"required"`
+	Message   string    `json:"message" binding:"required"`
+}
+
+// @Summary Insert log entry
+// @Description Inserts a single log entry into ClickHouse
+// @Tags logs
+// @Accept json
+// @Produce json
+// @Param body body InsertLogRequest true "Log entry"
+// @Success 201 {object} map[string]string
+// @Router /logs [post]
+func (lh *LogHandlers) Insert(c *gin.Context) {
+	var req InsertLogRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	if req.Timestamp.IsZero() {
+		req.Timestamp = time.Now()
+	}
+
+	log := storage.LogEntry{
+		Timestamp: req.Timestamp,
+		Level:     req.Level,
+		Service:   req.Service,
+		Agent:     req.Agent,
+		Message:   req.Message,
+	}
+
+	if err := lh.LogRepo.Insert(c.Request.Context(), log); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to insert log"})
+		return
+	}
+
+	c.JSON(http.StatusCreated, gin.H{"status": "ok"})
+}
+
+type InsertLogsRequest struct {
+	Logs []InsertLogRequest `json:"logs" binding:"required"`
+}
+
+// @Summary Insert log entries (batch)
+// @Description Inserts multiple log entries into ClickHouse
+// @Tags logs
+// @Accept json
+// @Produce json
+// @Param body body InsertLogsRequest true "Log entries"
+// @Success 201 {object} map[string]string
+// @Router /logs/batch [post]
+func (lh *LogHandlers) InsertBatch(c *gin.Context) {
+	var req InsertLogsRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	logs := make([]storage.LogEntry, len(req.Logs))
+	for i, l := range req.Logs {
+		if l.Timestamp.IsZero() {
+			l.Timestamp = time.Now()
+		}
+		logs[i] = storage.LogEntry{
+			Timestamp: l.Timestamp,
+			Level:     l.Level,
+			Service:   l.Service,
+			Agent:     l.Agent,
+			Message:   l.Message,
+		}
+	}
+
+	if err := lh.LogRepo.InsertBatch(c.Request.Context(), logs); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to insert logs"})
+		return
+	}
+
+	c.JSON(http.StatusCreated, gin.H{"status": "ok", "count": len(logs)})
+}
+
+type SearchLogsRequest struct {
+	Level   string `form:"level"`
+	Service string `form:"service"`
+	Agent   string `form:"agent"`
+	DateFrom string `form:"date_from"`
+	DateTo   string `form:"date_to"`
+	Limit   int    `form:"limit"`
+	Offset  int    `form:"offset"`
+}
+
+// @Summary Search logs
+// @Description Searches logs with various filters
+// @Tags logs
+// @Produce json
+// @Param level query string false "Log level (INFO, WARNING, ERROR, FATAL)"
+// @Param service query string false "Service name"
+// @Param agent query string false "Agent name"
+// @Param date_from query string false "Date from (RFC3339)"
+// @Param date_to query string false "Date to (RFC3339)"
+// @Param limit query int false "Limit results" default(100)
+// @Param offset query int false "Offset results" default(0)
+// @Success 200 {array} storage.LogEntry
+// @Router /logs [get]
+func (lh *LogHandlers) Search(c *gin.Context) {
+	var req SearchLogsRequest
+	if err := c.ShouldBindQuery(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	filter := repository.LogFilter{
+		Level:   req.Level,
+		Service: req.Service,
+		Agent:   req.Agent,
+		Limit:   req.Limit,
+		Offset:  req.Offset,
+	}
+
+	if req.DateFrom != "" {
+		if t, err := time.Parse(time.RFC3339, req.DateFrom); err == nil {
+			filter.DateFrom = t
+		}
+	}
+
+	if req.DateTo != "" {
+		if t, err := time.Parse(time.RFC3339, req.DateTo); err == nil {
+			filter.DateTo = t
+		}
+	}
+
+	if filter.Limit <= 0 {
+		filter.Limit = 100
+	}
+
+	logs, err := lh.LogRepo.Search(c.Request.Context(), filter)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to search logs"})
+		return
+	}
+
+	c.JSON(http.StatusOK, logs)
+}
+
+// @Summary Get distinct services
+// @Description Returns list of all unique service names in logs
+// @Tags logs
+// @Produce json
+// @Success 200 {array} string
+// @Router /logs/services [get]
+func (lh *LogHandlers) GetServices(c *gin.Context) {
+	services, err := lh.LogRepo.GetDistinctServices(c.Request.Context())
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get services"})
+		return
+	}
+
+	if services == nil {
+		services = []string{}
+	}
+
+	c.JSON(http.StatusOK, services)
+}
+
+// @Summary Get distinct agents
+// @Description Returns list of all unique agent names in logs
+// @Tags logs
+// @Produce json
+// @Success 200 {array} string
+// @Router /logs/agents [get]
+func (lh *LogHandlers) GetAgents(c *gin.Context) {
+	agents, err := lh.LogRepo.GetDistinctAgents(c.Request.Context())
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get agents"})
+		return
+	}
+
+	if agents == nil {
+		agents = []string{}
+	}
+
+	c.JSON(http.StatusOK, agents)
+}
+
+// @Summary Get distinct log levels
+// @Description Returns list of all unique log levels in logs
+// @Tags logs
+// @Produce json
+// @Success 200 {array} string
+// @Router /logs/levels [get]
+func (lh *LogHandlers) GetLevels(c *gin.Context) {
+	levels, err := lh.LogRepo.GetDistinctLevels(c.Request.Context())
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get levels"})
+		return
+	}
+
+	if levels == nil {
+		levels = []string{}
+	}
+
+	c.JSON(http.StatusOK, levels)
+}
+
+// Ensure context is used
+var _ = context.Background

backend/internal/repository/log_repository.go

@@ -0,0 +1,178 @@
+package repository
+
+import (
+	"context"
+	"time"
+
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+type LogRepository struct {
+	Conn driver.Conn
+}
+
+func NewLogRepository(conn driver.Conn) *LogRepository {
+	return &LogRepository{Conn: conn}
+}
+
+func (r *LogRepository) Init(ctx context.Context) error {
+	return r.Conn.Exec(ctx, storage.CreateLogsTable)
+}
+
+func (r *LogRepository) Insert(ctx context.Context, log storage.LogEntry) error {
+	return r.Conn.Exec(ctx, `
+		INSERT INTO logs (timestamp, level, service, agent, message)
+		VALUES ($1, $2, $3, $4, $5)
+	`, log.Timestamp, log.Level, log.Service, log.Agent, log.Message)
+}
+
+func (r *LogRepository) InsertBatch(ctx context.Context, logs []storage.LogEntry) error {
+	batch, err := r.Conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, level, service, agent, message)")
+	if err != nil {
+		return err
+	}
+
+	for _, log := range logs {
+		if err := batch.Append(log.Timestamp, log.Level, log.Service, log.Agent, log.Message); err != nil {
+			return err
+		}
+	}
+
+	return batch.Send()
+}
+
+type LogFilter struct {
+	Level   string
+	Service string
+	Agent   string
+	DateFrom time.Time
+	DateTo   time.Time
+	Limit   int
+	Offset  int
+}
+
+func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage.LogEntry, error) {
+	query := "SELECT timestamp, level, service, agent, message FROM logs WHERE 1=1"
+	args := make([]interface{}, 0)
+	argIdx := 1
+
+	if filter.Level != "" {
+		query += " AND level = $" + string(rune('0'+argIdx))
+		args = append(args, filter.Level)
+		argIdx++
+	}
+
+	if filter.Service != "" {
+		query += " AND service = $" + string(rune('0'+argIdx))
+		args = append(args, filter.Service)
+		argIdx++
+	}
+
+	if filter.Agent != "" {
+		query += " AND agent = $" + string(rune('0'+argIdx))
+		args = append(args, filter.Agent)
+		argIdx++
+	}
+
+	if !filter.DateFrom.IsZero() {
+		query += " AND timestamp >= $" + string(rune('0'+argIdx))
+		args = append(args, filter.DateFrom)
+		argIdx++
+	}
+
+	if !filter.DateTo.IsZero() {
+		query += " AND timestamp <= $" + string(rune('0'+argIdx))
+		args = append(args, filter.DateTo)
+		argIdx++
+	}
+
+	query += " ORDER BY timestamp DESC"
+
+	if filter.Limit > 0 {
+		query += " LIMIT $" + string(rune('0'+argIdx))
+		args = append(args, filter.Limit)
+		argIdx++
+	} else {
+		query += " LIMIT 100"
+	}
+
+	if filter.Offset > 0 {
+		query += " OFFSET $" + string(rune('0'+argIdx))
+		args = append(args, filter.Offset)
+	}
+
+	rows, err := r.Conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var logs []storage.LogEntry
+	for rows.Next() {
+		var log storage.LogEntry
+		if err := rows.Scan(&log.Timestamp, &log.Level, &log.Service, &log.Agent, &log.Message); err != nil {
+			return nil, err
+		}
+		logs = append(logs, log)
+	}
+
+	return logs, rows.Err()
+}
+
+func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, error) {
+	rows, err := r.Conn.Query(ctx, "SELECT DISTINCT service FROM logs ORDER BY service")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var services []string
+	for rows.Next() {
+		var service string
+		if err := rows.Scan(&service); err != nil {
+			return nil, err
+		}
+		services = append(services, service)
+	}
+
+	return services, rows.Err()
+}
+
+func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error) {
+	rows, err := r.Conn.Query(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var agents []string
+	for rows.Next() {
+		var agent string
+		if err := rows.Scan(&agent); err != nil {
+			return nil, err
+		}
+		agents = append(agents, agent)
+	}
+
+	return agents, rows.Err()
+}
+
+func (r *LogRepository) GetDistinctLevels(ctx context.Context) ([]string, error) {
+	rows, err := r.Conn.Query(ctx, "SELECT DISTINCT level FROM logs ORDER BY level")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var levels []string
+	for rows.Next() {
+		var level string
+		if err := rows.Scan(&level); err != nil {
+			return nil, err
+		}
+		levels = append(levels, level)
+	}
+
+	return levels, rows.Err()
+}

backend/internal/storage/clickhouse.go

@@ -1 +1,47 @@
 package storage
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+type ClickHouseConfig struct {
+	Host     string
+	User     string
+	Password string
+	Database string
+}
+
+func OpenClickHouse(cfg ClickHouseConfig) (driver.Conn, error) {
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.Host},
+		Auth: clickhouse.Auth{
+			Database: cfg.Database,
+			Username: cfg.User,
+			Password: cfg.Password,
+		},
+		Settings: clickhouse.Settings{
+			"max_execution_time": 60,
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+		DialTimeout:      30,
+		MaxOpenConns:     10,
+		MaxIdleConns:     5,
+		ConnMaxLifetime:  3600,
+		ConnOpenStrategy: clickhouse.ConnOpenInOrder,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse connect: %w", err)
+	}
+
+	if err := conn.Ping(context.Background()); err != nil {
+		return nil, fmt.Errorf("clickhouse ping: %w", err)
+	}
+
+	return conn, nil
+}

backend/internal/storage/log_entry.go

@@ -0,0 +1,11 @@
+package storage
+
+import "time"
+
+type LogEntry struct {
+	Timestamp time.Time `ch:"timestamp"`
+	Level     string    `ch:"level"`
+	Service   string    `ch:"service"`
+	Agent     string    `ch:"agent"`
+	Message   string    `ch:"message"`
+}

backend/internal/storage/migrations.go

@@ -1,3 +1,16 @@
 package storage
 
 const CreateSqlite = ``
+
+const CreateLogsTable = `
+CREATE TABLE IF NOT EXISTS logs (
+    timestamp DateTime64(3) DEFAULT now(),
+    level     LowCardinality(String),
+    service   LowCardinality(String),
+    agent     LowCardinality(String),
+    message   String
+) ENGINE = MergeTree()
+ORDER BY (timestamp, level, service, agent)
+TTL timestamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192
+`