chore: add logparser logic for agent and add parsed log to clickhouse

agent/internal/buffer/buffer.go

@@ -0,0 +1,161 @@
+package buffer
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// BufferedLog represents a log entry stored for later delivery
+type BufferedLog struct {
+	ID        int64
+	Service   string
+	Message   string
+	CreatedAt time.Time
+}
+
+// LogBuffer provides SQLite-backed log buffering
+type LogBuffer struct {
+	db *sql.DB
+}
+
+// NewLogBuffer creates a new log buffer with the given database path
+func NewLogBuffer(dbPath string) (*LogBuffer, error) {
+	db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000")
+	if err != nil {
+		return nil, fmt.Errorf("failed to open database: %w", err)
+	}
+
+	// Create table if not exists
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS buffered_logs (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			service TEXT NOT NULL,
+			message TEXT NOT NULL,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+		)
+	`)
+	if err != nil {
+		_ = db.Close()
+		return nil, fmt.Errorf("failed to create table: %w", err)
+	}
+
+	// Create index for efficient ordering
+	_, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_created_at ON buffered_logs(created_at ASC)`)
+
+	return &LogBuffer{db: db}, nil
+}
+
+// Close closes the database connection
+func (b *LogBuffer) Close() error {
+	return b.db.Close()
+}
+
+// Store stores a log entry in the buffer
+func (b *LogBuffer) Store(service, message string) error {
+	_, err := b.db.Exec(
+		"INSERT INTO buffered_logs (service, message) VALUES (?, ?)",
+		service, message,
+	)
+	return err
+}
+
+// StoreBatch stores multiple log entries in a single transaction
+func (b *LogBuffer) StoreBatch(entries []BufferedLog) error {
+	tx, err := b.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	stmt, err := tx.Prepare("INSERT INTO buffered_logs (service, message) VALUES (?, ?)")
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	for _, entry := range entries {
+		if _, err := stmt.Exec(entry.Service, entry.Message); err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+// GetPending retrieves pending logs in order of arrival, limited to batchSize
+func (b *LogBuffer) GetPending(batchSize int) ([]BufferedLog, error) {
+	rows, err := b.db.Query(
+		"SELECT id, service, message, created_at FROM buffered_logs ORDER BY created_at ASC LIMIT ?",
+		batchSize,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var logs []BufferedLog
+	for rows.Next() {
+		var log BufferedLog
+		var createdAt string
+		if err := rows.Scan(&log.ID, &log.Service, &log.Message, &createdAt); err != nil {
+			return nil, err
+		}
+		log.CreatedAt, _ = time.Parse("2006-01-02 15:04:05", createdAt)
+		logs = append(logs, log)
+	}
+
+	return logs, rows.Err()
+}
+
+// Delete removes a log entry from the buffer after successful delivery
+func (b *LogBuffer) Delete(id int64) error {
+	_, err := b.db.Exec("DELETE FROM buffered_logs WHERE id = ?", id)
+	return err
+}
+
+// DeleteBatch removes multiple log entries after successful delivery
+func (b *LogBuffer) DeleteBatch(ids []int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	tx, err := b.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	for _, id := range ids {
+		if _, err := tx.Exec("DELETE FROM buffered_logs WHERE id = ?", id); err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+// Count returns the number of buffered logs
+func (b *LogBuffer) Count() (int, error) {
+	var count int
+	err := b.db.QueryRow("SELECT COUNT(*) FROM buffered_logs").Scan(&count)
+	return count, err
+}
+
+// Clear removes all buffered logs
+func (b *LogBuffer) Clear() error {
+	_, err := b.db.Exec("DELETE FROM buffered_logs")
+	return err
+}
+
+// FlushToJSON exports buffered logs to JSON format for debugging
+func (b *LogBuffer) FlushToJSON() ([]byte, error) {
+	logs, err := b.GetPending(1000)
+	if err != nil {
+		return nil, err
+	}
+	return json.MarshalIndent(logs, "", "  ")
+}

agent/internal/config/config.go

@@ -1,17 +1,25 @@
 package config
 
 import (
+	"fmt"
 	"os"
 
 	"gopkg.in/yaml.v3"
 )
 
+type ServiceConfig struct {
+	Name string  `yaml:"name"`
+	Type string  `yaml:"type"`
+	Path *string `yaml:"path"`
+}
+
 type AgentConfig struct {
-	BackendURL        string `yaml:"backend_url"`
-	GRPCURL           string `yaml:"grpc_url"`
-	RegistrationToken string `yaml:"registration_token"`
-	Label             string `yaml:"label"`
-	CertDir           string `yaml:"cert_dir"`
+	BackendURL        string          `yaml:"backend_url"`
+	GRPCURL           string          `yaml:"grpc_url"`
+	RegistrationToken string          `yaml:"registration_token"`
+	Label             string          `yaml:"label"`
+	CertDir           string          `yaml:"cert_dir"`
+	Services          []ServiceConfig `yaml:"services"`
 }
 
 func Load(path string) (*AgentConfig, error) {
@@ -31,3 +39,19 @@ func Load(path string) (*AgentConfig, error) {
 
 	return &cfg, nil
 }
+
+func LoadFromString(data string) (*AgentConfig, error) {
+	var cfg AgentConfig
+	if err := yaml.Unmarshal([]byte(data), &cfg); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
+
+func validateConfigPath(path string) error {
+	if _, err := os.Stat(path); err != nil {
+		return fmt.Errorf("config file not found: %w", err)
+	}
+	return nil
+}

agent/internal/logger/logger.go

@@ -0,0 +1,27 @@
+package logger
+
+import (
+	"log/slog"
+	"os"
+)
+
+type Logger struct {
+	*slog.Logger
+}
+
+func New(debug bool) *Logger {
+	var level slog.Level
+	if debug {
+		level = slog.LevelDebug
+	} else {
+		level = slog.LevelInfo
+	}
+
+	handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: level,
+	})
+
+	return &Logger{
+		Logger: slog.New(handler),
+	}
+}

agent/internal/logsource/file/impl.go

@@ -0,0 +1,45 @@
+package file
+
+import (
+	"errors"
+	"os"
+
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource"
+	"github.com/hpcloud/tail"
+)
+
+var _ logsource.LogSource = new(FileLogSource)
+
+type FileLogSource struct {
+	*tail.Tail
+}
+
+func New(filepath string) (fls *FileLogSource, err error) {
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		if err := os.WriteFile(filepath, []byte{}, 0600); err != nil {
+			return nil, err
+		}
+	}
+	t, err := tail.TailFile(filepath, tail.Config{
+		Follow: true,
+		Location: &tail.SeekInfo{
+			Offset: 100,
+			Whence: 2,
+		},
+	})
+	if err != nil {
+		return
+	}
+	return &FileLogSource{t}, nil
+}
+func (f *FileLogSource) ReadLine() (string, error) {
+	select {
+	case <-f.Dead():
+		return "", errors.Join(logsource.ErrDead, f.Err())
+	case line := <-f.Lines:
+		return line.Text, line.Err
+	}
+}
+func (f *FileLogSource) Close() error {
+	return f.Stop()
+}

agent/internal/logsource/interface.go

@@ -0,0 +1,10 @@
+package logsource
+
+import "errors"
+
+type LogSource interface {
+	ReadLine() (string, error)
+	Close() error
+}
+
+var ErrDead = errors.New("shouldn't continue to read that")

agent/internal/logsource/journald/impl.go

@@ -0,0 +1,55 @@
+package journald
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os/exec"
+	"syscall"
+
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/config"
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource"
+)
+
+var _ logsource.LogSource = new(JournaldLogSource)
+
+type JournaldLogSource struct {
+	cmd           *exec.Cmd
+	stdout        io.ReadCloser
+	stdoutscanner *bufio.Scanner
+}
+
+// ReadLine implements logsource.LogSource.
+func (j *JournaldLogSource) ReadLine() (string, error) {
+	if j.stdoutscanner.Scan() {
+		return j.stdoutscanner.Text(), nil
+	} else {
+		if j.stdoutscanner.Err() == nil {
+			return "", fmt.Errorf("%w: %s", logsource.ErrDead, io.EOF)
+		}
+		return "", j.stdoutscanner.Err()
+	}
+}
+func (j *JournaldLogSource) Close() error {
+	_ = j.cmd.Process.Signal(syscall.SIGTERM)
+	return j.cmd.Wait()
+}
+
+func New(cfg config.ServiceConfig, logdir string) (*JournaldLogSource, error) {
+	args := make([]string, 0)
+	if cfg.Path != nil {
+		args = append(args, "-u", *cfg.Path)
+	}
+	args = append(args, "-f", "-n", "0", "-o", "short", "--no-pager", "--directory", logdir)
+	cmd := exec.Command("journalctl", args...) //nolint:gosec
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	err = cmd.Start()
+	if err != nil {
+		return nil, err
+	}
+	stdoutscanner := bufio.NewScanner(stdout)
+	return &JournaldLogSource{cmd, stdout, stdoutscanner}, nil
+}