chore: add logparser logic for agent and add parsed log to clickhouse

backend/internal/grpcsrv/collector/collector.go

@@ -0,0 +1,180 @@
+package collector
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"sync"
+	"time"
+
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository"
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage"
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto/proto"
+	"google.golang.org/grpc/metadata"
+)
+
+type Collector struct {
+	proto.UnimplementedCollectorServer
+	logRepo       *repository.LogRepository
+	agents        map[string]*Agent
+	mu            sync.RWMutex
+	batchSize     int
+	flushInterval time.Duration
+}
+
+type Agent struct {
+	ID          string
+	Label       string
+	Services    []string
+	ConnectedAt time.Time
+}
+
+func New(logRepo *repository.LogRepository) *Collector {
+	return &Collector{
+		logRepo:       logRepo,
+		agents:        make(map[string]*Agent),
+		batchSize:     100,
+		flushInterval: 2 * time.Second,
+	}
+}
+
+func (c *Collector) Stream(stream proto.Collector_StreamServer) error {
+	md, ok := metadata.FromIncomingContext(stream.Context())
+	if !ok {
+		return fmt.Errorf("no metadata in context")
+	}
+
+	whoamiVals := md["whoami"]
+	if len(whoamiVals) == 0 {
+		return fmt.Errorf("whoami metadata missing")
+	}
+	agentName := whoamiVals[0]
+
+	serviceVals := md["service"]
+	if len(serviceVals) == 0 {
+		return fmt.Errorf("service metadata missing")
+	}
+	service := serviceVals[0]
+
+	servicesVals := md["services"]
+	var services []string
+	if len(servicesVals) > 0 {
+		services = servicesVals
+	}
+
+	// Register agent
+	c.mu.Lock()
+	c.agents[agentName] = &Agent{
+		ID:          agentName,
+		Label:       agentName,
+		Services:    services,
+		ConnectedAt: time.Now(),
+	}
+	c.mu.Unlock()
+
+	defer func() {
+		c.mu.Lock()
+		delete(c.agents, agentName)
+		c.mu.Unlock()
+	}()
+
+	log.Printf("Agent %s connected, streaming logs for service: %s", agentName, service)
+
+	// If no ClickHouse, just consume the stream without storing
+	if c.logRepo == nil {
+		log.Printf("Warning: logRepo is nil, consuming logs without storing for agent %s", agentName)
+		for {
+			_, err := stream.Recv()
+			if err == io.EOF {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("failed to receive: %w", err)
+			}
+		}
+	}
+
+	// Channels for communication with recv goroutine
+	recvCh := make(chan *proto.CollectorRequest, 1)
+	errCh := make(chan error, 1)
+
+	// Goroutine that blocks on Recv
+	go func() {
+		for {
+			req, err := stream.Recv()
+			if err != nil {
+				errCh <- err
+				return
+			}
+			recvCh <- req
+		}
+	}()
+
+	// Buffer for batch inserts
+	var batch []storage.LogEntry
+	ticker := time.NewTicker(c.flushInterval)
+	defer ticker.Stop()
+
+	flush := func() error {
+		if len(batch) == 0 {
+			return nil
+		}
+		if err := c.logRepo.InsertBatch(stream.Context(), batch); err != nil {
+			log.Printf("Failed to insert batch for agent %s, service %s: %v", agentName, service, err)
+			return err
+		}
+		log.Printf("Flushed %d logs for agent %s, service %s", len(batch), agentName, service)
+		batch = batch[:0]
+		return nil
+	}
+
+	for {
+		select {
+		case <-stream.Context().Done():
+			// Context cancelled, flush remaining
+			_ = flush()
+			return stream.Context().Err()
+		case <-ticker.C:
+			if err := flush(); err != nil {
+				return err
+			}
+		case req := <-recvCh:
+			batch = append(batch, storage.LogEntry{
+				Timestamp: time.Now(),
+				Level:     "info",
+				Service:   service,
+				Agent:     agentName,
+				Message:   req.Message,
+			})
+
+			if len(batch) >= c.batchSize {
+				if err := flush(); err != nil {
+					return err
+				}
+			}
+		case err := <-errCh:
+			if err == io.EOF {
+				// Client closed stream
+				return flush()
+			}
+			return fmt.Errorf("failed to receive: %w", err)
+		}
+	}
+}
+
+func (c *Collector) GetAgent(name string) (*Agent, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	a, ok := c.agents[name]
+	return a, ok
+}
+
+func (c *Collector) Agents() []*Agent {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	result := make([]*Agent, 0, len(c.agents))
+	for _, a := range c.agents {
+		result = append(result, a)
+	}
+	return result
+}

backend/internal/handlers/agents.go

@@ -1,41 +1,44 @@
 package handlers
 
 import (
-	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/commander"
+	"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/collector"
 	"github.com/gin-gonic/gin"
 	"net/http"
 )
 
 type AgentsGroup struct {
 	*Handlers
-	cmder *commander.Commander
+	collector *collector.Collector
 }
 
-func NewAgentsGroup(h *Handlers, cmder *commander.Commander) AgentsGroup {
-	return AgentsGroup{Handlers: h, cmder: cmder}
+func NewAgentsGroup(h *Handlers, coll *collector.Collector) AgentsGroup {
+	return AgentsGroup{Handlers: h, collector: coll}
 }
 
 type AgentInfo struct {
-	Token    string   `json:"token"`
-	Label    string   `json:"label"`
-	Services []string `json:"services"`
+	Token       string   `json:"token"`
+	Label       string   `json:"label"`
+	Services    []string `json:"services"`
+	ConnectedAt string   `json:"connected_at"`
 }
 
 // @Summary Get connected agents
-// @Description Returns a list of all agents currently connected via gRPC streaming
+// @Description Returns a list of all agents currently connected via Collector (log streaming)
 // @Tags agents
 // @Produce json
 // @Success 200 {array} AgentInfo
 // @Router /agents [get]
 func (ag *AgentsGroup) List(c *gin.Context) {
 	agents := make([]AgentInfo, 0)
-	// iterate over the commander's agents map
-	for _, agent := range ag.cmder.Agents() {
+
+	for _, agent := range ag.collector.Agents() {
 		agents = append(agents, AgentInfo{
-			Token:    agent.Token,
-			Label:    agent.Label,
-			Services: agent.Services,
+			Token:       agent.ID,
+			Label:       agent.Label,
+			Services:    agent.Services,
+			ConnectedAt: agent.ConnectedAt.Format("2006-01-02 15:04:05"),
 		})
 	}
+
 	c.JSON(http.StatusOK, agents)
 }

backend/internal/handlers/logs_mock.go

@@ -21,6 +21,7 @@ import (
 // @Param limit query int false "Limit results" default(100)
 // @Param offset query int false "Offset results" default(0)
 // @Success 200 {array} storage.LogEntry
+// @Security Bearer
 // @Router /logs/mock [get]
 func (lh *LogHandlers) GetMockLogs(c *gin.Context) {
 	levelFilter := c.Query("level")