From 7e54d621709a8a6c14f1af3b5117bf4d1523b874 Mon Sep 17 00:00:00 2001 From: d3m0k1d Date: Sat, 4 Apr 2026 15:30:05 +0300 Subject: [PATCH] fix: clickhouse logs and reg agents --- agent/main.go | 51 ++-- backend/cmd/main.go | 48 +-- backend/docs/docs.go | 284 ++++++++++-------- backend/docs/swagger.json | 284 ++++++++++-------- backend/docs/swagger.yaml | 207 +++++++------ .../internal/grpcsrv/collector/collector.go | 4 +- backend/internal/handlers/agents.go | 1 + backend/internal/handlers/logs.go | 6 + backend/internal/repository/log_repository.go | 100 ++++-- backend/internal/storage/clickhouse.go | 65 ++-- infra/agent/config.yml | 3 + infra/clickhouse/init/01_create_logs_table.sh | 12 +- infra/docker-compose.yml | 15 +- 13 files changed, 637 insertions(+), 443 deletions(-) diff --git a/agent/main.go b/agent/main.go index 52fa7e6..ae506c1 100644 --- a/agent/main.go +++ b/agent/main.go @@ -39,33 +39,38 @@ func main() { lgr := logger.New(os.Getenv("IS_DEBUG") == "1") lgr.Debug("Config parsed", "cfg", cfg) - if cfg.RegistrationToken == "" { - lgr.Error("No registration token provided") - os.Exit(1) - } + // Check if certificates already exist (agent was previously registered) + if registration.CertsExist(cfg.CertDir) { + lgr.Info("Certificates found, skipping registration") + } else { + if cfg.RegistrationToken == "" { + lgr.Error("No registration token provided") + os.Exit(1) + } - // Generate key and CSR - key, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label) - if err != nil { - lgr.Error("Failed to generate key and CSR", "err", err) - os.Exit(1) - } - lgr.Info("Generated ECDSA key pair and CSR") + // Generate key and CSR + k, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label) + if err != nil { + lgr.Error("Failed to generate key and CSR", "err", err) + os.Exit(1) + } + lgr.Info("Generated ECDSA key pair and CSR") - // Register with backend - certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM) - if err != nil { - lgr.Error("Failed to register", "err", err) - os.Exit(1) - } - lgr.Info("Successfully registered, received certificates") + // Register with backend + certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM) + if err != nil { + lgr.Error("Failed to register", "err", err) + os.Exit(1) + } + lgr.Info("Successfully registered, received certificates") - // Save certificates - if err := registration.SaveCerts(cfg.CertDir, certs, key); err != nil { - lgr.Error("Failed to save certificates", "err", err) - os.Exit(1) + // Save certificates + if err := registration.SaveCerts(cfg.CertDir, certs, k); err != nil { + lgr.Error("Failed to save certificates", "err", err) + os.Exit(1) + } + lgr.Info("Certificates saved", "cert_dir", cfg.CertDir) } - lgr.Info("Certificates saved", "cert_dir", cfg.CertDir) creds, err := mtls.LoadMTLSCredentialsFromFiles( cfg.CertDir+"/ca.crt", diff --git a/backend/cmd/main.go b/backend/cmd/main.go index 39908b3..753bc2d 100644 --- a/backend/cmd/main.go +++ b/backend/cmd/main.go @@ -7,6 +7,7 @@ import ( "log" "net" "os" + "time" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/docs" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/config" @@ -59,25 +60,25 @@ func main() { } // Initialize ClickHouse and log repository - var logRepo *repository.LogRepository + logRepo := repository.NewLogRepository() if cfg.Database.Clickhouse_host != "" { - chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{ - Host: cfg.Database.Clickhouse_host, - User: cfg.Database.Clickhouse_user, - Password: cfg.Database.Clickhouse_password, - Database: cfg.Database.Clickhouse_database, - }) - if err != nil { - log.Printf("Warning: ClickHouse connection failed: %v", err) - } else { - logRepo = repository.NewLogRepository(chConn) + go func() { + db, err := storage.OpenClickHouseWithRetry(storage.ClickHouseConfig{ + Host: cfg.Database.Clickhouse_host, + User: cfg.Database.Clickhouse_user, + Password: cfg.Database.Clickhouse_password, + Database: cfg.Database.Clickhouse_database, + }, 10, 5*time.Second) + if err != nil { + log.Printf("Warning: ClickHouse connection failed: %v", err) + return + } + log.Println("ClickHouse connected successfully") + logRepo.SetDB(db) if err := logRepo.Init(context.Background()); err != nil { log.Printf("Warning: Failed to initialize logs table: %v", err) - } else { - log.Println("ClickHouse connected successfully") } - defer chConn.Close() - } + }() } // Initialize Collector gRPC service @@ -183,15 +184,14 @@ func main() { mockLogHandlers := handlers.NewLogHandlers(nil) logsGroup.GET("/mock", mockLogHandlers.GetMockLogs) - if logRepo != nil { - logHandlers := handlers.NewLogHandlers(logRepo) - logsGroup.POST("", logHandlers.Insert) - logsGroup.POST("/batch", logHandlers.InsertBatch) - logsGroup.GET("", logHandlers.Search) - logsGroup.GET("/services", logHandlers.GetServices) - logsGroup.GET("/agents", logHandlers.GetAgents) - logsGroup.GET("/levels", logHandlers.GetLevels) - } + // ClickHouse log handlers (always registered, work when ClickHouse connects) + logHandlers := handlers.NewLogHandlers(logRepo) + logsGroup.POST("", logHandlers.Insert) + logsGroup.POST("/batch", logHandlers.InsertBatch) + logsGroup.GET("", logHandlers.Search) + logsGroup.GET("/services", logHandlers.GetServices) + logsGroup.GET("/agents", logHandlers.GetAgents) + logsGroup.GET("/levels", logHandlers.GetLevels) } } diff --git a/backend/docs/docs.go b/backend/docs/docs.go index c41a54f..d7c99de 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -17,7 +17,12 @@ const docTemplate = `{ "paths": { "/agents": { "get": { - "description": "Returns a list of all agents currently connected via gRPC streaming", + "security": [ + { + "Bearer": [] + } + ], + "description": "Returns a list of all agents currently connected via Collector (log streaming)", "produces": [ "application/json" ], @@ -31,7 +36,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/internal_handlers.AgentInfo" + "$ref": "#/definitions/handlers.AgentInfo" } } } @@ -63,7 +68,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest" + "$ref": "#/definitions/repository.DeployAgentsRequest" } } ], @@ -71,7 +76,7 @@ const docTemplate = `{ "200": { "description": "Deployment results with tokens for each server", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse" + "$ref": "#/definitions/repository.DeployResponse" } }, "400": { @@ -114,7 +119,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.RegisterRequest" + "$ref": "#/definitions/handlers.RegisterRequest" } } ], @@ -122,7 +127,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_handlers.RegisterResponse" + "$ref": "#/definitions/handlers.RegisterResponse" } } } @@ -152,7 +157,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest" + "$ref": "#/definitions/repository.RegistrationRequest" } } ], @@ -186,7 +191,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest" + "$ref": "#/definitions/repository.LoginRequest" } } ], @@ -194,7 +199,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse" + "$ref": "#/definitions/repository.LoginResponse" } }, "400": { @@ -244,7 +249,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate" + "$ref": "#/definitions/repository.TokenCreate" } } ], @@ -340,7 +345,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } } }, @@ -426,7 +431,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } }, "400": { @@ -481,7 +486,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate" + "$ref": "#/definitions/repository.TokenUpdate" } } ], @@ -661,7 +666,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset" + "$ref": "#/definitions/repository.TokenPasswordReset" } } ], @@ -729,7 +734,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions" + "$ref": "#/definitions/repository.TokenUpdatePermissions" } } ], @@ -789,7 +794,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } } }, @@ -819,7 +824,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } }, "401": { @@ -836,6 +841,11 @@ const docTemplate = `{ }, "/logs": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Searches logs with various filters", "produces": [ "application/json" @@ -896,13 +906,18 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" + "$ref": "#/definitions/storage.LogEntry" } } } } }, "post": { + "security": [ + { + "Bearer": [] + } + ], "description": "Inserts a single log entry into ClickHouse", "consumes": [ "application/json" @@ -921,7 +936,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.InsertLogRequest" + "$ref": "#/definitions/handlers.InsertLogRequest" } } ], @@ -940,6 +955,11 @@ const docTemplate = `{ }, "/logs/agents": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique agent names in logs", "produces": [ "application/json" @@ -963,6 +983,11 @@ const docTemplate = `{ }, "/logs/batch": { "post": { + "security": [ + { + "Bearer": [] + } + ], "description": "Inserts multiple log entries into ClickHouse", "consumes": [ "application/json" @@ -981,7 +1006,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.InsertLogsRequest" + "$ref": "#/definitions/handlers.InsertLogsRequest" } } ], @@ -1000,6 +1025,11 @@ const docTemplate = `{ }, "/logs/levels": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique log levels in logs", "produces": [ "application/json" @@ -1076,7 +1106,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" + "$ref": "#/definitions/storage.LogEntry" } } } @@ -1085,6 +1115,11 @@ const docTemplate = `{ }, "/logs/services": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique service names in logs", "produces": [ "application/json" @@ -1108,7 +1143,93 @@ const docTemplate = `{ } }, "definitions": { - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": { + "handlers.AgentInfo": { + "type": "object", + "properties": { + "connected_at": { + "type": "string" + }, + "label": { + "type": "string" + }, + "services": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "string" + } + } + }, + "handlers.InsertLogRequest": { + "type": "object", + "required": [ + "agent", + "level", + "message", + "service" + ], + "properties": { + "agent": { + "type": "string" + }, + "level": { + "type": "string" + }, + "message": { + "type": "string" + }, + "service": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "handlers.InsertLogsRequest": { + "type": "object", + "required": [ + "logs" + ], + "properties": { + "logs": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.InsertLogRequest" + } + } + } + }, + "handlers.RegisterRequest": { + "type": "object", + "required": [ + "csr", + "token" + ], + "properties": { + "csr": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, + "handlers.RegisterResponse": { + "type": "object", + "properties": { + "ca_cert": { + "type": "string" + }, + "client_cert": { + "type": "string" + } + } + }, + "repository.AgentDeployConfig": { "description": "Configuration for deploying HellreigN agent to a single server", "type": "object", "required": [ @@ -1126,7 +1247,7 @@ const docTemplate = `{ "authMethod": { "allOf": [ { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod" + "$ref": "#/definitions/repository.AuthMethod" } ], "example": "key" @@ -1134,7 +1255,7 @@ const docTemplate = `{ "deployType": { "allOf": [ { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType" + "$ref": "#/definitions/repository.DeployType" } ], "example": "docker" @@ -1161,7 +1282,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": { + "repository.AuthMethod": { "description": "SSH authentication method: key or password", "type": "string", "enum": [ @@ -1173,7 +1294,7 @@ const docTemplate = `{ "AuthMethodPassword" ] }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": { + "repository.DeployAgentsRequest": { "description": "Request to deploy HellreigN agents to multiple servers", "type": "object", "required": [ @@ -1184,12 +1305,12 @@ const docTemplate = `{ "type": "array", "minItems": 1, "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig" + "$ref": "#/definitions/repository.AgentDeployConfig" } } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": { + "repository.DeployResponse": { "description": "Response containing deployment results and registration tokens", "type": "object", "properties": { @@ -1200,12 +1321,12 @@ const docTemplate = `{ "results": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult" + "$ref": "#/definitions/repository.DeployResult" } } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": { + "repository.DeployResult": { "description": "Result of deploying to a single server", "type": "object", "properties": { @@ -1231,7 +1352,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": { + "repository.DeployType": { "description": "Type of deployment: docker or binary", "type": "string", "enum": [ @@ -1243,7 +1364,7 @@ const docTemplate = `{ "DeployTypeBinary" ] }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": { + "repository.LoginRequest": { "type": "object", "required": [ "login", @@ -1258,7 +1379,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": { + "repository.LoginResponse": { "type": "object", "properties": { "is_active": { @@ -1287,7 +1408,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": { + "repository.RegistrationRequest": { "type": "object", "required": [ "label" @@ -1298,7 +1419,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": { + "repository.TokenCreate": { "type": "object", "required": [ "last_name", @@ -1333,7 +1454,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": { + "repository.TokenPasswordReset": { "type": "object", "required": [ "new_password" @@ -1344,7 +1465,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": { + "repository.TokenUpdate": { "type": "object", "properties": { "last_name": { @@ -1355,7 +1476,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": { + "repository.TokenUpdatePermissions": { "type": "object", "properties": { "is_active": { @@ -1372,7 +1493,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": { + "repository.Tokens": { "type": "object", "properties": { "id": { @@ -1404,7 +1525,7 @@ const docTemplate = `{ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": { + "storage.LogEntry": { "type": "object", "properties": { "agent": { @@ -1423,89 +1544,6 @@ const docTemplate = `{ "type": "string" } } - }, - "internal_handlers.AgentInfo": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "services": { - "type": "array", - "items": { - "type": "string" - } - }, - "token": { - "type": "string" - } - } - }, - "internal_handlers.InsertLogRequest": { - "type": "object", - "required": [ - "agent", - "level", - "message", - "service" - ], - "properties": { - "agent": { - "type": "string" - }, - "level": { - "type": "string" - }, - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } - }, - "internal_handlers.InsertLogsRequest": { - "type": "object", - "required": [ - "logs" - ], - "properties": { - "logs": { - "type": "array", - "items": { - "$ref": "#/definitions/internal_handlers.InsertLogRequest" - } - } - } - }, - "internal_handlers.RegisterRequest": { - "type": "object", - "required": [ - "csr", - "token" - ], - "properties": { - "csr": { - "type": "string" - }, - "token": { - "type": "string" - } - } - }, - "internal_handlers.RegisterResponse": { - "type": "object", - "properties": { - "ca_cert": { - "type": "string" - }, - "client_cert": { - "type": "string" - } - } } }, "securityDefinitions": { diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 9e820c9..3c8cb04 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -6,7 +6,12 @@ "paths": { "/agents": { "get": { - "description": "Returns a list of all agents currently connected via gRPC streaming", + "security": [ + { + "Bearer": [] + } + ], + "description": "Returns a list of all agents currently connected via Collector (log streaming)", "produces": [ "application/json" ], @@ -20,7 +25,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/internal_handlers.AgentInfo" + "$ref": "#/definitions/handlers.AgentInfo" } } } @@ -52,7 +57,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest" + "$ref": "#/definitions/repository.DeployAgentsRequest" } } ], @@ -60,7 +65,7 @@ "200": { "description": "Deployment results with tokens for each server", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse" + "$ref": "#/definitions/repository.DeployResponse" } }, "400": { @@ -103,7 +108,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.RegisterRequest" + "$ref": "#/definitions/handlers.RegisterRequest" } } ], @@ -111,7 +116,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_handlers.RegisterResponse" + "$ref": "#/definitions/handlers.RegisterResponse" } } } @@ -141,7 +146,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest" + "$ref": "#/definitions/repository.RegistrationRequest" } } ], @@ -175,7 +180,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest" + "$ref": "#/definitions/repository.LoginRequest" } } ], @@ -183,7 +188,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse" + "$ref": "#/definitions/repository.LoginResponse" } }, "400": { @@ -233,7 +238,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate" + "$ref": "#/definitions/repository.TokenCreate" } } ], @@ -329,7 +334,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } } }, @@ -415,7 +420,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } }, "400": { @@ -470,7 +475,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate" + "$ref": "#/definitions/repository.TokenUpdate" } } ], @@ -650,7 +655,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset" + "$ref": "#/definitions/repository.TokenPasswordReset" } } ], @@ -718,7 +723,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions" + "$ref": "#/definitions/repository.TokenUpdatePermissions" } } ], @@ -778,7 +783,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } } }, @@ -808,7 +813,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" + "$ref": "#/definitions/repository.Tokens" } }, "401": { @@ -825,6 +830,11 @@ }, "/logs": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Searches logs with various filters", "produces": [ "application/json" @@ -885,13 +895,18 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" + "$ref": "#/definitions/storage.LogEntry" } } } } }, "post": { + "security": [ + { + "Bearer": [] + } + ], "description": "Inserts a single log entry into ClickHouse", "consumes": [ "application/json" @@ -910,7 +925,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.InsertLogRequest" + "$ref": "#/definitions/handlers.InsertLogRequest" } } ], @@ -929,6 +944,11 @@ }, "/logs/agents": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique agent names in logs", "produces": [ "application/json" @@ -952,6 +972,11 @@ }, "/logs/batch": { "post": { + "security": [ + { + "Bearer": [] + } + ], "description": "Inserts multiple log entries into ClickHouse", "consumes": [ "application/json" @@ -970,7 +995,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/internal_handlers.InsertLogsRequest" + "$ref": "#/definitions/handlers.InsertLogsRequest" } } ], @@ -989,6 +1014,11 @@ }, "/logs/levels": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique log levels in logs", "produces": [ "application/json" @@ -1065,7 +1095,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" + "$ref": "#/definitions/storage.LogEntry" } } } @@ -1074,6 +1104,11 @@ }, "/logs/services": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns list of all unique service names in logs", "produces": [ "application/json" @@ -1097,7 +1132,93 @@ } }, "definitions": { - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": { + "handlers.AgentInfo": { + "type": "object", + "properties": { + "connected_at": { + "type": "string" + }, + "label": { + "type": "string" + }, + "services": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "string" + } + } + }, + "handlers.InsertLogRequest": { + "type": "object", + "required": [ + "agent", + "level", + "message", + "service" + ], + "properties": { + "agent": { + "type": "string" + }, + "level": { + "type": "string" + }, + "message": { + "type": "string" + }, + "service": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "handlers.InsertLogsRequest": { + "type": "object", + "required": [ + "logs" + ], + "properties": { + "logs": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.InsertLogRequest" + } + } + } + }, + "handlers.RegisterRequest": { + "type": "object", + "required": [ + "csr", + "token" + ], + "properties": { + "csr": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, + "handlers.RegisterResponse": { + "type": "object", + "properties": { + "ca_cert": { + "type": "string" + }, + "client_cert": { + "type": "string" + } + } + }, + "repository.AgentDeployConfig": { "description": "Configuration for deploying HellreigN agent to a single server", "type": "object", "required": [ @@ -1115,7 +1236,7 @@ "authMethod": { "allOf": [ { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod" + "$ref": "#/definitions/repository.AuthMethod" } ], "example": "key" @@ -1123,7 +1244,7 @@ "deployType": { "allOf": [ { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType" + "$ref": "#/definitions/repository.DeployType" } ], "example": "docker" @@ -1150,7 +1271,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": { + "repository.AuthMethod": { "description": "SSH authentication method: key or password", "type": "string", "enum": [ @@ -1162,7 +1283,7 @@ "AuthMethodPassword" ] }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": { + "repository.DeployAgentsRequest": { "description": "Request to deploy HellreigN agents to multiple servers", "type": "object", "required": [ @@ -1173,12 +1294,12 @@ "type": "array", "minItems": 1, "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig" + "$ref": "#/definitions/repository.AgentDeployConfig" } } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": { + "repository.DeployResponse": { "description": "Response containing deployment results and registration tokens", "type": "object", "properties": { @@ -1189,12 +1310,12 @@ "results": { "type": "array", "items": { - "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult" + "$ref": "#/definitions/repository.DeployResult" } } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": { + "repository.DeployResult": { "description": "Result of deploying to a single server", "type": "object", "properties": { @@ -1220,7 +1341,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": { + "repository.DeployType": { "description": "Type of deployment: docker or binary", "type": "string", "enum": [ @@ -1232,7 +1353,7 @@ "DeployTypeBinary" ] }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": { + "repository.LoginRequest": { "type": "object", "required": [ "login", @@ -1247,7 +1368,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": { + "repository.LoginResponse": { "type": "object", "properties": { "is_active": { @@ -1276,7 +1397,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": { + "repository.RegistrationRequest": { "type": "object", "required": [ "label" @@ -1287,7 +1408,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": { + "repository.TokenCreate": { "type": "object", "required": [ "last_name", @@ -1322,7 +1443,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": { + "repository.TokenPasswordReset": { "type": "object", "required": [ "new_password" @@ -1333,7 +1454,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": { + "repository.TokenUpdate": { "type": "object", "properties": { "last_name": { @@ -1344,7 +1465,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": { + "repository.TokenUpdatePermissions": { "type": "object", "properties": { "is_active": { @@ -1361,7 +1482,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": { + "repository.Tokens": { "type": "object", "properties": { "id": { @@ -1393,7 +1514,7 @@ } } }, - "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": { + "storage.LogEntry": { "type": "object", "properties": { "agent": { @@ -1412,89 +1533,6 @@ "type": "string" } } - }, - "internal_handlers.AgentInfo": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "services": { - "type": "array", - "items": { - "type": "string" - } - }, - "token": { - "type": "string" - } - } - }, - "internal_handlers.InsertLogRequest": { - "type": "object", - "required": [ - "agent", - "level", - "message", - "service" - ], - "properties": { - "agent": { - "type": "string" - }, - "level": { - "type": "string" - }, - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } - }, - "internal_handlers.InsertLogsRequest": { - "type": "object", - "required": [ - "logs" - ], - "properties": { - "logs": { - "type": "array", - "items": { - "$ref": "#/definitions/internal_handlers.InsertLogRequest" - } - } - } - }, - "internal_handlers.RegisterRequest": { - "type": "object", - "required": [ - "csr", - "token" - ], - "properties": { - "csr": { - "type": "string" - }, - "token": { - "type": "string" - } - } - }, - "internal_handlers.RegisterResponse": { - "type": "object", - "properties": { - "ca_cert": { - "type": "string" - }, - "client_cert": { - "type": "string" - } - } } }, "securityDefinitions": { diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index 01feea1..d73b4eb 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -1,5 +1,62 @@ definitions: - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig: + handlers.AgentInfo: + properties: + connected_at: + type: string + label: + type: string + services: + items: + type: string + type: array + token: + type: string + type: object + handlers.InsertLogRequest: + properties: + agent: + type: string + level: + type: string + message: + type: string + service: + type: string + timestamp: + type: string + required: + - agent + - level + - message + - service + type: object + handlers.InsertLogsRequest: + properties: + logs: + items: + $ref: '#/definitions/handlers.InsertLogRequest' + type: array + required: + - logs + type: object + handlers.RegisterRequest: + properties: + csr: + type: string + token: + type: string + required: + - csr + - token + type: object + handlers.RegisterResponse: + properties: + ca_cert: + type: string + client_cert: + type: string + type: object + repository.AgentDeployConfig: description: Configuration for deploying HellreigN agent to a single server properties: agentLabel: @@ -7,11 +64,11 @@ definitions: type: string authMethod: allOf: - - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod' + - $ref: '#/definitions/repository.AuthMethod' example: key deployType: allOf: - - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType' + - $ref: '#/definitions/repository.DeployType' example: docker ip: example: 192.168.1.100 @@ -35,7 +92,7 @@ definitions: - ip - user type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod: + repository.AuthMethod: description: 'SSH authentication method: key or password' enum: - key @@ -44,18 +101,18 @@ definitions: x-enum-varnames: - AuthMethodKey - AuthMethodPassword - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest: + repository.DeployAgentsRequest: description: Request to deploy HellreigN agents to multiple servers properties: servers: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig' + $ref: '#/definitions/repository.AgentDeployConfig' minItems: 1 type: array required: - servers type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse: + repository.DeployResponse: description: Response containing deployment results and registration tokens properties: message: @@ -63,10 +120,10 @@ definitions: type: string results: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult' + $ref: '#/definitions/repository.DeployResult' type: array type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult: + repository.DeployResult: description: Result of deploying to a single server properties: agent_label: @@ -85,7 +142,7 @@ definitions: example: abc123... type: string type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType: + repository.DeployType: description: 'Type of deployment: docker or binary' enum: - docker @@ -94,7 +151,7 @@ definitions: x-enum-varnames: - DeployTypeDocker - DeployTypeBinary - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest: + repository.LoginRequest: properties: login: type: string @@ -104,7 +161,7 @@ definitions: - login - password type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse: + repository.LoginResponse: properties: is_active: type: boolean @@ -123,14 +180,14 @@ definitions: token: type: string type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest: + repository.RegistrationRequest: properties: label: type: string required: - label type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate: + repository.TokenCreate: properties: is_active: type: boolean @@ -154,21 +211,21 @@ definitions: - name - password type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset: + repository.TokenPasswordReset: properties: new_password: type: string required: - new_password type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate: + repository.TokenUpdate: properties: last_name: type: string name: type: string type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions: + repository.TokenUpdatePermissions: properties: is_active: type: boolean @@ -179,7 +236,7 @@ definitions: permission_view: type: boolean type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens: + repository.Tokens: properties: id: type: integer @@ -200,7 +257,7 @@ definitions: token: type: string type: object - gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry: + storage.LogEntry: properties: agent: type: string @@ -213,67 +270,13 @@ definitions: timestamp: type: string type: object - internal_handlers.AgentInfo: - properties: - label: - type: string - services: - items: - type: string - type: array - token: - type: string - type: object - internal_handlers.InsertLogRequest: - properties: - agent: - type: string - level: - type: string - message: - type: string - service: - type: string - timestamp: - type: string - required: - - agent - - level - - message - - service - type: object - internal_handlers.InsertLogsRequest: - properties: - logs: - items: - $ref: '#/definitions/internal_handlers.InsertLogRequest' - type: array - required: - - logs - type: object - internal_handlers.RegisterRequest: - properties: - csr: - type: string - token: - type: string - required: - - csr - - token - type: object - internal_handlers.RegisterResponse: - properties: - ca_cert: - type: string - client_cert: - type: string - type: object info: contact: {} paths: /agents: get: - description: Returns a list of all agents currently connected via gRPC streaming + description: Returns a list of all agents currently connected via Collector + (log streaming) produces: - application/json responses: @@ -281,8 +284,10 @@ paths: description: OK schema: items: - $ref: '#/definitions/internal_handlers.AgentInfo' + $ref: '#/definitions/handlers.AgentInfo' type: array + security: + - Bearer: [] summary: Get connected agents tags: - agents @@ -298,14 +303,14 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest' + $ref: '#/definitions/repository.DeployAgentsRequest' produces: - application/json responses: "200": description: Deployment results with tokens for each server schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse' + $ref: '#/definitions/repository.DeployResponse' "400": description: Invalid request schema: @@ -333,14 +338,14 @@ paths: name: request required: true schema: - $ref: '#/definitions/internal_handlers.RegisterRequest' + $ref: '#/definitions/handlers.RegisterRequest' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/internal_handlers.RegisterResponse' + $ref: '#/definitions/handlers.RegisterResponse' summary: Register agent tags: - agents @@ -354,7 +359,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest' + $ref: '#/definitions/repository.RegistrationRequest' produces: - application/json responses: @@ -380,12 +385,12 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest' + $ref: '#/definitions/repository.LoginRequest' responses: "200": description: OK schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse' + $ref: '#/definitions/repository.LoginResponse' "400": description: Bad Request schema: @@ -442,7 +447,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate' + $ref: '#/definitions/repository.TokenCreate' responses: "200": description: OK @@ -481,7 +486,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' + $ref: '#/definitions/repository.Tokens' type: array "500": description: Internal Server Error @@ -538,7 +543,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' + $ref: '#/definitions/repository.Tokens' "400": description: Bad Request schema: @@ -575,7 +580,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate' + $ref: '#/definitions/repository.TokenUpdate' responses: "200": description: OK @@ -694,7 +699,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset' + $ref: '#/definitions/repository.TokenPasswordReset' responses: "200": description: OK @@ -739,7 +744,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions' + $ref: '#/definitions/repository.TokenUpdatePermissions' responses: "200": description: OK @@ -778,7 +783,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' + $ref: '#/definitions/repository.Tokens' type: array "500": description: Internal Server Error @@ -798,7 +803,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' + $ref: '#/definitions/repository.Tokens' "401": description: Unauthorized schema: @@ -849,8 +854,10 @@ paths: description: OK schema: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry' + $ref: '#/definitions/storage.LogEntry' type: array + security: + - Bearer: [] summary: Search logs tags: - logs @@ -864,7 +871,7 @@ paths: name: body required: true schema: - $ref: '#/definitions/internal_handlers.InsertLogRequest' + $ref: '#/definitions/handlers.InsertLogRequest' produces: - application/json responses: @@ -874,6 +881,8 @@ paths: additionalProperties: type: string type: object + security: + - Bearer: [] summary: Insert log entry tags: - logs @@ -889,6 +898,8 @@ paths: items: type: string type: array + security: + - Bearer: [] summary: Get distinct agents tags: - logs @@ -903,7 +914,7 @@ paths: name: body required: true schema: - $ref: '#/definitions/internal_handlers.InsertLogsRequest' + $ref: '#/definitions/handlers.InsertLogsRequest' produces: - application/json responses: @@ -913,6 +924,8 @@ paths: additionalProperties: type: string type: object + security: + - Bearer: [] summary: Insert log entries (batch) tags: - logs @@ -928,6 +941,8 @@ paths: items: type: string type: array + security: + - Bearer: [] summary: Get distinct log levels tags: - logs @@ -965,7 +980,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry' + $ref: '#/definitions/storage.LogEntry' type: array security: - Bearer: [] @@ -984,6 +999,8 @@ paths: items: type: string type: array + security: + - Bearer: [] summary: Get distinct services tags: - logs diff --git a/backend/internal/grpcsrv/collector/collector.go b/backend/internal/grpcsrv/collector/collector.go index 246d123..17036e8 100644 --- a/backend/internal/grpcsrv/collector/collector.go +++ b/backend/internal/grpcsrv/collector/collector.go @@ -81,8 +81,8 @@ func (c *Collector) Stream(stream proto.Collector_StreamServer) error { log.Printf("Agent %s connected, streaming logs for service: %s", agentName, service) // If no ClickHouse, just consume the stream without storing - if c.logRepo == nil { - log.Printf("Warning: logRepo is nil, consuming logs without storing for agent %s", agentName) + if !c.logRepo.IsConnected() { + log.Printf("Warning: ClickHouse not connected yet, consuming logs without storing for agent %s", agentName) for { _, err := stream.Recv() if err == io.EOF { diff --git a/backend/internal/handlers/agents.go b/backend/internal/handlers/agents.go index 8c55c53..cc29142 100644 --- a/backend/internal/handlers/agents.go +++ b/backend/internal/handlers/agents.go @@ -25,6 +25,7 @@ type AgentInfo struct { // @Summary Get connected agents // @Description Returns a list of all agents currently connected via Collector (log streaming) // @Tags agents +// @Security Bearer // @Produce json // @Success 200 {array} AgentInfo // @Router /agents [get] diff --git a/backend/internal/handlers/logs.go b/backend/internal/handlers/logs.go index 487b5fc..89d1f52 100644 --- a/backend/internal/handlers/logs.go +++ b/backend/internal/handlers/logs.go @@ -33,6 +33,7 @@ type InsertLogRequest struct { // @Produce json // @Param body body InsertLogRequest true "Log entry" // @Success 201 {object} map[string]string +// @Security Bearer // @Router /logs [post] func (lh *LogHandlers) Insert(c *gin.Context) { var req InsertLogRequest @@ -72,6 +73,7 @@ type InsertLogsRequest struct { // @Produce json // @Param body body InsertLogsRequest true "Log entries" // @Success 201 {object} map[string]string +// @Security Bearer // @Router /logs/batch [post] func (lh *LogHandlers) InsertBatch(c *gin.Context) { var req InsertLogsRequest @@ -124,6 +126,7 @@ type SearchLogsRequest struct { // @Param limit query int false "Limit results" default(100) // @Param offset query int false "Offset results" default(0) // @Success 200 {array} storage.LogEntry +// @Security Bearer // @Router /logs [get] func (lh *LogHandlers) Search(c *gin.Context) { var req SearchLogsRequest @@ -170,6 +173,7 @@ func (lh *LogHandlers) Search(c *gin.Context) { // @Tags logs // @Produce json // @Success 200 {array} string +// @Security Bearer // @Router /logs/services [get] func (lh *LogHandlers) GetServices(c *gin.Context) { services, err := lh.LogRepo.GetDistinctServices(c.Request.Context()) @@ -190,6 +194,7 @@ func (lh *LogHandlers) GetServices(c *gin.Context) { // @Tags logs // @Produce json // @Success 200 {array} string +// @Security Bearer // @Router /logs/agents [get] func (lh *LogHandlers) GetAgents(c *gin.Context) { agents, err := lh.LogRepo.GetDistinctAgents(c.Request.Context()) @@ -210,6 +215,7 @@ func (lh *LogHandlers) GetAgents(c *gin.Context) { // @Tags logs // @Produce json // @Success 200 {array} string +// @Security Bearer // @Router /logs/levels [get] func (lh *LogHandlers) GetLevels(c *gin.Context) { levels, err := lh.LogRepo.GetDistinctLevels(c.Request.Context()) diff --git a/backend/internal/repository/log_repository.go b/backend/internal/repository/log_repository.go index 5d5ec92..60221da 100644 --- a/backend/internal/repository/log_repository.go +++ b/backend/internal/repository/log_repository.go @@ -2,44 +2,85 @@ package repository import ( "context" + "database/sql" + "fmt" + "sync" "time" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" ) type LogRepository struct { - Conn driver.Conn + mu sync.RWMutex + DB *sql.DB } -func NewLogRepository(conn driver.Conn) *LogRepository { - return &LogRepository{Conn: conn} +func NewLogRepository() *LogRepository { + return &LogRepository{} +} + +func (r *LogRepository) SetDB(db *sql.DB) { + r.mu.Lock() + defer r.mu.Unlock() + r.DB = db +} + +func (r *LogRepository) IsConnected() bool { + r.mu.RLock() + defer r.mu.RUnlock() + return r.DB != nil +} + +func (r *LogRepository) getDB() *sql.DB { + r.mu.RLock() + defer r.mu.RUnlock() + return r.DB } func (r *LogRepository) Init(ctx context.Context) error { - return r.Conn.Exec(ctx, storage.CreateLogsTable) + db := r.getDB() + if db == nil { + return nil + } + _, err := db.ExecContext(ctx, storage.CreateLogsTable) + return err } func (r *LogRepository) Insert(ctx context.Context, log storage.LogEntry) error { - return r.Conn.Exec(ctx, ` + db := r.getDB() + if db == nil { + return nil + } + _, err := db.ExecContext(ctx, ` INSERT INTO logs (timestamp, level, service, agent, message) VALUES ($1, $2, $3, $4, $5) `, log.Timestamp, log.Level, log.Service, log.Agent, log.Message) + return err } func (r *LogRepository) InsertBatch(ctx context.Context, logs []storage.LogEntry) error { - batch, err := r.Conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, level, service, agent, message)") - if err != nil { - return err + db := r.getDB() + if db == nil { + return nil + } + if len(logs) == 0 { + return nil } - for _, log := range logs { - if err := batch.Append(log.Timestamp, log.Level, log.Service, log.Agent, log.Message); err != nil { - return err + // Build multi-row INSERT statement + query := "INSERT INTO logs (timestamp, level, service, agent, message) VALUES " + args := make([]interface{}, 0, len(logs)*5) + for i, log := range logs { + if i > 0 { + query += ", " } + query += fmt.Sprintf("($%d, $%d, $%d, $%d, $%d)", + i*5+1, i*5+2, i*5+3, i*5+4, i*5+5) + args = append(args, log.Timestamp, log.Level, log.Service, log.Agent, log.Message) } - return batch.Send() + _, err := db.ExecContext(ctx, query, args...) + return err } type LogFilter struct { @@ -53,6 +94,11 @@ type LogFilter struct { } func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage.LogEntry, error) { + db := r.getDB() + if db == nil { + return []storage.LogEntry{}, nil + } + query := "SELECT timestamp, level, service, agent, message FROM logs WHERE 1=1" args := make([]interface{}, 0) argIdx := 1 @@ -102,13 +148,13 @@ func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage args = append(args, filter.Offset) } - rows, err := r.Conn.Query(ctx, query, args...) + rows, err := db.QueryContext(ctx, query, args...) if err != nil { return nil, err } defer rows.Close() - var logs []storage.LogEntry + logs := make([]storage.LogEntry, 0) for rows.Next() { var log storage.LogEntry if err := rows.Scan(&log.Timestamp, &log.Level, &log.Service, &log.Agent, &log.Message); err != nil { @@ -121,13 +167,17 @@ func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage } func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, error) { - rows, err := r.Conn.Query(ctx, "SELECT DISTINCT service FROM logs ORDER BY service") + db := r.getDB() + if db == nil { + return []string{}, nil + } + rows, err := db.QueryContext(ctx, "SELECT DISTINCT service FROM logs ORDER BY service") if err != nil { return nil, err } defer rows.Close() - var services []string + services := make([]string, 0) for rows.Next() { var service string if err := rows.Scan(&service); err != nil { @@ -140,13 +190,17 @@ func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, erro } func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error) { - rows, err := r.Conn.Query(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent") + db := r.getDB() + if db == nil { + return []string{}, nil + } + rows, err := db.QueryContext(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent") if err != nil { return nil, err } defer rows.Close() - var agents []string + agents := make([]string, 0) for rows.Next() { var agent string if err := rows.Scan(&agent); err != nil { @@ -159,13 +213,17 @@ func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error) } func (r *LogRepository) GetDistinctLevels(ctx context.Context) ([]string, error) { - rows, err := r.Conn.Query(ctx, "SELECT DISTINCT level FROM logs ORDER BY level") + db := r.getDB() + if db == nil { + return []string{}, nil + } + rows, err := db.QueryContext(ctx, "SELECT DISTINCT level FROM logs ORDER BY level") if err != nil { return nil, err } defer rows.Close() - var levels []string + levels := make([]string, 0) for rows.Next() { var level string if err := rows.Scan(&level); err != nil { diff --git a/backend/internal/storage/clickhouse.go b/backend/internal/storage/clickhouse.go index 60917d3..5759f10 100644 --- a/backend/internal/storage/clickhouse.go +++ b/backend/internal/storage/clickhouse.go @@ -2,10 +2,12 @@ package storage import ( "context" + "database/sql" "fmt" + "log" + "time" - "github.com/ClickHouse/clickhouse-go/v2" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + _ "github.com/ClickHouse/clickhouse-go/v2" ) type ClickHouseConfig struct { @@ -15,33 +17,46 @@ type ClickHouseConfig struct { Database string } -func OpenClickHouse(cfg ClickHouseConfig) (driver.Conn, error) { - conn, err := clickhouse.Open(&clickhouse.Options{ - Addr: []string{cfg.Host}, - Auth: clickhouse.Auth{ - Database: cfg.Database, - Username: cfg.User, - Password: cfg.Password, - }, - Settings: clickhouse.Settings{ - "max_execution_time": 60, - }, - Compression: &clickhouse.Compression{ - Method: clickhouse.CompressionLZ4, - }, - DialTimeout: 30, - MaxOpenConns: 10, - MaxIdleConns: 5, - ConnMaxLifetime: 3600, - ConnOpenStrategy: clickhouse.ConnOpenInOrder, - }) +func OpenClickHouse(cfg ClickHouseConfig) (*sql.DB, error) { + dsn := fmt.Sprintf("clickhouse://%s:%s@%s/%s", + cfg.User, cfg.Password, cfg.Host, cfg.Database) + + db, err := sql.Open("clickhouse", dsn) if err != nil { - return nil, fmt.Errorf("clickhouse connect: %w", err) + return nil, fmt.Errorf("clickhouse open: %w", err) } - if err := conn.Ping(context.Background()); err != nil { + db.SetMaxOpenConns(5) + db.SetMaxIdleConns(2) + db.SetConnMaxLifetime(10 * time.Minute) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := db.PingContext(ctx); err != nil { + db.Close() return nil, fmt.Errorf("clickhouse ping: %w", err) } - return conn, nil + log.Printf("ClickHouse connected via database/sql: %s", cfg.Host) + return db, nil +} + +// OpenClickHouseWithRetry attempts to connect to ClickHouse with retries and backoff. +func OpenClickHouseWithRetry(cfg ClickHouseConfig, maxRetries int, initialDelay time.Duration) (*sql.DB, error) { + var lastErr error + delay := initialDelay + + for i := 0; i < maxRetries; i++ { + db, err := OpenClickHouse(cfg) + if err == nil { + return db, nil + } + lastErr = err + log.Printf("ClickHouse connection attempt %d/%d failed: %v, retrying in %v...", i+1, maxRetries, err, delay) + time.Sleep(delay) + delay *= 2 + } + + return nil, fmt.Errorf("clickhouse connection failed after %d attempts: %w", maxRetries, lastErr) } diff --git a/infra/agent/config.yml b/infra/agent/config.yml index 1b572ea..5ab5317 100644 --- a/infra/agent/config.yml +++ b/infra/agent/config.yml @@ -3,3 +3,6 @@ grpc_url: backend:9001 label: test-agent-1 registration_token: "156616b56774d59ba53f1eb4b096488bb5f755bbf5b737d93a42bb1b583ad7fb" cert_dir: /etc/hellreign-agent/certs +services: + - name: system + type: journald diff --git a/infra/clickhouse/init/01_create_logs_table.sh b/infra/clickhouse/init/01_create_logs_table.sh index b6a900a..ba1cd9a 100644 --- a/infra/clickhouse/init/01_create_logs_table.sh +++ b/infra/clickhouse/init/01_create_logs_table.sh @@ -6,11 +6,11 @@ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS hellreign;" clickhouse-client --query " CREATE TABLE IF NOT EXISTS hellreign.logs ( timestamp DateTime64(3) DEFAULT now(), - level String, - service String, - message String, - host String, - trace_id String + level LowCardinality(String), + service LowCardinality(String), + agent LowCardinality(String), + message String ) ENGINE = MergeTree() -ORDER BY (timestamp, service, level); +ORDER BY (timestamp, level, service, agent) +SETTINGS index_granularity = 8192; " diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 1310e54..ef0cd68 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -13,6 +13,12 @@ services: volumes: - clickhouse_data:/var/lib/clickhouse - ./clickhouse/init:/docker-entrypoint-initdb.d + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 30s networks: - hellreign @@ -33,7 +39,8 @@ services: - ./backend/config.yml:/etc/hellreign/config.yml:ro - backend_data:/var/lib/hellreign depends_on: - - clickhouse + clickhouse: + condition: service_healthy networks: - hellreign @@ -56,9 +63,13 @@ services: container_name: hellreign-agent environment: CONFIG_FILE: /etc/hellreign-agent/config.yml + JOURNALD_LOGDIR: /var/log/journal + BUFFER_DB: /var/lib/hellreign-agent/agent_buffer.db volumes: - ./agent/config.yml:/etc/hellreign-agent/config.yml:ro - agent_certs:/etc/hellreign-agent/certs + - agent_data:/var/lib/hellreign-agent + - /var/log/journal:/var/log/journal:ro depends_on: - backend networks: @@ -71,6 +82,8 @@ volumes: driver: local agent_certs: driver: local + agent_data: + driver: local networks: hellreign: