d3m0k1d/HellreigN
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix: clickhouse logs and reg agents
ci-agent / build (push) Failing after 9m40s
Details

Browse Source
This commit is contained in:
d3m0k1d
2026-04-04 15:30:05 +03:00
parent 477dd94227
commit 7e54d62170
13 changed files with 637 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/main.go
+28 -23
View File
@@ -39,33 +39,38 @@ func main() {
lgr := logger.New(os.Getenv("IS_DEBUG") == "1")
lgr.Debug("Config parsed", "cfg", cfg)
if cfg.RegistrationToken == "" {
lgr.Error("No registration token provided")
os.Exit(1)
}
// Check if certificates already exist (agent was previously registered)
if registration.CertsExist(cfg.CertDir) {
lgr.Info("Certificates found, skipping registration")
} else {
if cfg.RegistrationToken == "" {
lgr.Error("No registration token provided")
os.Exit(1)
}
// Generate key and CSR
key, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label)
if err != nil {
lgr.Error("Failed to generate key and CSR", "err", err)
os.Exit(1)
}
lgr.Info("Generated ECDSA key pair and CSR")
// Generate key and CSR
k, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label)
if err != nil {
lgr.Error("Failed to generate key and CSR", "err", err)
os.Exit(1)
}
lgr.Info("Generated ECDSA key pair and CSR")
// Register with backend
certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM)
if err != nil {
lgr.Error("Failed to register", "err", err)
os.Exit(1)
}
lgr.Info("Successfully registered, received certificates")
// Register with backend
certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM)
if err != nil {
lgr.Error("Failed to register", "err", err)
os.Exit(1)
}
lgr.Info("Successfully registered, received certificates")
// Save certificates
if err := registration.SaveCerts(cfg.CertDir, certs, key); err != nil {
lgr.Error("Failed to save certificates", "err", err)
os.Exit(1)
// Save certificates
if err := registration.SaveCerts(cfg.CertDir, certs, k); err != nil {
lgr.Error("Failed to save certificates", "err", err)
os.Exit(1)
}
lgr.Info("Certificates saved", "cert_dir", cfg.CertDir)
}
lgr.Info("Certificates saved", "cert_dir", cfg.CertDir)
creds, err := mtls.LoadMTLSCredentialsFromFiles(
cfg.CertDir+"/ca.crt",
backend/cmd/main.go
+24 -24
View File
@@ -7,6 +7,7 @@ import (
"log"
"net"
"os"
"time"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/docs"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/config"
@@ -59,25 +60,25 @@ func main() {
}
// Initialize ClickHouse and log repository
var logRepo *repository.LogRepository
logRepo := repository.NewLogRepository()
if cfg.Database.Clickhouse_host != "" {
chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{
Host: cfg.Database.Clickhouse_host,
User: cfg.Database.Clickhouse_user,
Password: cfg.Database.Clickhouse_password,
Database: cfg.Database.Clickhouse_database,
})
if err != nil {
log.Printf("Warning: ClickHouse connection failed: %v", err)
} else {
logRepo = repository.NewLogRepository(chConn)
go func() {
db, err := storage.OpenClickHouseWithRetry(storage.ClickHouseConfig{
Host: cfg.Database.Clickhouse_host,
User: cfg.Database.Clickhouse_user,
Password: cfg.Database.Clickhouse_password,
Database: cfg.Database.Clickhouse_database,
}, 10, 5*time.Second)
if err != nil {
log.Printf("Warning: ClickHouse connection failed: %v", err)
return
}
log.Println("ClickHouse connected successfully")
logRepo.SetDB(db)
if err := logRepo.Init(context.Background()); err != nil {
log.Printf("Warning: Failed to initialize logs table: %v", err)
} else {
log.Println("ClickHouse connected successfully")
}
defer chConn.Close()
}
}()
}
// Initialize Collector gRPC service
@@ -183,15 +184,14 @@ func main() {
mockLogHandlers := handlers.NewLogHandlers(nil)
logsGroup.GET("/mock", mockLogHandlers.GetMockLogs)
if logRepo != nil {
logHandlers := handlers.NewLogHandlers(logRepo)
logsGroup.POST("", logHandlers.Insert)
logsGroup.POST("/batch", logHandlers.InsertBatch)
logsGroup.GET("", logHandlers.Search)
logsGroup.GET("/services", logHandlers.GetServices)
logsGroup.GET("/agents", logHandlers.GetAgents)
logsGroup.GET("/levels", logHandlers.GetLevels)
}
// ClickHouse log handlers (always registered, work when ClickHouse connects)
logHandlers := handlers.NewLogHandlers(logRepo)
logsGroup.POST("", logHandlers.Insert)
logsGroup.POST("/batch", logHandlers.InsertBatch)
logsGroup.GET("", logHandlers.Search)
logsGroup.GET("/services", logHandlers.GetServices)
logsGroup.GET("/agents", logHandlers.GetAgents)
logsGroup.GET("/levels", logHandlers.GetLevels)
}
}
backend/docs/docs.go
+161 -123
View File
@@ -17,7 +17,12 @@ const docTemplate = `{
"paths": {
"/agents": {
"get": {
"description": "Returns a list of all agents currently connected via gRPC streaming",
"security": [
{
"Bearer": []
}
],
"description": "Returns a list of all agents currently connected via Collector (log streaming)",
"produces": [
"application/json"
],
@@ -31,7 +36,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.AgentInfo"
"$ref": "#/definitions/handlers.AgentInfo"
}
}
}
@@ -63,7 +68,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest"
"$ref": "#/definitions/repository.DeployAgentsRequest"
}
}
],
@@ -71,7 +76,7 @@ const docTemplate = `{
"200": {
"description": "Deployment results with tokens for each server",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse"
"$ref": "#/definitions/repository.DeployResponse"
}
},
"400": {
@@ -114,7 +119,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.RegisterRequest"
"$ref": "#/definitions/handlers.RegisterRequest"
}
}
],
@@ -122,7 +127,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/internal_handlers.RegisterResponse"
"$ref": "#/definitions/handlers.RegisterResponse"
}
}
}
@@ -152,7 +157,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest"
"$ref": "#/definitions/repository.RegistrationRequest"
}
}
],
@@ -186,7 +191,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest"
"$ref": "#/definitions/repository.LoginRequest"
}
}
],
@@ -194,7 +199,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse"
"$ref": "#/definitions/repository.LoginResponse"
}
},
"400": {
@@ -244,7 +249,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate"
"$ref": "#/definitions/repository.TokenCreate"
}
}
],
@@ -340,7 +345,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
}
},
@@ -426,7 +431,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
},
"400": {
@@ -481,7 +486,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate"
"$ref": "#/definitions/repository.TokenUpdate"
}
}
],
@@ -661,7 +666,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset"
"$ref": "#/definitions/repository.TokenPasswordReset"
}
}
],
@@ -729,7 +734,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions"
"$ref": "#/definitions/repository.TokenUpdatePermissions"
}
}
],
@@ -789,7 +794,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
}
},
@@ -819,7 +824,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
},
"401": {
@@ -836,6 +841,11 @@ const docTemplate = `{
},
"/logs": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Searches logs with various filters",
"produces": [
"application/json"
@@ -896,13 +906,18 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
"$ref": "#/definitions/storage.LogEntry"
}
}
}
}
},
"post": {
"security": [
{
"Bearer": []
}
],
"description": "Inserts a single log entry into ClickHouse",
"consumes": [
"application/json"
@@ -921,7 +936,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
],
@@ -940,6 +955,11 @@ const docTemplate = `{
},
"/logs/agents": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique agent names in logs",
"produces": [
"application/json"
@@ -963,6 +983,11 @@ const docTemplate = `{
},
"/logs/batch": {
"post": {
"security": [
{
"Bearer": []
}
],
"description": "Inserts multiple log entries into ClickHouse",
"consumes": [
"application/json"
@@ -981,7 +1006,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.InsertLogsRequest"
"$ref": "#/definitions/handlers.InsertLogsRequest"
}
}
],
@@ -1000,6 +1025,11 @@ const docTemplate = `{
},
"/logs/levels": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique log levels in logs",
"produces": [
"application/json"
@@ -1076,7 +1106,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
"$ref": "#/definitions/storage.LogEntry"
}
}
}
@@ -1085,6 +1115,11 @@ const docTemplate = `{
},
"/logs/services": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique service names in logs",
"produces": [
"application/json"
@@ -1108,7 +1143,93 @@ const docTemplate = `{
}
},
"definitions": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": {
"handlers.AgentInfo": {
"type": "object",
"properties": {
"connected_at": {
"type": "string"
},
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
}
},
"handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
},
"repository.AgentDeployConfig": {
"description": "Configuration for deploying HellreigN agent to a single server",
"type": "object",
"required": [
@@ -1126,7 +1247,7 @@ const docTemplate = `{
"authMethod": {
"allOf": [
{
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod"
"$ref": "#/definitions/repository.AuthMethod"
}
],
"example": "key"
@@ -1134,7 +1255,7 @@ const docTemplate = `{
"deployType": {
"allOf": [
{
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType"
"$ref": "#/definitions/repository.DeployType"
}
],
"example": "docker"
@@ -1161,7 +1282,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": {
"repository.AuthMethod": {
"description": "SSH authentication method: key or password",
"type": "string",
"enum": [
@@ -1173,7 +1294,7 @@ const docTemplate = `{
"AuthMethodPassword"
]
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": {
"repository.DeployAgentsRequest": {
"description": "Request to deploy HellreigN agents to multiple servers",
"type": "object",
"required": [
@@ -1184,12 +1305,12 @@ const docTemplate = `{
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig"
"$ref": "#/definitions/repository.AgentDeployConfig"
}
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": {
"repository.DeployResponse": {
"description": "Response containing deployment results and registration tokens",
"type": "object",
"properties": {
@@ -1200,12 +1321,12 @@ const docTemplate = `{
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult"
"$ref": "#/definitions/repository.DeployResult"
}
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": {
"repository.DeployResult": {
"description": "Result of deploying to a single server",
"type": "object",
"properties": {
@@ -1231,7 +1352,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": {
"repository.DeployType": {
"description": "Type of deployment: docker or binary",
"type": "string",
"enum": [
@@ -1243,7 +1364,7 @@ const docTemplate = `{
"DeployTypeBinary"
]
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": {
"repository.LoginRequest": {
"type": "object",
"required": [
"login",
@@ -1258,7 +1379,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": {
"repository.LoginResponse": {
"type": "object",
"properties": {
"is_active": {
@@ -1287,7 +1408,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": {
"repository.RegistrationRequest": {
"type": "object",
"required": [
"label"
@@ -1298,7 +1419,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": {
"repository.TokenCreate": {
"type": "object",
"required": [
"last_name",
@@ -1333,7 +1454,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": {
"repository.TokenPasswordReset": {
"type": "object",
"required": [
"new_password"
@@ -1344,7 +1465,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": {
"repository.TokenUpdate": {
"type": "object",
"properties": {
"last_name": {
@@ -1355,7 +1476,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": {
"repository.TokenUpdatePermissions": {
"type": "object",
"properties": {
"is_active": {
@@ -1372,7 +1493,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": {
"repository.Tokens": {
"type": "object",
"properties": {
"id": {
@@ -1404,7 +1525,7 @@ const docTemplate = `{
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": {
"storage.LogEntry": {
"type": "object",
"properties": {
"agent": {
@@ -1423,89 +1544,6 @@ const docTemplate = `{
"type": "string"
}
}
},
"internal_handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"internal_handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"internal_handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
}
},
"internal_handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"internal_handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
}
},
"securityDefinitions": {
backend/docs/swagger.json
+161 -123
View File
@@ -6,7 +6,12 @@
"paths": {
"/agents": {
"get": {
"description": "Returns a list of all agents currently connected via gRPC streaming",
"security": [
{
"Bearer": []
}
],
"description": "Returns a list of all agents currently connected via Collector (log streaming)",
"produces": [
"application/json"
],
@@ -20,7 +25,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.AgentInfo"
"$ref": "#/definitions/handlers.AgentInfo"
}
}
}
@@ -52,7 +57,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest"
"$ref": "#/definitions/repository.DeployAgentsRequest"
}
}
],
@@ -60,7 +65,7 @@
"200": {
"description": "Deployment results with tokens for each server",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse"
"$ref": "#/definitions/repository.DeployResponse"
}
},
"400": {
@@ -103,7 +108,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.RegisterRequest"
"$ref": "#/definitions/handlers.RegisterRequest"
}
}
],
@@ -111,7 +116,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/internal_handlers.RegisterResponse"
"$ref": "#/definitions/handlers.RegisterResponse"
}
}
}
@@ -141,7 +146,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest"
"$ref": "#/definitions/repository.RegistrationRequest"
}
}
],
@@ -175,7 +180,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest"
"$ref": "#/definitions/repository.LoginRequest"
}
}
],
@@ -183,7 +188,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse"
"$ref": "#/definitions/repository.LoginResponse"
}
},
"400": {
@@ -233,7 +238,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate"
"$ref": "#/definitions/repository.TokenCreate"
}
}
],
@@ -329,7 +334,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
}
},
@@ -415,7 +420,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
},
"400": {
@@ -470,7 +475,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate"
"$ref": "#/definitions/repository.TokenUpdate"
}
}
],
@@ -650,7 +655,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset"
"$ref": "#/definitions/repository.TokenPasswordReset"
}
}
],
@@ -718,7 +723,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions"
"$ref": "#/definitions/repository.TokenUpdatePermissions"
}
}
],
@@ -778,7 +783,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
}
},
@@ -808,7 +813,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
"$ref": "#/definitions/repository.Tokens"
}
},
"401": {
@@ -825,6 +830,11 @@
},
"/logs": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Searches logs with various filters",
"produces": [
"application/json"
@@ -885,13 +895,18 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
"$ref": "#/definitions/storage.LogEntry"
}
}
}
}
},
"post": {
"security": [
{
"Bearer": []
}
],
"description": "Inserts a single log entry into ClickHouse",
"consumes": [
"application/json"
@@ -910,7 +925,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
],
@@ -929,6 +944,11 @@
},
"/logs/agents": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique agent names in logs",
"produces": [
"application/json"
@@ -952,6 +972,11 @@
},
"/logs/batch": {
"post": {
"security": [
{
"Bearer": []
}
],
"description": "Inserts multiple log entries into ClickHouse",
"consumes": [
"application/json"
@@ -970,7 +995,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handlers.InsertLogsRequest"
"$ref": "#/definitions/handlers.InsertLogsRequest"
}
}
],
@@ -989,6 +1014,11 @@
},
"/logs/levels": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique log levels in logs",
"produces": [
"application/json"
@@ -1065,7 +1095,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
"$ref": "#/definitions/storage.LogEntry"
}
}
}
@@ -1074,6 +1104,11 @@
},
"/logs/services": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns list of all unique service names in logs",
"produces": [
"application/json"
@@ -1097,7 +1132,93 @@
}
},
"definitions": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": {
"handlers.AgentInfo": {
"type": "object",
"properties": {
"connected_at": {
"type": "string"
},
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
}
},
"handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
},
"repository.AgentDeployConfig": {
"description": "Configuration for deploying HellreigN agent to a single server",
"type": "object",
"required": [
@@ -1115,7 +1236,7 @@
"authMethod": {
"allOf": [
{
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod"
"$ref": "#/definitions/repository.AuthMethod"
}
],
"example": "key"
@@ -1123,7 +1244,7 @@
"deployType": {
"allOf": [
{
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType"
"$ref": "#/definitions/repository.DeployType"
}
],
"example": "docker"
@@ -1150,7 +1271,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": {
"repository.AuthMethod": {
"description": "SSH authentication method: key or password",
"type": "string",
"enum": [
@@ -1162,7 +1283,7 @@
"AuthMethodPassword"
]
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": {
"repository.DeployAgentsRequest": {
"description": "Request to deploy HellreigN agents to multiple servers",
"type": "object",
"required": [
@@ -1173,12 +1294,12 @@
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig"
"$ref": "#/definitions/repository.AgentDeployConfig"
}
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": {
"repository.DeployResponse": {
"description": "Response containing deployment results and registration tokens",
"type": "object",
"properties": {
@@ -1189,12 +1310,12 @@
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult"
"$ref": "#/definitions/repository.DeployResult"
}
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": {
"repository.DeployResult": {
"description": "Result of deploying to a single server",
"type": "object",
"properties": {
@@ -1220,7 +1341,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": {
"repository.DeployType": {
"description": "Type of deployment: docker or binary",
"type": "string",
"enum": [
@@ -1232,7 +1353,7 @@
"DeployTypeBinary"
]
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": {
"repository.LoginRequest": {
"type": "object",
"required": [
"login",
@@ -1247,7 +1368,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": {
"repository.LoginResponse": {
"type": "object",
"properties": {
"is_active": {
@@ -1276,7 +1397,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": {
"repository.RegistrationRequest": {
"type": "object",
"required": [
"label"
@@ -1287,7 +1408,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": {
"repository.TokenCreate": {
"type": "object",
"required": [
"last_name",
@@ -1322,7 +1443,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": {
"repository.TokenPasswordReset": {
"type": "object",
"required": [
"new_password"
@@ -1333,7 +1454,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": {
"repository.TokenUpdate": {
"type": "object",
"properties": {
"last_name": {
@@ -1344,7 +1465,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": {
"repository.TokenUpdatePermissions": {
"type": "object",
"properties": {
"is_active": {
@@ -1361,7 +1482,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": {
"repository.Tokens": {
"type": "object",
"properties": {
"id": {
@@ -1393,7 +1514,7 @@
}
}
},
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": {
"storage.LogEntry": {
"type": "object",
"properties": {
"agent": {
@@ -1412,89 +1533,6 @@
"type": "string"
}
}
},
"internal_handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"internal_handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"internal_handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
}
},
"internal_handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"internal_handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
}
},
"securityDefinitions": {
backend/docs/swagger.yaml
+112 -95
View File
@@ -1,5 +1,62 @@
definitions:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig:
handlers.AgentInfo:
properties:
connected_at:
type: string
label:
type: string
services:
items:
type: string
type: array
token:
type: string
type: object
handlers.InsertLogRequest:
properties:
agent:
type: string
level:
type: string
message:
type: string
service:
type: string
timestamp:
type: string
required:
- agent
- level
- message
- service
type: object
handlers.InsertLogsRequest:
properties:
logs:
items:
$ref: '#/definitions/handlers.InsertLogRequest'
type: array
required:
- logs
type: object
handlers.RegisterRequest:
properties:
csr:
type: string
token:
type: string
required:
- csr
- token
type: object
handlers.RegisterResponse:
properties:
ca_cert:
type: string
client_cert:
type: string
type: object
repository.AgentDeployConfig:
description: Configuration for deploying HellreigN agent to a single server
properties:
agentLabel:
@@ -7,11 +64,11 @@ definitions:
type: string
authMethod:
allOf:
- $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod'
- $ref: '#/definitions/repository.AuthMethod'
example: key
deployType:
allOf:
- $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType'
- $ref: '#/definitions/repository.DeployType'
example: docker
ip:
example: 192.168.1.100
@@ -35,7 +92,7 @@ definitions:
- ip
- user
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod:
repository.AuthMethod:
description: 'SSH authentication method: key or password'
enum:
- key
@@ -44,18 +101,18 @@ definitions:
x-enum-varnames:
- AuthMethodKey
- AuthMethodPassword
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest:
repository.DeployAgentsRequest:
description: Request to deploy HellreigN agents to multiple servers
properties:
servers:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig'
$ref: '#/definitions/repository.AgentDeployConfig'
minItems: 1
type: array
required:
- servers
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse:
repository.DeployResponse:
description: Response containing deployment results and registration tokens
properties:
message:
@@ -63,10 +120,10 @@ definitions:
type: string
results:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult'
$ref: '#/definitions/repository.DeployResult'
type: array
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult:
repository.DeployResult:
description: Result of deploying to a single server
properties:
agent_label:
@@ -85,7 +142,7 @@ definitions:
example: abc123...
type: string
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType:
repository.DeployType:
description: 'Type of deployment: docker or binary'
enum:
- docker
@@ -94,7 +151,7 @@ definitions:
x-enum-varnames:
- DeployTypeDocker
- DeployTypeBinary
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest:
repository.LoginRequest:
properties:
login:
type: string
@@ -104,7 +161,7 @@ definitions:
- login
- password
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse:
repository.LoginResponse:
properties:
is_active:
type: boolean
@@ -123,14 +180,14 @@ definitions:
token:
type: string
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest:
repository.RegistrationRequest:
properties:
label:
type: string
required:
- label
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate:
repository.TokenCreate:
properties:
is_active:
type: boolean
@@ -154,21 +211,21 @@ definitions:
- name
- password
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset:
repository.TokenPasswordReset:
properties:
new_password:
type: string
required:
- new_password
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate:
repository.TokenUpdate:
properties:
last_name:
type: string
name:
type: string
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions:
repository.TokenUpdatePermissions:
properties:
is_active:
type: boolean
@@ -179,7 +236,7 @@ definitions:
permission_view:
type: boolean
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens:
repository.Tokens:
properties:
id:
type: integer
@@ -200,7 +257,7 @@ definitions:
token:
type: string
type: object
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry:
storage.LogEntry:
properties:
agent:
type: string
@@ -213,67 +270,13 @@ definitions:
timestamp:
type: string
type: object
internal_handlers.AgentInfo:
properties:
label:
type: string
services:
items:
type: string
type: array
token:
type: string
type: object
internal_handlers.InsertLogRequest:
properties:
agent:
type: string
level:
type: string
message:
type: string
service:
type: string
timestamp:
type: string
required:
- agent
- level
- message
- service
type: object
internal_handlers.InsertLogsRequest:
properties:
logs:
items:
$ref: '#/definitions/internal_handlers.InsertLogRequest'
type: array
required:
- logs
type: object
internal_handlers.RegisterRequest:
properties:
csr:
type: string
token:
type: string
required:
- csr
- token
type: object
internal_handlers.RegisterResponse:
properties:
ca_cert:
type: string
client_cert:
type: string
type: object
info:
contact: {}
paths:
/agents:
get:
description: Returns a list of all agents currently connected via gRPC streaming
description: Returns a list of all agents currently connected via Collector
(log streaming)
produces:
- application/json
responses:
@@ -281,8 +284,10 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/internal_handlers.AgentInfo'
$ref: '#/definitions/handlers.AgentInfo'
type: array
security:
- Bearer: []
summary: Get connected agents
tags:
- agents
@@ -298,14 +303,14 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest'
$ref: '#/definitions/repository.DeployAgentsRequest'
produces:
- application/json
responses:
"200":
description: Deployment results with tokens for each server
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse'
$ref: '#/definitions/repository.DeployResponse'
"400":
description: Invalid request
schema:
@@ -333,14 +338,14 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/internal_handlers.RegisterRequest'
$ref: '#/definitions/handlers.RegisterRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/internal_handlers.RegisterResponse'
$ref: '#/definitions/handlers.RegisterResponse'
summary: Register agent
tags:
- agents
@@ -354,7 +359,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest'
$ref: '#/definitions/repository.RegistrationRequest'
produces:
- application/json
responses:
@@ -380,12 +385,12 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest'
$ref: '#/definitions/repository.LoginRequest'
responses:
"200":
description: OK
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse'
$ref: '#/definitions/repository.LoginResponse'
"400":
description: Bad Request
schema:
@@ -442,7 +447,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate'
$ref: '#/definitions/repository.TokenCreate'
responses:
"200":
description: OK
@@ -481,7 +486,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
$ref: '#/definitions/repository.Tokens'
type: array
"500":
description: Internal Server Error
@@ -538,7 +543,7 @@ paths:
"200":
description: OK
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
$ref: '#/definitions/repository.Tokens'
"400":
description: Bad Request
schema:
@@ -575,7 +580,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate'
$ref: '#/definitions/repository.TokenUpdate'
responses:
"200":
description: OK
@@ -694,7 +699,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset'
$ref: '#/definitions/repository.TokenPasswordReset'
responses:
"200":
description: OK
@@ -739,7 +744,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions'
$ref: '#/definitions/repository.TokenUpdatePermissions'
responses:
"200":
description: OK
@@ -778,7 +783,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
$ref: '#/definitions/repository.Tokens'
type: array
"500":
description: Internal Server Error
@@ -798,7 +803,7 @@ paths:
"200":
description: OK
schema:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
$ref: '#/definitions/repository.Tokens'
"401":
description: Unauthorized
schema:
@@ -849,8 +854,10 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry'
$ref: '#/definitions/storage.LogEntry'
type: array
security:
- Bearer: []
summary: Search logs
tags:
- logs
@@ -864,7 +871,7 @@ paths:
name: body
required: true
schema:
$ref: '#/definitions/internal_handlers.InsertLogRequest'
$ref: '#/definitions/handlers.InsertLogRequest'
produces:
- application/json
responses:
@@ -874,6 +881,8 @@ paths:
additionalProperties:
type: string
type: object
security:
- Bearer: []
summary: Insert log entry
tags:
- logs
@@ -889,6 +898,8 @@ paths:
items:
type: string
type: array
security:
- Bearer: []
summary: Get distinct agents
tags:
- logs
@@ -903,7 +914,7 @@ paths:
name: body
required: true
schema:
$ref: '#/definitions/internal_handlers.InsertLogsRequest'
$ref: '#/definitions/handlers.InsertLogsRequest'
produces:
- application/json
responses:
@@ -913,6 +924,8 @@ paths:
additionalProperties:
type: string
type: object
security:
- Bearer: []
summary: Insert log entries (batch)
tags:
- logs
@@ -928,6 +941,8 @@ paths:
items:
type: string
type: array
security:
- Bearer: []
summary: Get distinct log levels
tags:
- logs
@@ -965,7 +980,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry'
$ref: '#/definitions/storage.LogEntry'
type: array
security:
- Bearer: []
@@ -984,6 +999,8 @@ paths:
items:
type: string
type: array
security:
- Bearer: []
summary: Get distinct services
tags:
- logs
backend/internal/grpcsrv/collector/collector.go
+2 -2
View File
@@ -81,8 +81,8 @@ func (c *Collector) Stream(stream proto.Collector_StreamServer) error {
log.Printf("Agent %s connected, streaming logs for service: %s", agentName, service)
// If no ClickHouse, just consume the stream without storing
if c.logRepo == nil {
log.Printf("Warning: logRepo is nil, consuming logs without storing for agent %s", agentName)
if !c.logRepo.IsConnected() {
log.Printf("Warning: ClickHouse not connected yet, consuming logs without storing for agent %s", agentName)
for {
_, err := stream.Recv()
if err == io.EOF {
backend/internal/handlers/agents.go
+1
View File
@@ -25,6 +25,7 @@ type AgentInfo struct {
// @Summary Get connected agents
// @Description Returns a list of all agents currently connected via Collector (log streaming)
// @Tags agents
// @Security Bearer
// @Produce json
// @Success 200 {array} AgentInfo
// @Router /agents [get]
backend/internal/handlers/logs.go
+6
View File
@@ -33,6 +33,7 @@ type InsertLogRequest struct {
// @Produce json
// @Param body body InsertLogRequest true "Log entry"
// @Success 201 {object} map[string]string
// @Security Bearer
// @Router /logs [post]
func (lh *LogHandlers) Insert(c *gin.Context) {
var req InsertLogRequest
@@ -72,6 +73,7 @@ type InsertLogsRequest struct {
// @Produce json
// @Param body body InsertLogsRequest true "Log entries"
// @Success 201 {object} map[string]string
// @Security Bearer
// @Router /logs/batch [post]
func (lh *LogHandlers) InsertBatch(c *gin.Context) {
var req InsertLogsRequest
@@ -124,6 +126,7 @@ type SearchLogsRequest struct {
// @Param limit query int false "Limit results" default(100)
// @Param offset query int false "Offset results" default(0)
// @Success 200 {array} storage.LogEntry
// @Security Bearer
// @Router /logs [get]
func (lh *LogHandlers) Search(c *gin.Context) {
var req SearchLogsRequest
@@ -170,6 +173,7 @@ func (lh *LogHandlers) Search(c *gin.Context) {
// @Tags logs
// @Produce json
// @Success 200 {array} string
// @Security Bearer
// @Router /logs/services [get]
func (lh *LogHandlers) GetServices(c *gin.Context) {
services, err := lh.LogRepo.GetDistinctServices(c.Request.Context())
@@ -190,6 +194,7 @@ func (lh *LogHandlers) GetServices(c *gin.Context) {
// @Tags logs
// @Produce json
// @Success 200 {array} string
// @Security Bearer
// @Router /logs/agents [get]
func (lh *LogHandlers) GetAgents(c *gin.Context) {
agents, err := lh.LogRepo.GetDistinctAgents(c.Request.Context())
@@ -210,6 +215,7 @@ func (lh *LogHandlers) GetAgents(c *gin.Context) {
// @Tags logs
// @Produce json
// @Success 200 {array} string
// @Security Bearer
// @Router /logs/levels [get]
func (lh *LogHandlers) GetLevels(c *gin.Context) {
levels, err := lh.LogRepo.GetDistinctLevels(c.Request.Context())
backend/internal/repository/log_repository.go
+79 -21
View File
@@ -2,44 +2,85 @@ package repository
import (
"context"
"database/sql"
"fmt"
"sync"
"time"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
type LogRepository struct {
Conn driver.Conn
mu sync.RWMutex
DB *sql.DB
}
func NewLogRepository(conn driver.Conn) *LogRepository {
return &LogRepository{Conn: conn}
func NewLogRepository() *LogRepository {
return &LogRepository{}
}
func (r *LogRepository) SetDB(db *sql.DB) {
r.mu.Lock()
defer r.mu.Unlock()
r.DB = db
}
func (r *LogRepository) IsConnected() bool {
r.mu.RLock()
defer r.mu.RUnlock()
return r.DB != nil
}
func (r *LogRepository) getDB() *sql.DB {
r.mu.RLock()
defer r.mu.RUnlock()
return r.DB
}
func (r *LogRepository) Init(ctx context.Context) error {
return r.Conn.Exec(ctx, storage.CreateLogsTable)
db := r.getDB()
if db == nil {
return nil
}
_, err := db.ExecContext(ctx, storage.CreateLogsTable)
return err
}
func (r *LogRepository) Insert(ctx context.Context, log storage.LogEntry) error {
return r.Conn.Exec(ctx, `
db := r.getDB()
if db == nil {
return nil
}
_, err := db.ExecContext(ctx, `
INSERT INTO logs (timestamp, level, service, agent, message)
VALUES ($1, $2, $3, $4, $5)
`, log.Timestamp, log.Level, log.Service, log.Agent, log.Message)
return err
}
func (r *LogRepository) InsertBatch(ctx context.Context, logs []storage.LogEntry) error {
batch, err := r.Conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, level, service, agent, message)")
if err != nil {
return err
db := r.getDB()
if db == nil {
return nil
}
if len(logs) == 0 {
return nil
}
for _, log := range logs {
if err := batch.Append(log.Timestamp, log.Level, log.Service, log.Agent, log.Message); err != nil {
return err
// Build multi-row INSERT statement
query := "INSERT INTO logs (timestamp, level, service, agent, message) VALUES "
args := make([]interface{}, 0, len(logs)*5)
for i, log := range logs {
if i > 0 {
query += ", "
}
query += fmt.Sprintf("($%d, $%d, $%d, $%d, $%d)",
i*5+1, i*5+2, i*5+3, i*5+4, i*5+5)
args = append(args, log.Timestamp, log.Level, log.Service, log.Agent, log.Message)
}
return batch.Send()
_, err := db.ExecContext(ctx, query, args...)
return err
}
type LogFilter struct {
@@ -53,6 +94,11 @@ type LogFilter struct {
}
func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage.LogEntry, error) {
db := r.getDB()
if db == nil {
return []storage.LogEntry{}, nil
}
query := "SELECT timestamp, level, service, agent, message FROM logs WHERE 1=1"
args := make([]interface{}, 0)
argIdx := 1
@@ -102,13 +148,13 @@ func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage
args = append(args, filter.Offset)
}
rows, err := r.Conn.Query(ctx, query, args...)
rows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var logs []storage.LogEntry
logs := make([]storage.LogEntry, 0)
for rows.Next() {
var log storage.LogEntry
if err := rows.Scan(&log.Timestamp, &log.Level, &log.Service, &log.Agent, &log.Message); err != nil {
@@ -121,13 +167,17 @@ func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage
}
func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, error) {
rows, err := r.Conn.Query(ctx, "SELECT DISTINCT service FROM logs ORDER BY service")
db := r.getDB()
if db == nil {
return []string{}, nil
}
rows, err := db.QueryContext(ctx, "SELECT DISTINCT service FROM logs ORDER BY service")
if err != nil {
return nil, err
}
defer rows.Close()
var services []string
services := make([]string, 0)
for rows.Next() {
var service string
if err := rows.Scan(&service); err != nil {
@@ -140,13 +190,17 @@ func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, erro
}
func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error) {
rows, err := r.Conn.Query(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent")
db := r.getDB()
if db == nil {
return []string{}, nil
}
rows, err := db.QueryContext(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent")
if err != nil {
return nil, err
}
defer rows.Close()
var agents []string
agents := make([]string, 0)
for rows.Next() {
var agent string
if err := rows.Scan(&agent); err != nil {
@@ -159,13 +213,17 @@ func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error)
}
func (r *LogRepository) GetDistinctLevels(ctx context.Context) ([]string, error) {
rows, err := r.Conn.Query(ctx, "SELECT DISTINCT level FROM logs ORDER BY level")
db := r.getDB()
if db == nil {
return []string{}, nil
}
rows, err := db.QueryContext(ctx, "SELECT DISTINCT level FROM logs ORDER BY level")
if err != nil {
return nil, err
}
defer rows.Close()
var levels []string
levels := make([]string, 0)
for rows.Next() {
var level string
if err := rows.Scan(&level); err != nil {
backend/internal/storage/clickhouse.go
+40 -25
View File
@@ -2,10 +2,12 @@ package storage
import (
"context"
"database/sql"
"fmt"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
_ "github.com/ClickHouse/clickhouse-go/v2"
)
type ClickHouseConfig struct {
@@ -15,33 +17,46 @@ type ClickHouseConfig struct {
Database string
}
func OpenClickHouse(cfg ClickHouseConfig) (driver.Conn, error) {
conn, err := clickhouse.Open(&clickhouse.Options{
Addr: []string{cfg.Host},
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.User,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": 60,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: 30,
MaxOpenConns: 10,
MaxIdleConns: 5,
ConnMaxLifetime: 3600,
ConnOpenStrategy: clickhouse.ConnOpenInOrder,
})
func OpenClickHouse(cfg ClickHouseConfig) (*sql.DB, error) {
dsn := fmt.Sprintf("clickhouse://%s:%s@%s/%s",
cfg.User, cfg.Password, cfg.Host, cfg.Database)
db, err := sql.Open("clickhouse", dsn)
if err != nil {
return nil, fmt.Errorf("clickhouse connect: %w", err)
return nil, fmt.Errorf("clickhouse open: %w", err)
}
if err := conn.Ping(context.Background()); err != nil {
db.SetMaxOpenConns(5)
db.SetMaxIdleConns(2)
db.SetConnMaxLifetime(10 * time.Minute)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := db.PingContext(ctx); err != nil {
db.Close()
return nil, fmt.Errorf("clickhouse ping: %w", err)
}
return conn, nil
log.Printf("ClickHouse connected via database/sql: %s", cfg.Host)
return db, nil
}
// OpenClickHouseWithRetry attempts to connect to ClickHouse with retries and backoff.
func OpenClickHouseWithRetry(cfg ClickHouseConfig, maxRetries int, initialDelay time.Duration) (*sql.DB, error) {
var lastErr error
delay := initialDelay
for i := 0; i < maxRetries; i++ {
db, err := OpenClickHouse(cfg)
if err == nil {
return db, nil
}
lastErr = err
log.Printf("ClickHouse connection attempt %d/%d failed: %v, retrying in %v...", i+1, maxRetries, err, delay)
time.Sleep(delay)
delay *= 2
}
return nil, fmt.Errorf("clickhouse connection failed after %d attempts: %w", maxRetries, lastErr)
}
infra/agent/config.yml
+3
View File
@@ -3,3 +3,6 @@ grpc_url: backend:9001
label: test-agent-1
registration_token: "156616b56774d59ba53f1eb4b096488bb5f755bbf5b737d93a42bb1b583ad7fb"
cert_dir: /etc/hellreign-agent/certs
services:
- name: system
type: journald
infra/clickhouse/init/01_create_logs_table.sh
+6 -6
View File
@@ -6,11 +6,11 @@ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS hellreign;"
clickhouse-client --query "
CREATE TABLE IF NOT EXISTS hellreign.logs (
timestamp DateTime64(3) DEFAULT now(),
level String,
service String,
message String,
host String,
trace_id String
level LowCardinality(String),
service LowCardinality(String),
agent LowCardinality(String),
message String
) ENGINE = MergeTree()
ORDER BY (timestamp, service, level);
ORDER BY (timestamp, level, service, agent)
SETTINGS index_granularity = 8192;
"
infra/docker-compose.yml
+14 -1
View File
@@ -13,6 +13,12 @@ services:
volumes:
- clickhouse_data:/var/lib/clickhouse
- ./clickhouse/init:/docker-entrypoint-initdb.d
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
interval: 5s
timeout: 3s
retries: 20
start_period: 30s
networks:
- hellreign
@@ -33,7 +39,8 @@ services:
- ./backend/config.yml:/etc/hellreign/config.yml:ro
- backend_data:/var/lib/hellreign
depends_on:
- clickhouse
clickhouse:
condition: service_healthy
networks:
- hellreign
@@ -56,9 +63,13 @@ services:
container_name: hellreign-agent
environment:
CONFIG_FILE: /etc/hellreign-agent/config.yml
JOURNALD_LOGDIR: /var/log/journal
BUFFER_DB: /var/lib/hellreign-agent/agent_buffer.db
volumes:
- ./agent/config.yml:/etc/hellreign-agent/config.yml:ro
- agent_certs:/etc/hellreign-agent/certs
- agent_data:/var/lib/hellreign-agent
- /var/log/journal:/var/log/journal:ro
depends_on:
- backend
networks:
@@ -71,6 +82,8 @@ volumes:
driver: local
agent_certs:
driver: local
agent_data:
driver: local
networks:
hellreign: