d3m0k1d/HellreigN
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

chore: add logparser logic for agent and add parsed log to clickhouse
ci-agent / build (push) Failing after 3m30s
Details

Browse Source
This commit is contained in:
d3m0k1d
2026-04-04 06:29:07 +03:00
parent c59d122e04
commit 477dd94227
16 changed files with 1226 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/go.mod
+14
View File
@@ -4,12 +4,21 @@ go 1.26.1
require (
gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto v0.0.0-20260403214837-94be9799f47d
github.com/hpcloud/tail v1.0.0
github.com/samber/lo v1.53.0
golang.org/x/sync v0.20.0
google.golang.org/grpc v1.80.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.34.5
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
go.opentelemetry.io/otel/metric v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.41.0 // indirect
golang.org/x/net v0.52.0 // indirect
@@ -17,6 +26,11 @@ require (
golang.org/x/text v0.35.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
modernc.org/libc v1.55.3 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
)
replace gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto => ../proto
agent/go.sum
+49
View File
@@ -1,5 +1,9 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -8,8 +12,20 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
@@ -22,14 +38,19 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU=
@@ -40,5 +61,33 @@ google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBN
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
agent/internal/buffer/buffer.go
+161
View File
@@ -0,0 +1,161 @@
package buffer
import (
"database/sql"
"encoding/json"
"fmt"
"time"
_ "modernc.org/sqlite"
)
// BufferedLog represents a log entry stored for later delivery
type BufferedLog struct {
ID int64
Service string
Message string
CreatedAt time.Time
}
// LogBuffer provides SQLite-backed log buffering
type LogBuffer struct {
db *sql.DB
}
// NewLogBuffer creates a new log buffer with the given database path
func NewLogBuffer(dbPath string) (*LogBuffer, error) {
db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000")
if err != nil {
return nil, fmt.Errorf("failed to open database: %w", err)
}
// Create table if not exists
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS buffered_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
service TEXT NOT NULL,
message TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`)
if err != nil {
_ = db.Close()
return nil, fmt.Errorf("failed to create table: %w", err)
}
// Create index for efficient ordering
_, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_created_at ON buffered_logs(created_at ASC)`)
return &LogBuffer{db: db}, nil
}
// Close closes the database connection
func (b *LogBuffer) Close() error {
return b.db.Close()
}
// Store stores a log entry in the buffer
func (b *LogBuffer) Store(service, message string) error {
_, err := b.db.Exec(
"INSERT INTO buffered_logs (service, message) VALUES (?, ?)",
service, message,
)
return err
}
// StoreBatch stores multiple log entries in a single transaction
func (b *LogBuffer) StoreBatch(entries []BufferedLog) error {
tx, err := b.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
stmt, err := tx.Prepare("INSERT INTO buffered_logs (service, message) VALUES (?, ?)")
if err != nil {
return err
}
defer stmt.Close()
for _, entry := range entries {
if _, err := stmt.Exec(entry.Service, entry.Message); err != nil {
return err
}
}
return tx.Commit()
}
// GetPending retrieves pending logs in order of arrival, limited to batchSize
func (b *LogBuffer) GetPending(batchSize int) ([]BufferedLog, error) {
rows, err := b.db.Query(
"SELECT id, service, message, created_at FROM buffered_logs ORDER BY created_at ASC LIMIT ?",
batchSize,
)
if err != nil {
return nil, err
}
defer rows.Close()
var logs []BufferedLog
for rows.Next() {
var log BufferedLog
var createdAt string
if err := rows.Scan(&log.ID, &log.Service, &log.Message, &createdAt); err != nil {
return nil, err
}
log.CreatedAt, _ = time.Parse("2006-01-02 15:04:05", createdAt)
logs = append(logs, log)
}
return logs, rows.Err()
}
// Delete removes a log entry from the buffer after successful delivery
func (b *LogBuffer) Delete(id int64) error {
_, err := b.db.Exec("DELETE FROM buffered_logs WHERE id = ?", id)
return err
}
// DeleteBatch removes multiple log entries after successful delivery
func (b *LogBuffer) DeleteBatch(ids []int64) error {
if len(ids) == 0 {
return nil
}
tx, err := b.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
for _, id := range ids {
if _, err := tx.Exec("DELETE FROM buffered_logs WHERE id = ?", id); err != nil {
return err
}
}
return tx.Commit()
}
// Count returns the number of buffered logs
func (b *LogBuffer) Count() (int, error) {
var count int
err := b.db.QueryRow("SELECT COUNT(*) FROM buffered_logs").Scan(&count)
return count, err
}
// Clear removes all buffered logs
func (b *LogBuffer) Clear() error {
_, err := b.db.Exec("DELETE FROM buffered_logs")
return err
}
// FlushToJSON exports buffered logs to JSON format for debugging
func (b *LogBuffer) FlushToJSON() ([]byte, error) {
logs, err := b.GetPending(1000)
if err != nil {
return nil, err
}
return json.MarshalIndent(logs, "", " ")
}
agent/internal/config/config.go
+29 -5
View File
@@ -1,17 +1,25 @@
package config
import (
"fmt"
"os"
"gopkg.in/yaml.v3"
)
type ServiceConfig struct {
Name string `yaml:"name"`
Type string `yaml:"type"`
Path *string `yaml:"path"`
}
type AgentConfig struct {
BackendURL string `yaml:"backend_url"`
GRPCURL string `yaml:"grpc_url"`
RegistrationToken string `yaml:"registration_token"`
Label string `yaml:"label"`
CertDir string `yaml:"cert_dir"`
BackendURL string `yaml:"backend_url"`
GRPCURL string `yaml:"grpc_url"`
RegistrationToken string `yaml:"registration_token"`
Label string `yaml:"label"`
CertDir string `yaml:"cert_dir"`
Services []ServiceConfig `yaml:"services"`
}
func Load(path string) (*AgentConfig, error) {
@@ -31,3 +39,19 @@ func Load(path string) (*AgentConfig, error) {
return &cfg, nil
}
func LoadFromString(data string) (*AgentConfig, error) {
var cfg AgentConfig
if err := yaml.Unmarshal([]byte(data), &cfg); err != nil {
return nil, err
}
return &cfg, nil
}
func validateConfigPath(path string) error {
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("config file not found: %w", err)
}
return nil
}
agent/internal/logger/logger.go
+27
View File
@@ -0,0 +1,27 @@
package logger
import (
"log/slog"
"os"
)
type Logger struct {
*slog.Logger
}
func New(debug bool) *Logger {
var level slog.Level
if debug {
level = slog.LevelDebug
} else {
level = slog.LevelInfo
}
handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: level,
})
return &Logger{
Logger: slog.New(handler),
}
}
agent/internal/logsource/file/impl.go
+45
View File
@@ -0,0 +1,45 @@
package file
import (
"errors"
"os"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource"
"github.com/hpcloud/tail"
)
var _ logsource.LogSource = new(FileLogSource)
type FileLogSource struct {
*tail.Tail
}
func New(filepath string) (fls *FileLogSource, err error) {
if _, err := os.Stat(filepath); os.IsNotExist(err) {
if err := os.WriteFile(filepath, []byte{}, 0600); err != nil {
return nil, err
}
}
t, err := tail.TailFile(filepath, tail.Config{
Follow: true,
Location: &tail.SeekInfo{
Offset: 100,
Whence: 2,
},
})
if err != nil {
return
}
return &FileLogSource{t}, nil
}
func (f *FileLogSource) ReadLine() (string, error) {
select {
case <-f.Dead():
return "", errors.Join(logsource.ErrDead, f.Err())
case line := <-f.Lines:
return line.Text, line.Err
}
}
func (f *FileLogSource) Close() error {
return f.Stop()
}
agent/internal/logsource/interface.go
+10
View File
@@ -0,0 +1,10 @@
package logsource
import "errors"
type LogSource interface {
ReadLine() (string, error)
Close() error
}
var ErrDead = errors.New("shouldn't continue to read that")
agent/internal/logsource/journald/impl.go
+55
View File
@@ -0,0 +1,55 @@
package journald
import (
"bufio"
"fmt"
"io"
"os/exec"
"syscall"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/config"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource"
)
var _ logsource.LogSource = new(JournaldLogSource)
type JournaldLogSource struct {
cmd *exec.Cmd
stdout io.ReadCloser
stdoutscanner *bufio.Scanner
}
// ReadLine implements logsource.LogSource.
func (j *JournaldLogSource) ReadLine() (string, error) {
if j.stdoutscanner.Scan() {
return j.stdoutscanner.Text(), nil
} else {
if j.stdoutscanner.Err() == nil {
return "", fmt.Errorf("%w: %s", logsource.ErrDead, io.EOF)
}
return "", j.stdoutscanner.Err()
}
}
func (j *JournaldLogSource) Close() error {
_ = j.cmd.Process.Signal(syscall.SIGTERM)
return j.cmd.Wait()
}
func New(cfg config.ServiceConfig, logdir string) (*JournaldLogSource, error) {
args := make([]string, 0)
if cfg.Path != nil {
args = append(args, "-u", *cfg.Path)
}
args = append(args, "-f", "-n", "0", "-o", "short", "--no-pager", "--directory", logdir)
cmd := exec.Command("journalctl", args...) //nolint:gosec
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
err = cmd.Start()
if err != nil {
return nil, err
}
stdoutscanner := bufio.NewScanner(stdout)
return &JournaldLogSource{cmd, stdout, stdoutscanner}, nil
}
agent/main.go
+251 -25
View File
@@ -2,15 +2,27 @@ package main
import (
"context"
"fmt"
"log"
"os"
"strings"
"time"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/buffer"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/client"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/commander"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/config"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logger"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource/file"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource/journald"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/mtls"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/registration"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto/proto"
"github.com/samber/lo"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
func main() {
@@ -24,57 +36,271 @@ func main() {
log.Fatalf("Failed to load config: %v", err)
}
log.Printf("Agent label: %s", cfg.Label)
lgr := logger.New(os.Getenv("IS_DEBUG") == "1")
lgr.Debug("Config parsed", "cfg", cfg)
if cfg.RegistrationToken == "" {
log.Fatal("No registration token provided")
lgr.Error("No registration token provided")
os.Exit(1)
}
// Generate key and CSR
key, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label)
if err != nil {
log.Fatalf("Failed to generate key and CSR: %v", err)
lgr.Error("Failed to generate key and CSR", "err", err)
os.Exit(1)
}
log.Println("Generated ECDSA key pair and CSR")
lgr.Info("Generated ECDSA key pair and CSR")
// Register with backend
certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM)
if err != nil {
log.Fatalf("Failed to register: %v", err)
lgr.Error("Failed to register", "err", err)
os.Exit(1)
}
log.Println("Successfully registered, received certificates")
lgr.Info("Successfully registered, received certificates")
// Save certificates
if err := registration.SaveCerts(cfg.CertDir, certs, key); err != nil {
log.Fatalf("Failed to save certificates: %v", err)
lgr.Error("Failed to save certificates", "err", err)
os.Exit(1)
}
log.Printf("Certificates saved to %s", cfg.CertDir)
lgr.Info("Certificates saved", "cert_dir", cfg.CertDir)
log.Println("Agent registration complete")
err = func() error {
creds, err := mtls.LoadMTLSCredentialsFromFiles(
cfg.CertDir+"/ca.crt",
cfg.CertDir+"/client.crt",
cfg.CertDir+"/client.key",
)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
creds, err := mtls.LoadMTLSCredentialsFromFiles(
cfg.CertDir+"/ca.crt",
cfg.CertDir+"/client.crt",
cfg.CertDir+"/client.key",
)
if err != nil {
lgr.Error("Failed to load TLS credentials", "err", err)
os.Exit(1)
}
// Initialize log buffer for offline storage
dbPath := getEnvOrDefault("BUFFER_DB", "/var/lib/hellreign-agent/agent_buffer.db")
logBuf, err := buffer.NewLogBuffer(dbPath)
if err != nil {
lgr.Error("Failed to create log buffer", "err", err)
os.Exit(1)
}
defer func() { _ = logBuf.Close() }()
lgr.Info("Log buffer initialized", "path", dbPath)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg := &errgroup.Group{}
// Start command executor
wg.Go(func() error {
cmdexe := new(commander.CommandExecutor)
ccli := client.New(cmdexe, cfg.Label, cfg.Label)
// Use grpc_url for gRPC connection, strip scheme if present
grpcAddr := cfg.GRPCURL
if grpcAddr == "" {
// Fallback: derive from backend_url by stripping scheme
grpcAddr = cfg.BackendURL
}
grpcAddr = strings.TrimPrefix(grpcAddr, "http://")
grpcAddr = strings.TrimPrefix(grpcAddr, "https://")
return ccli.HandleCommands(ctx, grpcAddr, creds)
}()
if err != nil {
log.Fatalf("Failed to generate key and CSR: %v", err)
})
// Start log collectors
if len(cfg.Services) > 0 {
grpcAddr := cfg.GRPCURL
if grpcAddr == "" {
grpcAddr = cfg.BackendURL
}
grpcAddr = strings.TrimPrefix(grpcAddr, "http://")
grpcAddr = strings.TrimPrefix(grpcAddr, "https://")
conn, err := grpc.NewClient(grpcAddr, grpc.WithTransportCredentials(creds))
if err != nil {
lgr.Error("Failed to connect to gRPC", "err", err)
os.Exit(1)
}
defer func() { _ = conn.Close() }()
ccli := proto.NewCollectorClient(conn)
for _, svc := range cfg.Services {
svc := svc
var src logsource.LogSource
switch svc.Type {
case "journald":
src, err = journald.New(svc, os.Getenv("JOURNALD_LOGDIR"))
if err != nil {
lgr.Error("Failed to create journald source", "service", svc.Name, "err", err)
os.Exit(1)
}
case "file":
if svc.Path == nil {
lgr.Error("Path is required for file log source", "service", svc.Name)
os.Exit(1)
}
src, err = file.New(*svc.Path)
if err != nil {
lgr.Error("Failed to create file source", "service", svc.Name, "err", err)
os.Exit(1)
}
default:
lgr.Error("Unknown log source type", "type", svc.Type, "service", svc.Name)
os.Exit(1)
}
wg.Go(func() error {
lgr.Info("Starting log stream", "service", svc.Name)
// First, flush any buffered logs from offline period
if err := flushBufferedLogs(ctx, ccli, logBuf, svc.Name, cfg.Label, cfg.RegistrationToken, lgr); err != nil {
lgr.Error("Failed to flush buffered logs", "service", svc.Name, "err", err)
}
scli, err := ccli.Stream(
metadata.NewOutgoingContext(ctx, metadata.MD{
"whoami": []string{cfg.Label},
"service": []string{svc.Name},
"token": []string{cfg.RegistrationToken},
"services": lo.Map(cfg.Services, func(item config.ServiceConfig, _ int) string {
return item.Name
}),
}),
)
if err != nil {
return fmt.Errorf("failed to create stream: %w", err)
}
for {
line, err := src.ReadLine()
if err != nil {
lgr.Error("ReadLine error", "service", svc.Name, "err", err)
return err
}
if err := scli.Send(&proto.CollectorRequest{
Message: line,
}); err != nil {
// Connection failed, buffer the log
lgr.Warn("Send failed, buffering log", "service", svc.Name, "err", err)
if storeErr := logBuf.Store(svc.Name, line); storeErr != nil {
lgr.Error("Failed to buffer log", "service", svc.Name, "err", storeErr)
}
// Try to reconnect
if reconnectErr := reconnectStream(ctx, &scli, ccli, svc.Name, cfg.Label, cfg.RegistrationToken, logBuf, lgr); reconnectErr != nil {
return reconnectErr
}
continue
}
}
})
}
}
if err := wg.Wait(); err != nil {
lgr.Error("Agent dead", "err", err)
os.Exit(1)
}
}
func getEnvOrDefault(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
// flushBufferedLogs sends any buffered logs to the server
func flushBufferedLogs(
ctx context.Context,
ccli proto.CollectorClient,
logBuf *buffer.LogBuffer,
service, agentName, token string,
lgr *logger.Logger,
) error {
count, err := logBuf.Count()
if err != nil {
return err
}
if count == 0 {
return nil
}
lgr.Info("Flushing buffered logs", "service", service, "count", count)
scli, err := ccli.Stream(
metadata.NewOutgoingContext(ctx, metadata.MD{
"whoami": []string{agentName},
"service": []string{service},
"token": []string{token},
}),
)
if err != nil {
return fmt.Errorf("failed to create stream for flush: %w", err)
}
const batchSize = 100
var deletedIDs []int64
for {
logs, err := logBuf.GetPending(batchSize)
if err != nil {
return err
}
if len(logs) == 0 {
break
}
for _, logEntry := range logs {
if err := scli.Send(&proto.CollectorRequest{Message: logEntry.Message}); err != nil {
lgr.Error("Failed to send buffered log", "service", service, "err", err)
return err
}
deletedIDs = append(deletedIDs, logEntry.ID)
}
// Delete successfully sent logs
if err := logBuf.DeleteBatch(deletedIDs); err != nil {
lgr.Error("Failed to delete sent logs from buffer", "service", service, "err", err)
}
deletedIDs = deletedIDs[:0]
}
_, err = scli.CloseAndRecv()
lgr.Info("Buffer flush complete", "service", service)
return err
}
// reconnectStream attempts to recreate a gRPC stream connection
func reconnectStream(
ctx context.Context,
scli *grpc.ClientStreamingClient[proto.CollectorRequest, proto.CollectorResponse],
ccli proto.CollectorClient,
service, agentName, token string,
buf *buffer.LogBuffer,
lgr *logger.Logger,
) error {
lgr.Info("Attempting to reconnect stream...", "service", service)
// Try up to 5 times with exponential backoff
for i := 0; i < 5; i++ {
time.Sleep(time.Duration(i+1) * time.Second)
newCli, err := ccli.Stream(
metadata.NewOutgoingContext(ctx, metadata.MD{
"whoami": []string{agentName},
"service": []string{service},
"token": []string{token},
}),
)
if err != nil {
lgr.Warn("Reconnect attempt failed", "service", service, "attempt", i+1, "err", err)
continue
}
*scli = newCli
lgr.Info("Stream reconnected successfully", "service", service)
return flushBufferedLogs(ctx, ccli, buf, service, agentName, token, lgr)
}
return fmt.Errorf("failed to reconnect after 5 attempts for service %s", service)
}
backend/cmd/main.go
+38 -28
View File
@@ -10,6 +10,7 @@ import (
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/docs"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/config"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/collector"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/commander"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/handlers"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository"
@@ -57,9 +58,34 @@ func main() {
log.Printf("Warning: failed to initialize jobs table: %v", err)
}
// Initialize ClickHouse and log repository
var logRepo *repository.LogRepository
if cfg.Database.Clickhouse_host != "" {
chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{
Host: cfg.Database.Clickhouse_host,
User: cfg.Database.Clickhouse_user,
Password: cfg.Database.Clickhouse_password,
Database: cfg.Database.Clickhouse_database,
})
if err != nil {
log.Printf("Warning: ClickHouse connection failed: %v", err)
} else {
logRepo = repository.NewLogRepository(chConn)
if err := logRepo.Init(context.Background()); err != nil {
log.Printf("Warning: Failed to initialize logs table: %v", err)
} else {
log.Println("ClickHouse connected successfully")
}
defer chConn.Close()
}
}
// Initialize Collector gRPC service
coll := collector.New(logRepo)
cmdr := commander.New(jobRepo)
agents := handlers.NewAgentsGroup(h, cmdr)
agents := handlers.NewAgentsGroup(h, coll)
auth := handlers.AuthGroup{Handlers: h}
agentReg := handlers.NewAgentRegistrationGroup(h)
agentDeploy := handlers.NewAgentDeployGroup(h)
@@ -117,13 +143,13 @@ func main() {
authTokenGroup.GET("/tokens", handlers.RequireAdmin(), auth.ListTokens)
authTokenGroup.DELETE("/token", auth.DeleteMyToken)
authTokenGroup.DELETE("/tokens/:login", handlers.RequireAdmin(), auth.DeleteToken)
// User management (admin only) - Full CRUD
authTokenGroup.GET("/users/:login", handlers.RequireAdmin(), auth.GetUser)
authTokenGroup.PUT("/users/:login", handlers.RequireAdmin(), auth.UpdateUser)
authTokenGroup.PUT("/users/:login/permissions", handlers.RequireAdmin(), auth.UpdateUserPermissions)
authTokenGroup.PUT("/users/:login/password", handlers.RequireAdmin(), auth.ResetUserPassword)
// User activation management (admin only)
authTokenGroup.POST("/users/:login/activate", handlers.RequireAdmin(), auth.ActivateUser)
authTokenGroup.POST("/users/:login/deactivate", handlers.RequireAdmin(), auth.DeactivateUser)
@@ -157,31 +183,14 @@ func main() {
mockLogHandlers := handlers.NewLogHandlers(nil)
logsGroup.GET("/mock", mockLogHandlers.GetMockLogs)
if cfg.Database.Clickhouse_host != "" {
chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{
Host: cfg.Database.Clickhouse_host,
User: cfg.Database.Clickhouse_user,
Password: cfg.Database.Clickhouse_password,
Database: cfg.Database.Clickhouse_database,
})
if err != nil {
log.Printf("Warning: ClickHouse connection failed: %v", err)
} else {
defer chConn.Close()
logRepo := repository.NewLogRepository(chConn)
if err := logRepo.Init(context.Background()); err != nil {
log.Printf("Warning: Failed to initialize logs table: %v", err)
}
logHandlers := handlers.NewLogHandlers(logRepo)
logsGroup.POST("", logHandlers.Insert)
logsGroup.POST("/batch", logHandlers.InsertBatch)
logsGroup.GET("", logHandlers.Search)
logsGroup.GET("/services", logHandlers.GetServices)
logsGroup.GET("/agents", logHandlers.GetAgents)
logsGroup.GET("/levels", logHandlers.GetLevels)
}
if logRepo != nil {
logHandlers := handlers.NewLogHandlers(logRepo)
logsGroup.POST("", logHandlers.Insert)
logsGroup.POST("/batch", logHandlers.InsertBatch)
logsGroup.GET("", logHandlers.Search)
logsGroup.GET("/services", logHandlers.GetServices)
logsGroup.GET("/agents", logHandlers.GetAgents)
logsGroup.GET("/levels", logHandlers.GetLevels)
}
}
}
@@ -224,6 +233,7 @@ func main() {
grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
proto.RegisterCommanderServer(grpcServer, cmdr)
proto.RegisterCollectorServer(grpcServer, coll)
lis, err := net.Listen("tcp", ":"+grpcPort)
if err != nil {
backend/docs/docs.go
+127 -122
View File
@@ -31,7 +31,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.AgentInfo"
"$ref": "#/definitions/internal_handlers.AgentInfo"
}
}
}
@@ -63,7 +63,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.DeployAgentsRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest"
}
}
],
@@ -71,7 +71,7 @@ const docTemplate = `{
"200": {
"description": "Deployment results with tokens for each server",
"schema": {
"$ref": "#/definitions/repository.DeployResponse"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse"
}
},
"400": {
@@ -114,7 +114,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.RegisterRequest"
"$ref": "#/definitions/internal_handlers.RegisterRequest"
}
}
],
@@ -122,7 +122,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/handlers.RegisterResponse"
"$ref": "#/definitions/internal_handlers.RegisterResponse"
}
}
}
@@ -152,7 +152,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.RegistrationRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest"
}
}
],
@@ -186,7 +186,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.LoginRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest"
}
}
],
@@ -194,7 +194,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.LoginResponse"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse"
}
},
"400": {
@@ -244,7 +244,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenCreate"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate"
}
}
],
@@ -340,7 +340,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
}
},
@@ -426,7 +426,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
},
"400": {
@@ -481,7 +481,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenUpdate"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate"
}
}
],
@@ -661,7 +661,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenPasswordReset"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset"
}
}
],
@@ -729,7 +729,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenUpdatePermissions"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions"
}
}
],
@@ -789,7 +789,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
}
},
@@ -819,7 +819,7 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
},
"401": {
@@ -896,7 +896,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/storage.LogEntry"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
}
}
}
@@ -921,7 +921,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.InsertLogRequest"
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
],
@@ -981,7 +981,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.InsertLogsRequest"
"$ref": "#/definitions/internal_handlers.InsertLogsRequest"
}
}
],
@@ -1023,6 +1023,11 @@ const docTemplate = `{
},
"/logs/mock": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns 100 mock log entries for frontend development (no ClickHouse required)",
"produces": [
"application/json"
@@ -1071,7 +1076,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/storage.LogEntry"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
}
}
}
@@ -1103,90 +1108,7 @@ const docTemplate = `{
}
},
"definitions": {
"handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
}
},
"handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
},
"repository.AgentDeployConfig": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": {
"description": "Configuration for deploying HellreigN agent to a single server",
"type": "object",
"required": [
@@ -1204,7 +1126,7 @@ const docTemplate = `{
"authMethod": {
"allOf": [
{
"$ref": "#/definitions/repository.AuthMethod"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod"
}
],
"example": "key"
@@ -1212,7 +1134,7 @@ const docTemplate = `{
"deployType": {
"allOf": [
{
"$ref": "#/definitions/repository.DeployType"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType"
}
],
"example": "docker"
@@ -1239,7 +1161,7 @@ const docTemplate = `{
}
}
},
"repository.AuthMethod": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": {
"description": "SSH authentication method: key or password",
"type": "string",
"enum": [
@@ -1251,7 +1173,7 @@ const docTemplate = `{
"AuthMethodPassword"
]
},
"repository.DeployAgentsRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": {
"description": "Request to deploy HellreigN agents to multiple servers",
"type": "object",
"required": [
@@ -1262,12 +1184,12 @@ const docTemplate = `{
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/definitions/repository.AgentDeployConfig"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig"
}
}
}
},
"repository.DeployResponse": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": {
"description": "Response containing deployment results and registration tokens",
"type": "object",
"properties": {
@@ -1278,12 +1200,12 @@ const docTemplate = `{
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.DeployResult"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult"
}
}
}
},
"repository.DeployResult": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": {
"description": "Result of deploying to a single server",
"type": "object",
"properties": {
@@ -1309,7 +1231,7 @@ const docTemplate = `{
}
}
},
"repository.DeployType": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": {
"description": "Type of deployment: docker or binary",
"type": "string",
"enum": [
@@ -1321,7 +1243,7 @@ const docTemplate = `{
"DeployTypeBinary"
]
},
"repository.LoginRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": {
"type": "object",
"required": [
"login",
@@ -1336,7 +1258,7 @@ const docTemplate = `{
}
}
},
"repository.LoginResponse": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": {
"type": "object",
"properties": {
"is_active": {
@@ -1365,7 +1287,7 @@ const docTemplate = `{
}
}
},
"repository.RegistrationRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": {
"type": "object",
"required": [
"label"
@@ -1376,7 +1298,7 @@ const docTemplate = `{
}
}
},
"repository.TokenCreate": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": {
"type": "object",
"required": [
"last_name",
@@ -1411,7 +1333,7 @@ const docTemplate = `{
}
}
},
"repository.TokenPasswordReset": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": {
"type": "object",
"required": [
"new_password"
@@ -1422,7 +1344,7 @@ const docTemplate = `{
}
}
},
"repository.TokenUpdate": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": {
"type": "object",
"properties": {
"last_name": {
@@ -1433,7 +1355,7 @@ const docTemplate = `{
}
}
},
"repository.TokenUpdatePermissions": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": {
"type": "object",
"properties": {
"is_active": {
@@ -1450,7 +1372,7 @@ const docTemplate = `{
}
}
},
"repository.Tokens": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": {
"type": "object",
"properties": {
"id": {
@@ -1482,7 +1404,7 @@ const docTemplate = `{
}
}
},
"storage.LogEntry": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": {
"type": "object",
"properties": {
"agent": {
@@ -1501,6 +1423,89 @@ const docTemplate = `{
"type": "string"
}
}
},
"internal_handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"internal_handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"internal_handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
}
},
"internal_handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"internal_handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
}
},
"securityDefinitions": {
backend/docs/swagger.json
+127 -122
View File
@@ -20,7 +20,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.AgentInfo"
"$ref": "#/definitions/internal_handlers.AgentInfo"
}
}
}
@@ -52,7 +52,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.DeployAgentsRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest"
}
}
],
@@ -60,7 +60,7 @@
"200": {
"description": "Deployment results with tokens for each server",
"schema": {
"$ref": "#/definitions/repository.DeployResponse"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse"
}
},
"400": {
@@ -103,7 +103,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.RegisterRequest"
"$ref": "#/definitions/internal_handlers.RegisterRequest"
}
}
],
@@ -111,7 +111,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/handlers.RegisterResponse"
"$ref": "#/definitions/internal_handlers.RegisterResponse"
}
}
}
@@ -141,7 +141,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.RegistrationRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest"
}
}
],
@@ -175,7 +175,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.LoginRequest"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest"
}
}
],
@@ -183,7 +183,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.LoginResponse"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse"
}
},
"400": {
@@ -233,7 +233,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenCreate"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate"
}
}
],
@@ -329,7 +329,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
}
},
@@ -415,7 +415,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
},
"400": {
@@ -470,7 +470,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenUpdate"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate"
}
}
],
@@ -650,7 +650,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenPasswordReset"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset"
}
}
],
@@ -718,7 +718,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/repository.TokenUpdatePermissions"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions"
}
}
],
@@ -778,7 +778,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
}
},
@@ -808,7 +808,7 @@
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/repository.Tokens"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens"
}
},
"401": {
@@ -885,7 +885,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/storage.LogEntry"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
}
}
}
@@ -910,7 +910,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.InsertLogRequest"
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
],
@@ -970,7 +970,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handlers.InsertLogsRequest"
"$ref": "#/definitions/internal_handlers.InsertLogsRequest"
}
}
],
@@ -1012,6 +1012,11 @@
},
"/logs/mock": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Returns 100 mock log entries for frontend development (no ClickHouse required)",
"produces": [
"application/json"
@@ -1060,7 +1065,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/storage.LogEntry"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry"
}
}
}
@@ -1092,90 +1097,7 @@
}
},
"definitions": {
"handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/handlers.InsertLogRequest"
}
}
}
},
"handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
},
"repository.AgentDeployConfig": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": {
"description": "Configuration for deploying HellreigN agent to a single server",
"type": "object",
"required": [
@@ -1193,7 +1115,7 @@
"authMethod": {
"allOf": [
{
"$ref": "#/definitions/repository.AuthMethod"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod"
}
],
"example": "key"
@@ -1201,7 +1123,7 @@
"deployType": {
"allOf": [
{
"$ref": "#/definitions/repository.DeployType"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType"
}
],
"example": "docker"
@@ -1228,7 +1150,7 @@
}
}
},
"repository.AuthMethod": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": {
"description": "SSH authentication method: key or password",
"type": "string",
"enum": [
@@ -1240,7 +1162,7 @@
"AuthMethodPassword"
]
},
"repository.DeployAgentsRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": {
"description": "Request to deploy HellreigN agents to multiple servers",
"type": "object",
"required": [
@@ -1251,12 +1173,12 @@
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/definitions/repository.AgentDeployConfig"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig"
}
}
}
},
"repository.DeployResponse": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": {
"description": "Response containing deployment results and registration tokens",
"type": "object",
"properties": {
@@ -1267,12 +1189,12 @@
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/repository.DeployResult"
"$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult"
}
}
}
},
"repository.DeployResult": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": {
"description": "Result of deploying to a single server",
"type": "object",
"properties": {
@@ -1298,7 +1220,7 @@
}
}
},
"repository.DeployType": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": {
"description": "Type of deployment: docker or binary",
"type": "string",
"enum": [
@@ -1310,7 +1232,7 @@
"DeployTypeBinary"
]
},
"repository.LoginRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": {
"type": "object",
"required": [
"login",
@@ -1325,7 +1247,7 @@
}
}
},
"repository.LoginResponse": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": {
"type": "object",
"properties": {
"is_active": {
@@ -1354,7 +1276,7 @@
}
}
},
"repository.RegistrationRequest": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": {
"type": "object",
"required": [
"label"
@@ -1365,7 +1287,7 @@
}
}
},
"repository.TokenCreate": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": {
"type": "object",
"required": [
"last_name",
@@ -1400,7 +1322,7 @@
}
}
},
"repository.TokenPasswordReset": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": {
"type": "object",
"required": [
"new_password"
@@ -1411,7 +1333,7 @@
}
}
},
"repository.TokenUpdate": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": {
"type": "object",
"properties": {
"last_name": {
@@ -1422,7 +1344,7 @@
}
}
},
"repository.TokenUpdatePermissions": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": {
"type": "object",
"properties": {
"is_active": {
@@ -1439,7 +1361,7 @@
}
}
},
"repository.Tokens": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": {
"type": "object",
"properties": {
"id": {
@@ -1471,7 +1393,7 @@
}
}
},
"storage.LogEntry": {
"gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": {
"type": "object",
"properties": {
"agent": {
@@ -1490,6 +1412,89 @@
"type": "string"
}
}
},
"internal_handlers.AgentInfo": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"services": {
"type": "array",
"items": {
"type": "string"
}
},
"token": {
"type": "string"
}
}
},
"internal_handlers.InsertLogRequest": {
"type": "object",
"required": [
"agent",
"level",
"message",
"service"
],
"properties": {
"agent": {
"type": "string"
},
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"service": {
"type": "string"
},
"timestamp": {
"type": "string"
}
}
},
"internal_handlers.InsertLogsRequest": {
"type": "object",
"required": [
"logs"
],
"properties": {
"logs": {
"type": "array",
"items": {
"$ref": "#/definitions/internal_handlers.InsertLogRequest"
}
}
}
},
"internal_handlers.RegisterRequest": {
"type": "object",
"required": [
"csr",
"token"
],
"properties": {
"csr": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"internal_handlers.RegisterResponse": {
"type": "object",
"properties": {
"ca_cert": {
"type": "string"
},
"client_cert": {
"type": "string"
}
}
}
},
"securityDefinitions": {
backend/docs/swagger.yaml
+96 -94
View File
@@ -1,60 +1,5 @@
definitions:
handlers.AgentInfo:
properties:
label:
type: string
services:
items:
type: string
type: array
token:
type: string
type: object
handlers.InsertLogRequest:
properties:
agent:
type: string
level:
type: string
message:
type: string
service:
type: string
timestamp:
type: string
required:
- agent
- level
- message
- service
type: object
handlers.InsertLogsRequest:
properties:
logs:
items:
$ref: '#/definitions/handlers.InsertLogRequest'
type: array
required:
- logs
type: object
handlers.RegisterRequest:
properties:
csr:
type: string
token:
type: string
required:
- csr
- token
type: object
handlers.RegisterResponse:
properties:
ca_cert:
type: string
client_cert:
type: string
type: object
repository.AgentDeployConfig:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig:
description: Configuration for deploying HellreigN agent to a single server
properties:
agentLabel:
@@ -62,11 +7,11 @@ definitions:
type: string
authMethod:
allOf:
- $ref: '#/definitions/repository.AuthMethod'
- $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod'
example: key
deployType:
allOf:
- $ref: '#/definitions/repository.DeployType'
- $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType'
example: docker
ip:
example: 192.168.1.100
@@ -90,7 +35,7 @@ definitions:
- ip
- user
type: object
repository.AuthMethod:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod:
description: 'SSH authentication method: key or password'
enum:
- key
@@ -99,18 +44,18 @@ definitions:
x-enum-varnames:
- AuthMethodKey
- AuthMethodPassword
repository.DeployAgentsRequest:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest:
description: Request to deploy HellreigN agents to multiple servers
properties:
servers:
items:
$ref: '#/definitions/repository.AgentDeployConfig'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig'
minItems: 1
type: array
required:
- servers
type: object
repository.DeployResponse:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse:
description: Response containing deployment results and registration tokens
properties:
message:
@@ -118,10 +63,10 @@ definitions:
type: string
results:
items:
$ref: '#/definitions/repository.DeployResult'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult'
type: array
type: object
repository.DeployResult:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult:
description: Result of deploying to a single server
properties:
agent_label:
@@ -140,7 +85,7 @@ definitions:
example: abc123...
type: string
type: object
repository.DeployType:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType:
description: 'Type of deployment: docker or binary'
enum:
- docker
@@ -149,7 +94,7 @@ definitions:
x-enum-varnames:
- DeployTypeDocker
- DeployTypeBinary
repository.LoginRequest:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest:
properties:
login:
type: string
@@ -159,7 +104,7 @@ definitions:
- login
- password
type: object
repository.LoginResponse:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse:
properties:
is_active:
type: boolean
@@ -178,14 +123,14 @@ definitions:
token:
type: string
type: object
repository.RegistrationRequest:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest:
properties:
label:
type: string
required:
- label
type: object
repository.TokenCreate:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate:
properties:
is_active:
type: boolean
@@ -209,21 +154,21 @@ definitions:
- name
- password
type: object
repository.TokenPasswordReset:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset:
properties:
new_password:
type: string
required:
- new_password
type: object
repository.TokenUpdate:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate:
properties:
last_name:
type: string
name:
type: string
type: object
repository.TokenUpdatePermissions:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions:
properties:
is_active:
type: boolean
@@ -234,7 +179,7 @@ definitions:
permission_view:
type: boolean
type: object
repository.Tokens:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens:
properties:
id:
type: integer
@@ -255,7 +200,7 @@ definitions:
token:
type: string
type: object
storage.LogEntry:
gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry:
properties:
agent:
type: string
@@ -268,6 +213,61 @@ definitions:
timestamp:
type: string
type: object
internal_handlers.AgentInfo:
properties:
label:
type: string
services:
items:
type: string
type: array
token:
type: string
type: object
internal_handlers.InsertLogRequest:
properties:
agent:
type: string
level:
type: string
message:
type: string
service:
type: string
timestamp:
type: string
required:
- agent
- level
- message
- service
type: object
internal_handlers.InsertLogsRequest:
properties:
logs:
items:
$ref: '#/definitions/internal_handlers.InsertLogRequest'
type: array
required:
- logs
type: object
internal_handlers.RegisterRequest:
properties:
csr:
type: string
token:
type: string
required:
- csr
- token
type: object
internal_handlers.RegisterResponse:
properties:
ca_cert:
type: string
client_cert:
type: string
type: object
info:
contact: {}
paths:
@@ -281,7 +281,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/handlers.AgentInfo'
$ref: '#/definitions/internal_handlers.AgentInfo'
type: array
summary: Get connected agents
tags:
@@ -298,14 +298,14 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.DeployAgentsRequest'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest'
produces:
- application/json
responses:
"200":
description: Deployment results with tokens for each server
schema:
$ref: '#/definitions/repository.DeployResponse'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse'
"400":
description: Invalid request
schema:
@@ -333,14 +333,14 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/handlers.RegisterRequest'
$ref: '#/definitions/internal_handlers.RegisterRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/handlers.RegisterResponse'
$ref: '#/definitions/internal_handlers.RegisterResponse'
summary: Register agent
tags:
- agents
@@ -354,7 +354,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.RegistrationRequest'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest'
produces:
- application/json
responses:
@@ -380,12 +380,12 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.LoginRequest'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest'
responses:
"200":
description: OK
schema:
$ref: '#/definitions/repository.LoginResponse'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse'
"400":
description: Bad Request
schema:
@@ -442,7 +442,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.TokenCreate'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate'
responses:
"200":
description: OK
@@ -481,7 +481,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/repository.Tokens'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
type: array
"500":
description: Internal Server Error
@@ -538,7 +538,7 @@ paths:
"200":
description: OK
schema:
$ref: '#/definitions/repository.Tokens'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
"400":
description: Bad Request
schema:
@@ -575,7 +575,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.TokenUpdate'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate'
responses:
"200":
description: OK
@@ -694,7 +694,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.TokenPasswordReset'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset'
responses:
"200":
description: OK
@@ -739,7 +739,7 @@ paths:
name: request
required: true
schema:
$ref: '#/definitions/repository.TokenUpdatePermissions'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions'
responses:
"200":
description: OK
@@ -778,7 +778,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/repository.Tokens'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
type: array
"500":
description: Internal Server Error
@@ -798,7 +798,7 @@ paths:
"200":
description: OK
schema:
$ref: '#/definitions/repository.Tokens'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens'
"401":
description: Unauthorized
schema:
@@ -849,7 +849,7 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/storage.LogEntry'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry'
type: array
summary: Search logs
tags:
@@ -864,7 +864,7 @@ paths:
name: body
required: true
schema:
$ref: '#/definitions/handlers.InsertLogRequest'
$ref: '#/definitions/internal_handlers.InsertLogRequest'
produces:
- application/json
responses:
@@ -903,7 +903,7 @@ paths:
name: body
required: true
schema:
$ref: '#/definitions/handlers.InsertLogsRequest'
$ref: '#/definitions/internal_handlers.InsertLogsRequest'
produces:
- application/json
responses:
@@ -965,8 +965,10 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/storage.LogEntry'
$ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry'
type: array
security:
- Bearer: []
summary: Get mock logs
tags:
- logs
backend/internal/grpcsrv/collector/collector.go
+180
View File
@@ -0,0 +1,180 @@
package collector
import (
"fmt"
"io"
"log"
"sync"
"time"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto/proto"
"google.golang.org/grpc/metadata"
)
type Collector struct {
proto.UnimplementedCollectorServer
logRepo *repository.LogRepository
agents map[string]*Agent
mu sync.RWMutex
batchSize int
flushInterval time.Duration
}
type Agent struct {
ID string
Label string
Services []string
ConnectedAt time.Time
}
func New(logRepo *repository.LogRepository) *Collector {
return &Collector{
logRepo: logRepo,
agents: make(map[string]*Agent),
batchSize: 100,
flushInterval: 2 * time.Second,
}
}
func (c *Collector) Stream(stream proto.Collector_StreamServer) error {
md, ok := metadata.FromIncomingContext(stream.Context())
if !ok {
return fmt.Errorf("no metadata in context")
}
whoamiVals := md["whoami"]
if len(whoamiVals) == 0 {
return fmt.Errorf("whoami metadata missing")
}
agentName := whoamiVals[0]
serviceVals := md["service"]
if len(serviceVals) == 0 {
return fmt.Errorf("service metadata missing")
}
service := serviceVals[0]
servicesVals := md["services"]
var services []string
if len(servicesVals) > 0 {
services = servicesVals
}
// Register agent
c.mu.Lock()
c.agents[agentName] = &Agent{
ID: agentName,
Label: agentName,
Services: services,
ConnectedAt: time.Now(),
}
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.agents, agentName)
c.mu.Unlock()
}()
log.Printf("Agent %s connected, streaming logs for service: %s", agentName, service)
// If no ClickHouse, just consume the stream without storing
if c.logRepo == nil {
log.Printf("Warning: logRepo is nil, consuming logs without storing for agent %s", agentName)
for {
_, err := stream.Recv()
if err == io.EOF {
return nil
}
if err != nil {
return fmt.Errorf("failed to receive: %w", err)
}
}
}
// Channels for communication with recv goroutine
recvCh := make(chan *proto.CollectorRequest, 1)
errCh := make(chan error, 1)
// Goroutine that blocks on Recv
go func() {
for {
req, err := stream.Recv()
if err != nil {
errCh <- err
return
}
recvCh <- req
}
}()
// Buffer for batch inserts
var batch []storage.LogEntry
ticker := time.NewTicker(c.flushInterval)
defer ticker.Stop()
flush := func() error {
if len(batch) == 0 {
return nil
}
if err := c.logRepo.InsertBatch(stream.Context(), batch); err != nil {
log.Printf("Failed to insert batch for agent %s, service %s: %v", agentName, service, err)
return err
}
log.Printf("Flushed %d logs for agent %s, service %s", len(batch), agentName, service)
batch = batch[:0]
return nil
}
for {
select {
case <-stream.Context().Done():
// Context cancelled, flush remaining
_ = flush()
return stream.Context().Err()
case <-ticker.C:
if err := flush(); err != nil {
return err
}
case req := <-recvCh:
batch = append(batch, storage.LogEntry{
Timestamp: time.Now(),
Level: "info",
Service: service,
Agent: agentName,
Message: req.Message,
})
if len(batch) >= c.batchSize {
if err := flush(); err != nil {
return err
}
}
case err := <-errCh:
if err == io.EOF {
// Client closed stream
return flush()
}
return fmt.Errorf("failed to receive: %w", err)
}
}
}
func (c *Collector) GetAgent(name string) (*Agent, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
a, ok := c.agents[name]
return a, ok
}
func (c *Collector) Agents() []*Agent {
c.mu.RLock()
defer c.mu.RUnlock()
result := make([]*Agent, 0, len(c.agents))
for _, a := range c.agents {
result = append(result, a)
}
return result
}
backend/internal/handlers/agents.go
+16 -13
View File
@@ -1,41 +1,44 @@
package handlers
import (
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/commander"
"gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/collector"
"github.com/gin-gonic/gin"
"net/http"
)
type AgentsGroup struct {
*Handlers
cmder *commander.Commander
collector *collector.Collector
}
func NewAgentsGroup(h *Handlers, cmder *commander.Commander) AgentsGroup {
return AgentsGroup{Handlers: h, cmder: cmder}
func NewAgentsGroup(h *Handlers, coll *collector.Collector) AgentsGroup {
return AgentsGroup{Handlers: h, collector: coll}
}
type AgentInfo struct {
Token string `json:"token"`
Label string `json:"label"`
Services []string `json:"services"`
Token string `json:"token"`
Label string `json:"label"`
Services []string `json:"services"`
ConnectedAt string `json:"connected_at"`
}
// @Summary Get connected agents
// @Description Returns a list of all agents currently connected via gRPC streaming
// @Description Returns a list of all agents currently connected via Collector (log streaming)
// @Tags agents
// @Produce json
// @Success 200 {array} AgentInfo
// @Router /agents [get]
func (ag *AgentsGroup) List(c *gin.Context) {
agents := make([]AgentInfo, 0)
// iterate over the commander's agents map
for _, agent := range ag.cmder.Agents() {
for _, agent := range ag.collector.Agents() {
agents = append(agents, AgentInfo{
Token: agent.Token,
Label: agent.Label,
Services: agent.Services,
Token: agent.ID,
Label: agent.Label,
Services: agent.Services,
ConnectedAt: agent.ConnectedAt.Format("2006-01-02 15:04:05"),
})
}
c.JSON(http.StatusOK, agents)
}
backend/internal/handlers/logs_mock.go
+1
View File
@@ -21,6 +21,7 @@ import (
// @Param limit query int false "Limit results" default(100)
// @Param offset query int false "Offset results" default(0)
// @Success 200 {array} storage.LogEntry
// @Security Bearer
// @Router /logs/mock [get]
func (lh *LogHandlers) GetMockLogs(c *gin.Context) {
levelFilter := c.Query("level")