diff --git a/agent/go.mod b/agent/go.mod index b7929f9..64b0c24 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -4,12 +4,21 @@ go 1.26.1 require ( gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto v0.0.0-20260403214837-94be9799f47d + github.com/hpcloud/tail v1.0.0 + github.com/samber/lo v1.53.0 golang.org/x/sync v0.20.0 google.golang.org/grpc v1.80.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.34.5 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/net v0.52.0 // indirect @@ -17,6 +26,11 @@ require ( golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect ) replace gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto => ../proto diff --git a/agent/go.sum b/agent/go.sum index 0d58fe4..4a32cbf 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -1,5 +1,9 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -8,8 +12,20 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= +github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= @@ -22,14 +38,19 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= @@ -40,5 +61,33 @@ google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBN google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/agent/internal/buffer/buffer.go b/agent/internal/buffer/buffer.go new file mode 100644 index 0000000..ef78d48 --- /dev/null +++ b/agent/internal/buffer/buffer.go @@ -0,0 +1,161 @@ +package buffer + +import ( + "database/sql" + "encoding/json" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +// BufferedLog represents a log entry stored for later delivery +type BufferedLog struct { + ID int64 + Service string + Message string + CreatedAt time.Time +} + +// LogBuffer provides SQLite-backed log buffering +type LogBuffer struct { + db *sql.DB +} + +// NewLogBuffer creates a new log buffer with the given database path +func NewLogBuffer(dbPath string) (*LogBuffer, error) { + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + return nil, fmt.Errorf("failed to open database: %w", err) + } + + // Create table if not exists + _, err = db.Exec(` + CREATE TABLE IF NOT EXISTS buffered_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + service TEXT NOT NULL, + message TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + `) + if err != nil { + _ = db.Close() + return nil, fmt.Errorf("failed to create table: %w", err) + } + + // Create index for efficient ordering + _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_created_at ON buffered_logs(created_at ASC)`) + + return &LogBuffer{db: db}, nil +} + +// Close closes the database connection +func (b *LogBuffer) Close() error { + return b.db.Close() +} + +// Store stores a log entry in the buffer +func (b *LogBuffer) Store(service, message string) error { + _, err := b.db.Exec( + "INSERT INTO buffered_logs (service, message) VALUES (?, ?)", + service, message, + ) + return err +} + +// StoreBatch stores multiple log entries in a single transaction +func (b *LogBuffer) StoreBatch(entries []BufferedLog) error { + tx, err := b.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + stmt, err := tx.Prepare("INSERT INTO buffered_logs (service, message) VALUES (?, ?)") + if err != nil { + return err + } + defer stmt.Close() + + for _, entry := range entries { + if _, err := stmt.Exec(entry.Service, entry.Message); err != nil { + return err + } + } + + return tx.Commit() +} + +// GetPending retrieves pending logs in order of arrival, limited to batchSize +func (b *LogBuffer) GetPending(batchSize int) ([]BufferedLog, error) { + rows, err := b.db.Query( + "SELECT id, service, message, created_at FROM buffered_logs ORDER BY created_at ASC LIMIT ?", + batchSize, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var logs []BufferedLog + for rows.Next() { + var log BufferedLog + var createdAt string + if err := rows.Scan(&log.ID, &log.Service, &log.Message, &createdAt); err != nil { + return nil, err + } + log.CreatedAt, _ = time.Parse("2006-01-02 15:04:05", createdAt) + logs = append(logs, log) + } + + return logs, rows.Err() +} + +// Delete removes a log entry from the buffer after successful delivery +func (b *LogBuffer) Delete(id int64) error { + _, err := b.db.Exec("DELETE FROM buffered_logs WHERE id = ?", id) + return err +} + +// DeleteBatch removes multiple log entries after successful delivery +func (b *LogBuffer) DeleteBatch(ids []int64) error { + if len(ids) == 0 { + return nil + } + + tx, err := b.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + for _, id := range ids { + if _, err := tx.Exec("DELETE FROM buffered_logs WHERE id = ?", id); err != nil { + return err + } + } + + return tx.Commit() +} + +// Count returns the number of buffered logs +func (b *LogBuffer) Count() (int, error) { + var count int + err := b.db.QueryRow("SELECT COUNT(*) FROM buffered_logs").Scan(&count) + return count, err +} + +// Clear removes all buffered logs +func (b *LogBuffer) Clear() error { + _, err := b.db.Exec("DELETE FROM buffered_logs") + return err +} + +// FlushToJSON exports buffered logs to JSON format for debugging +func (b *LogBuffer) FlushToJSON() ([]byte, error) { + logs, err := b.GetPending(1000) + if err != nil { + return nil, err + } + return json.MarshalIndent(logs, "", " ") +} diff --git a/agent/internal/config/config.go b/agent/internal/config/config.go index 6381206..2628ac2 100644 --- a/agent/internal/config/config.go +++ b/agent/internal/config/config.go @@ -1,17 +1,25 @@ package config import ( + "fmt" "os" "gopkg.in/yaml.v3" ) +type ServiceConfig struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Path *string `yaml:"path"` +} + type AgentConfig struct { - BackendURL string `yaml:"backend_url"` - GRPCURL string `yaml:"grpc_url"` - RegistrationToken string `yaml:"registration_token"` - Label string `yaml:"label"` - CertDir string `yaml:"cert_dir"` + BackendURL string `yaml:"backend_url"` + GRPCURL string `yaml:"grpc_url"` + RegistrationToken string `yaml:"registration_token"` + Label string `yaml:"label"` + CertDir string `yaml:"cert_dir"` + Services []ServiceConfig `yaml:"services"` } func Load(path string) (*AgentConfig, error) { @@ -31,3 +39,19 @@ func Load(path string) (*AgentConfig, error) { return &cfg, nil } + +func LoadFromString(data string) (*AgentConfig, error) { + var cfg AgentConfig + if err := yaml.Unmarshal([]byte(data), &cfg); err != nil { + return nil, err + } + + return &cfg, nil +} + +func validateConfigPath(path string) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("config file not found: %w", err) + } + return nil +} diff --git a/agent/internal/logger/logger.go b/agent/internal/logger/logger.go new file mode 100644 index 0000000..13df9fa --- /dev/null +++ b/agent/internal/logger/logger.go @@ -0,0 +1,27 @@ +package logger + +import ( + "log/slog" + "os" +) + +type Logger struct { + *slog.Logger +} + +func New(debug bool) *Logger { + var level slog.Level + if debug { + level = slog.LevelDebug + } else { + level = slog.LevelInfo + } + + handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + }) + + return &Logger{ + Logger: slog.New(handler), + } +} diff --git a/agent/internal/logsource/file/impl.go b/agent/internal/logsource/file/impl.go new file mode 100644 index 0000000..b10d01f --- /dev/null +++ b/agent/internal/logsource/file/impl.go @@ -0,0 +1,45 @@ +package file + +import ( + "errors" + "os" + + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource" + "github.com/hpcloud/tail" +) + +var _ logsource.LogSource = new(FileLogSource) + +type FileLogSource struct { + *tail.Tail +} + +func New(filepath string) (fls *FileLogSource, err error) { + if _, err := os.Stat(filepath); os.IsNotExist(err) { + if err := os.WriteFile(filepath, []byte{}, 0600); err != nil { + return nil, err + } + } + t, err := tail.TailFile(filepath, tail.Config{ + Follow: true, + Location: &tail.SeekInfo{ + Offset: 100, + Whence: 2, + }, + }) + if err != nil { + return + } + return &FileLogSource{t}, nil +} +func (f *FileLogSource) ReadLine() (string, error) { + select { + case <-f.Dead(): + return "", errors.Join(logsource.ErrDead, f.Err()) + case line := <-f.Lines: + return line.Text, line.Err + } +} +func (f *FileLogSource) Close() error { + return f.Stop() +} diff --git a/agent/internal/logsource/interface.go b/agent/internal/logsource/interface.go new file mode 100644 index 0000000..71de31d --- /dev/null +++ b/agent/internal/logsource/interface.go @@ -0,0 +1,10 @@ +package logsource + +import "errors" + +type LogSource interface { + ReadLine() (string, error) + Close() error +} + +var ErrDead = errors.New("shouldn't continue to read that") diff --git a/agent/internal/logsource/journald/impl.go b/agent/internal/logsource/journald/impl.go new file mode 100644 index 0000000..ce162e8 --- /dev/null +++ b/agent/internal/logsource/journald/impl.go @@ -0,0 +1,55 @@ +package journald + +import ( + "bufio" + "fmt" + "io" + "os/exec" + "syscall" + + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/config" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource" +) + +var _ logsource.LogSource = new(JournaldLogSource) + +type JournaldLogSource struct { + cmd *exec.Cmd + stdout io.ReadCloser + stdoutscanner *bufio.Scanner +} + +// ReadLine implements logsource.LogSource. +func (j *JournaldLogSource) ReadLine() (string, error) { + if j.stdoutscanner.Scan() { + return j.stdoutscanner.Text(), nil + } else { + if j.stdoutscanner.Err() == nil { + return "", fmt.Errorf("%w: %s", logsource.ErrDead, io.EOF) + } + return "", j.stdoutscanner.Err() + } +} +func (j *JournaldLogSource) Close() error { + _ = j.cmd.Process.Signal(syscall.SIGTERM) + return j.cmd.Wait() +} + +func New(cfg config.ServiceConfig, logdir string) (*JournaldLogSource, error) { + args := make([]string, 0) + if cfg.Path != nil { + args = append(args, "-u", *cfg.Path) + } + args = append(args, "-f", "-n", "0", "-o", "short", "--no-pager", "--directory", logdir) + cmd := exec.Command("journalctl", args...) //nolint:gosec + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + err = cmd.Start() + if err != nil { + return nil, err + } + stdoutscanner := bufio.NewScanner(stdout) + return &JournaldLogSource{cmd, stdout, stdoutscanner}, nil +} diff --git a/agent/main.go b/agent/main.go index 9a4b665..52fa7e6 100644 --- a/agent/main.go +++ b/agent/main.go @@ -2,15 +2,27 @@ package main import ( "context" + "fmt" "log" "os" "strings" + "time" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/buffer" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/client" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/commander" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/config" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logger" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource/file" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/logsource/journald" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/mtls" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/agent/internal/registration" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto/proto" + "github.com/samber/lo" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) func main() { @@ -24,57 +36,271 @@ func main() { log.Fatalf("Failed to load config: %v", err) } - log.Printf("Agent label: %s", cfg.Label) + lgr := logger.New(os.Getenv("IS_DEBUG") == "1") + lgr.Debug("Config parsed", "cfg", cfg) if cfg.RegistrationToken == "" { - log.Fatal("No registration token provided") + lgr.Error("No registration token provided") + os.Exit(1) } // Generate key and CSR key, csrPEM, err := registration.GenerateKeyAndCSR(cfg.Label) if err != nil { - log.Fatalf("Failed to generate key and CSR: %v", err) + lgr.Error("Failed to generate key and CSR", "err", err) + os.Exit(1) } - log.Println("Generated ECDSA key pair and CSR") + lgr.Info("Generated ECDSA key pair and CSR") // Register with backend certs, err := registration.Register(cfg.BackendURL, cfg.RegistrationToken, csrPEM) if err != nil { - log.Fatalf("Failed to register: %v", err) + lgr.Error("Failed to register", "err", err) + os.Exit(1) } - log.Println("Successfully registered, received certificates") + lgr.Info("Successfully registered, received certificates") // Save certificates if err := registration.SaveCerts(cfg.CertDir, certs, key); err != nil { - log.Fatalf("Failed to save certificates: %v", err) + lgr.Error("Failed to save certificates", "err", err) + os.Exit(1) } - log.Printf("Certificates saved to %s", cfg.CertDir) + lgr.Info("Certificates saved", "cert_dir", cfg.CertDir) - log.Println("Agent registration complete") - err = func() error { - creds, err := mtls.LoadMTLSCredentialsFromFiles( - cfg.CertDir+"/ca.crt", - cfg.CertDir+"/client.crt", - cfg.CertDir+"/client.key", - ) - if err != nil { - return err - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + creds, err := mtls.LoadMTLSCredentialsFromFiles( + cfg.CertDir+"/ca.crt", + cfg.CertDir+"/client.crt", + cfg.CertDir+"/client.key", + ) + if err != nil { + lgr.Error("Failed to load TLS credentials", "err", err) + os.Exit(1) + } + + // Initialize log buffer for offline storage + dbPath := getEnvOrDefault("BUFFER_DB", "/var/lib/hellreign-agent/agent_buffer.db") + logBuf, err := buffer.NewLogBuffer(dbPath) + if err != nil { + lgr.Error("Failed to create log buffer", "err", err) + os.Exit(1) + } + defer func() { _ = logBuf.Close() }() + lgr.Info("Log buffer initialized", "path", dbPath) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + wg := &errgroup.Group{} + + // Start command executor + wg.Go(func() error { cmdexe := new(commander.CommandExecutor) ccli := client.New(cmdexe, cfg.Label, cfg.Label) - // Use grpc_url for gRPC connection, strip scheme if present grpcAddr := cfg.GRPCURL if grpcAddr == "" { - // Fallback: derive from backend_url by stripping scheme grpcAddr = cfg.BackendURL } grpcAddr = strings.TrimPrefix(grpcAddr, "http://") grpcAddr = strings.TrimPrefix(grpcAddr, "https://") return ccli.HandleCommands(ctx, grpcAddr, creds) - }() - if err != nil { - log.Fatalf("Failed to generate key and CSR: %v", err) + }) + + // Start log collectors + if len(cfg.Services) > 0 { + grpcAddr := cfg.GRPCURL + if grpcAddr == "" { + grpcAddr = cfg.BackendURL + } + grpcAddr = strings.TrimPrefix(grpcAddr, "http://") + grpcAddr = strings.TrimPrefix(grpcAddr, "https://") + + conn, err := grpc.NewClient(grpcAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + lgr.Error("Failed to connect to gRPC", "err", err) + os.Exit(1) + } + defer func() { _ = conn.Close() }() + + ccli := proto.NewCollectorClient(conn) + + for _, svc := range cfg.Services { + svc := svc + var src logsource.LogSource + switch svc.Type { + case "journald": + src, err = journald.New(svc, os.Getenv("JOURNALD_LOGDIR")) + if err != nil { + lgr.Error("Failed to create journald source", "service", svc.Name, "err", err) + os.Exit(1) + } + case "file": + if svc.Path == nil { + lgr.Error("Path is required for file log source", "service", svc.Name) + os.Exit(1) + } + src, err = file.New(*svc.Path) + if err != nil { + lgr.Error("Failed to create file source", "service", svc.Name, "err", err) + os.Exit(1) + } + default: + lgr.Error("Unknown log source type", "type", svc.Type, "service", svc.Name) + os.Exit(1) + } + + wg.Go(func() error { + lgr.Info("Starting log stream", "service", svc.Name) + + // First, flush any buffered logs from offline period + if err := flushBufferedLogs(ctx, ccli, logBuf, svc.Name, cfg.Label, cfg.RegistrationToken, lgr); err != nil { + lgr.Error("Failed to flush buffered logs", "service", svc.Name, "err", err) + } + + scli, err := ccli.Stream( + metadata.NewOutgoingContext(ctx, metadata.MD{ + "whoami": []string{cfg.Label}, + "service": []string{svc.Name}, + "token": []string{cfg.RegistrationToken}, + "services": lo.Map(cfg.Services, func(item config.ServiceConfig, _ int) string { + return item.Name + }), + }), + ) + if err != nil { + return fmt.Errorf("failed to create stream: %w", err) + } + + for { + line, err := src.ReadLine() + if err != nil { + lgr.Error("ReadLine error", "service", svc.Name, "err", err) + return err + } + + if err := scli.Send(&proto.CollectorRequest{ + Message: line, + }); err != nil { + // Connection failed, buffer the log + lgr.Warn("Send failed, buffering log", "service", svc.Name, "err", err) + if storeErr := logBuf.Store(svc.Name, line); storeErr != nil { + lgr.Error("Failed to buffer log", "service", svc.Name, "err", storeErr) + } + // Try to reconnect + if reconnectErr := reconnectStream(ctx, &scli, ccli, svc.Name, cfg.Label, cfg.RegistrationToken, logBuf, lgr); reconnectErr != nil { + return reconnectErr + } + continue + } + } + }) + } + } + + if err := wg.Wait(); err != nil { + lgr.Error("Agent dead", "err", err) + os.Exit(1) } } + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// flushBufferedLogs sends any buffered logs to the server +func flushBufferedLogs( + ctx context.Context, + ccli proto.CollectorClient, + logBuf *buffer.LogBuffer, + service, agentName, token string, + lgr *logger.Logger, +) error { + count, err := logBuf.Count() + if err != nil { + return err + } + if count == 0 { + return nil + } + + lgr.Info("Flushing buffered logs", "service", service, "count", count) + + scli, err := ccli.Stream( + metadata.NewOutgoingContext(ctx, metadata.MD{ + "whoami": []string{agentName}, + "service": []string{service}, + "token": []string{token}, + }), + ) + if err != nil { + return fmt.Errorf("failed to create stream for flush: %w", err) + } + + const batchSize = 100 + var deletedIDs []int64 + + for { + logs, err := logBuf.GetPending(batchSize) + if err != nil { + return err + } + if len(logs) == 0 { + break + } + + for _, logEntry := range logs { + if err := scli.Send(&proto.CollectorRequest{Message: logEntry.Message}); err != nil { + lgr.Error("Failed to send buffered log", "service", service, "err", err) + return err + } + deletedIDs = append(deletedIDs, logEntry.ID) + } + + // Delete successfully sent logs + if err := logBuf.DeleteBatch(deletedIDs); err != nil { + lgr.Error("Failed to delete sent logs from buffer", "service", service, "err", err) + } + deletedIDs = deletedIDs[:0] + } + + _, err = scli.CloseAndRecv() + lgr.Info("Buffer flush complete", "service", service) + return err +} + +// reconnectStream attempts to recreate a gRPC stream connection +func reconnectStream( + ctx context.Context, + scli *grpc.ClientStreamingClient[proto.CollectorRequest, proto.CollectorResponse], + ccli proto.CollectorClient, + service, agentName, token string, + buf *buffer.LogBuffer, + lgr *logger.Logger, +) error { + lgr.Info("Attempting to reconnect stream...", "service", service) + + // Try up to 5 times with exponential backoff + for i := 0; i < 5; i++ { + time.Sleep(time.Duration(i+1) * time.Second) + + newCli, err := ccli.Stream( + metadata.NewOutgoingContext(ctx, metadata.MD{ + "whoami": []string{agentName}, + "service": []string{service}, + "token": []string{token}, + }), + ) + if err != nil { + lgr.Warn("Reconnect attempt failed", "service", service, "attempt", i+1, "err", err) + continue + } + + *scli = newCli + lgr.Info("Stream reconnected successfully", "service", service) + return flushBufferedLogs(ctx, ccli, buf, service, agentName, token, lgr) + } + + return fmt.Errorf("failed to reconnect after 5 attempts for service %s", service) +} diff --git a/backend/cmd/main.go b/backend/cmd/main.go index b74779b..39908b3 100644 --- a/backend/cmd/main.go +++ b/backend/cmd/main.go @@ -10,6 +10,7 @@ import ( "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/docs" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/config" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/collector" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/commander" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/handlers" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository" @@ -57,9 +58,34 @@ func main() { log.Printf("Warning: failed to initialize jobs table: %v", err) } + // Initialize ClickHouse and log repository + var logRepo *repository.LogRepository + if cfg.Database.Clickhouse_host != "" { + chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{ + Host: cfg.Database.Clickhouse_host, + User: cfg.Database.Clickhouse_user, + Password: cfg.Database.Clickhouse_password, + Database: cfg.Database.Clickhouse_database, + }) + if err != nil { + log.Printf("Warning: ClickHouse connection failed: %v", err) + } else { + logRepo = repository.NewLogRepository(chConn) + if err := logRepo.Init(context.Background()); err != nil { + log.Printf("Warning: Failed to initialize logs table: %v", err) + } else { + log.Println("ClickHouse connected successfully") + } + defer chConn.Close() + } + } + + // Initialize Collector gRPC service + coll := collector.New(logRepo) + cmdr := commander.New(jobRepo) - agents := handlers.NewAgentsGroup(h, cmdr) + agents := handlers.NewAgentsGroup(h, coll) auth := handlers.AuthGroup{Handlers: h} agentReg := handlers.NewAgentRegistrationGroup(h) agentDeploy := handlers.NewAgentDeployGroup(h) @@ -117,13 +143,13 @@ func main() { authTokenGroup.GET("/tokens", handlers.RequireAdmin(), auth.ListTokens) authTokenGroup.DELETE("/token", auth.DeleteMyToken) authTokenGroup.DELETE("/tokens/:login", handlers.RequireAdmin(), auth.DeleteToken) - + // User management (admin only) - Full CRUD authTokenGroup.GET("/users/:login", handlers.RequireAdmin(), auth.GetUser) authTokenGroup.PUT("/users/:login", handlers.RequireAdmin(), auth.UpdateUser) authTokenGroup.PUT("/users/:login/permissions", handlers.RequireAdmin(), auth.UpdateUserPermissions) authTokenGroup.PUT("/users/:login/password", handlers.RequireAdmin(), auth.ResetUserPassword) - + // User activation management (admin only) authTokenGroup.POST("/users/:login/activate", handlers.RequireAdmin(), auth.ActivateUser) authTokenGroup.POST("/users/:login/deactivate", handlers.RequireAdmin(), auth.DeactivateUser) @@ -157,31 +183,14 @@ func main() { mockLogHandlers := handlers.NewLogHandlers(nil) logsGroup.GET("/mock", mockLogHandlers.GetMockLogs) - if cfg.Database.Clickhouse_host != "" { - chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{ - Host: cfg.Database.Clickhouse_host, - User: cfg.Database.Clickhouse_user, - Password: cfg.Database.Clickhouse_password, - Database: cfg.Database.Clickhouse_database, - }) - if err != nil { - log.Printf("Warning: ClickHouse connection failed: %v", err) - } else { - defer chConn.Close() - - logRepo := repository.NewLogRepository(chConn) - if err := logRepo.Init(context.Background()); err != nil { - log.Printf("Warning: Failed to initialize logs table: %v", err) - } - - logHandlers := handlers.NewLogHandlers(logRepo) - logsGroup.POST("", logHandlers.Insert) - logsGroup.POST("/batch", logHandlers.InsertBatch) - logsGroup.GET("", logHandlers.Search) - logsGroup.GET("/services", logHandlers.GetServices) - logsGroup.GET("/agents", logHandlers.GetAgents) - logsGroup.GET("/levels", logHandlers.GetLevels) - } + if logRepo != nil { + logHandlers := handlers.NewLogHandlers(logRepo) + logsGroup.POST("", logHandlers.Insert) + logsGroup.POST("/batch", logHandlers.InsertBatch) + logsGroup.GET("", logHandlers.Search) + logsGroup.GET("/services", logHandlers.GetServices) + logsGroup.GET("/agents", logHandlers.GetAgents) + logsGroup.GET("/levels", logHandlers.GetLevels) } } } @@ -224,6 +233,7 @@ func main() { grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) proto.RegisterCommanderServer(grpcServer, cmdr) + proto.RegisterCollectorServer(grpcServer, coll) lis, err := net.Listen("tcp", ":"+grpcPort) if err != nil { diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 8dd0dd9..c41a54f 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -31,7 +31,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/handlers.AgentInfo" + "$ref": "#/definitions/internal_handlers.AgentInfo" } } } @@ -63,7 +63,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.DeployAgentsRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest" } } ], @@ -71,7 +71,7 @@ const docTemplate = `{ "200": { "description": "Deployment results with tokens for each server", "schema": { - "$ref": "#/definitions/repository.DeployResponse" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse" } }, "400": { @@ -114,7 +114,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.RegisterRequest" + "$ref": "#/definitions/internal_handlers.RegisterRequest" } } ], @@ -122,7 +122,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/handlers.RegisterResponse" + "$ref": "#/definitions/internal_handlers.RegisterResponse" } } } @@ -152,7 +152,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.RegistrationRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest" } } ], @@ -186,7 +186,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.LoginRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest" } } ], @@ -194,7 +194,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.LoginResponse" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse" } }, "400": { @@ -244,7 +244,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenCreate" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate" } } ], @@ -340,7 +340,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } } }, @@ -426,7 +426,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } }, "400": { @@ -481,7 +481,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenUpdate" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate" } } ], @@ -661,7 +661,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenPasswordReset" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset" } } ], @@ -729,7 +729,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenUpdatePermissions" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions" } } ], @@ -789,7 +789,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } } }, @@ -819,7 +819,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } }, "401": { @@ -896,7 +896,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/storage.LogEntry" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" } } } @@ -921,7 +921,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.InsertLogRequest" + "$ref": "#/definitions/internal_handlers.InsertLogRequest" } } ], @@ -981,7 +981,7 @@ const docTemplate = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.InsertLogsRequest" + "$ref": "#/definitions/internal_handlers.InsertLogsRequest" } } ], @@ -1023,6 +1023,11 @@ const docTemplate = `{ }, "/logs/mock": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns 100 mock log entries for frontend development (no ClickHouse required)", "produces": [ "application/json" @@ -1071,7 +1076,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/storage.LogEntry" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" } } } @@ -1103,90 +1108,7 @@ const docTemplate = `{ } }, "definitions": { - "handlers.AgentInfo": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "services": { - "type": "array", - "items": { - "type": "string" - } - }, - "token": { - "type": "string" - } - } - }, - "handlers.InsertLogRequest": { - "type": "object", - "required": [ - "agent", - "level", - "message", - "service" - ], - "properties": { - "agent": { - "type": "string" - }, - "level": { - "type": "string" - }, - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } - }, - "handlers.InsertLogsRequest": { - "type": "object", - "required": [ - "logs" - ], - "properties": { - "logs": { - "type": "array", - "items": { - "$ref": "#/definitions/handlers.InsertLogRequest" - } - } - } - }, - "handlers.RegisterRequest": { - "type": "object", - "required": [ - "csr", - "token" - ], - "properties": { - "csr": { - "type": "string" - }, - "token": { - "type": "string" - } - } - }, - "handlers.RegisterResponse": { - "type": "object", - "properties": { - "ca_cert": { - "type": "string" - }, - "client_cert": { - "type": "string" - } - } - }, - "repository.AgentDeployConfig": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": { "description": "Configuration for deploying HellreigN agent to a single server", "type": "object", "required": [ @@ -1204,7 +1126,7 @@ const docTemplate = `{ "authMethod": { "allOf": [ { - "$ref": "#/definitions/repository.AuthMethod" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod" } ], "example": "key" @@ -1212,7 +1134,7 @@ const docTemplate = `{ "deployType": { "allOf": [ { - "$ref": "#/definitions/repository.DeployType" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType" } ], "example": "docker" @@ -1239,7 +1161,7 @@ const docTemplate = `{ } } }, - "repository.AuthMethod": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": { "description": "SSH authentication method: key or password", "type": "string", "enum": [ @@ -1251,7 +1173,7 @@ const docTemplate = `{ "AuthMethodPassword" ] }, - "repository.DeployAgentsRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": { "description": "Request to deploy HellreigN agents to multiple servers", "type": "object", "required": [ @@ -1262,12 +1184,12 @@ const docTemplate = `{ "type": "array", "minItems": 1, "items": { - "$ref": "#/definitions/repository.AgentDeployConfig" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig" } } } }, - "repository.DeployResponse": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": { "description": "Response containing deployment results and registration tokens", "type": "object", "properties": { @@ -1278,12 +1200,12 @@ const docTemplate = `{ "results": { "type": "array", "items": { - "$ref": "#/definitions/repository.DeployResult" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult" } } } }, - "repository.DeployResult": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": { "description": "Result of deploying to a single server", "type": "object", "properties": { @@ -1309,7 +1231,7 @@ const docTemplate = `{ } } }, - "repository.DeployType": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": { "description": "Type of deployment: docker or binary", "type": "string", "enum": [ @@ -1321,7 +1243,7 @@ const docTemplate = `{ "DeployTypeBinary" ] }, - "repository.LoginRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": { "type": "object", "required": [ "login", @@ -1336,7 +1258,7 @@ const docTemplate = `{ } } }, - "repository.LoginResponse": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": { "type": "object", "properties": { "is_active": { @@ -1365,7 +1287,7 @@ const docTemplate = `{ } } }, - "repository.RegistrationRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": { "type": "object", "required": [ "label" @@ -1376,7 +1298,7 @@ const docTemplate = `{ } } }, - "repository.TokenCreate": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": { "type": "object", "required": [ "last_name", @@ -1411,7 +1333,7 @@ const docTemplate = `{ } } }, - "repository.TokenPasswordReset": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": { "type": "object", "required": [ "new_password" @@ -1422,7 +1344,7 @@ const docTemplate = `{ } } }, - "repository.TokenUpdate": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": { "type": "object", "properties": { "last_name": { @@ -1433,7 +1355,7 @@ const docTemplate = `{ } } }, - "repository.TokenUpdatePermissions": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": { "type": "object", "properties": { "is_active": { @@ -1450,7 +1372,7 @@ const docTemplate = `{ } } }, - "repository.Tokens": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": { "type": "object", "properties": { "id": { @@ -1482,7 +1404,7 @@ const docTemplate = `{ } } }, - "storage.LogEntry": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": { "type": "object", "properties": { "agent": { @@ -1501,6 +1423,89 @@ const docTemplate = `{ "type": "string" } } + }, + "internal_handlers.AgentInfo": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "services": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "string" + } + } + }, + "internal_handlers.InsertLogRequest": { + "type": "object", + "required": [ + "agent", + "level", + "message", + "service" + ], + "properties": { + "agent": { + "type": "string" + }, + "level": { + "type": "string" + }, + "message": { + "type": "string" + }, + "service": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "internal_handlers.InsertLogsRequest": { + "type": "object", + "required": [ + "logs" + ], + "properties": { + "logs": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_handlers.InsertLogRequest" + } + } + } + }, + "internal_handlers.RegisterRequest": { + "type": "object", + "required": [ + "csr", + "token" + ], + "properties": { + "csr": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, + "internal_handlers.RegisterResponse": { + "type": "object", + "properties": { + "ca_cert": { + "type": "string" + }, + "client_cert": { + "type": "string" + } + } } }, "securityDefinitions": { diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 8f04013..9e820c9 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -20,7 +20,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/handlers.AgentInfo" + "$ref": "#/definitions/internal_handlers.AgentInfo" } } } @@ -52,7 +52,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.DeployAgentsRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest" } } ], @@ -60,7 +60,7 @@ "200": { "description": "Deployment results with tokens for each server", "schema": { - "$ref": "#/definitions/repository.DeployResponse" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse" } }, "400": { @@ -103,7 +103,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.RegisterRequest" + "$ref": "#/definitions/internal_handlers.RegisterRequest" } } ], @@ -111,7 +111,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/handlers.RegisterResponse" + "$ref": "#/definitions/internal_handlers.RegisterResponse" } } } @@ -141,7 +141,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.RegistrationRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest" } } ], @@ -175,7 +175,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.LoginRequest" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest" } } ], @@ -183,7 +183,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.LoginResponse" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse" } }, "400": { @@ -233,7 +233,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenCreate" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate" } } ], @@ -329,7 +329,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } } }, @@ -415,7 +415,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } }, "400": { @@ -470,7 +470,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenUpdate" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate" } } ], @@ -650,7 +650,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenPasswordReset" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset" } } ], @@ -718,7 +718,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/repository.TokenUpdatePermissions" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions" } } ], @@ -778,7 +778,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } } }, @@ -808,7 +808,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/repository.Tokens" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens" } }, "401": { @@ -885,7 +885,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/storage.LogEntry" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" } } } @@ -910,7 +910,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.InsertLogRequest" + "$ref": "#/definitions/internal_handlers.InsertLogRequest" } } ], @@ -970,7 +970,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/handlers.InsertLogsRequest" + "$ref": "#/definitions/internal_handlers.InsertLogsRequest" } } ], @@ -1012,6 +1012,11 @@ }, "/logs/mock": { "get": { + "security": [ + { + "Bearer": [] + } + ], "description": "Returns 100 mock log entries for frontend development (no ClickHouse required)", "produces": [ "application/json" @@ -1060,7 +1065,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/storage.LogEntry" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry" } } } @@ -1092,90 +1097,7 @@ } }, "definitions": { - "handlers.AgentInfo": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "services": { - "type": "array", - "items": { - "type": "string" - } - }, - "token": { - "type": "string" - } - } - }, - "handlers.InsertLogRequest": { - "type": "object", - "required": [ - "agent", - "level", - "message", - "service" - ], - "properties": { - "agent": { - "type": "string" - }, - "level": { - "type": "string" - }, - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } - }, - "handlers.InsertLogsRequest": { - "type": "object", - "required": [ - "logs" - ], - "properties": { - "logs": { - "type": "array", - "items": { - "$ref": "#/definitions/handlers.InsertLogRequest" - } - } - } - }, - "handlers.RegisterRequest": { - "type": "object", - "required": [ - "csr", - "token" - ], - "properties": { - "csr": { - "type": "string" - }, - "token": { - "type": "string" - } - } - }, - "handlers.RegisterResponse": { - "type": "object", - "properties": { - "ca_cert": { - "type": "string" - }, - "client_cert": { - "type": "string" - } - } - }, - "repository.AgentDeployConfig": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig": { "description": "Configuration for deploying HellreigN agent to a single server", "type": "object", "required": [ @@ -1193,7 +1115,7 @@ "authMethod": { "allOf": [ { - "$ref": "#/definitions/repository.AuthMethod" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod" } ], "example": "key" @@ -1201,7 +1123,7 @@ "deployType": { "allOf": [ { - "$ref": "#/definitions/repository.DeployType" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType" } ], "example": "docker" @@ -1228,7 +1150,7 @@ } } }, - "repository.AuthMethod": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod": { "description": "SSH authentication method: key or password", "type": "string", "enum": [ @@ -1240,7 +1162,7 @@ "AuthMethodPassword" ] }, - "repository.DeployAgentsRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest": { "description": "Request to deploy HellreigN agents to multiple servers", "type": "object", "required": [ @@ -1251,12 +1173,12 @@ "type": "array", "minItems": 1, "items": { - "$ref": "#/definitions/repository.AgentDeployConfig" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig" } } } }, - "repository.DeployResponse": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse": { "description": "Response containing deployment results and registration tokens", "type": "object", "properties": { @@ -1267,12 +1189,12 @@ "results": { "type": "array", "items": { - "$ref": "#/definitions/repository.DeployResult" + "$ref": "#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult" } } } }, - "repository.DeployResult": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult": { "description": "Result of deploying to a single server", "type": "object", "properties": { @@ -1298,7 +1220,7 @@ } } }, - "repository.DeployType": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType": { "description": "Type of deployment: docker or binary", "type": "string", "enum": [ @@ -1310,7 +1232,7 @@ "DeployTypeBinary" ] }, - "repository.LoginRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest": { "type": "object", "required": [ "login", @@ -1325,7 +1247,7 @@ } } }, - "repository.LoginResponse": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse": { "type": "object", "properties": { "is_active": { @@ -1354,7 +1276,7 @@ } } }, - "repository.RegistrationRequest": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest": { "type": "object", "required": [ "label" @@ -1365,7 +1287,7 @@ } } }, - "repository.TokenCreate": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate": { "type": "object", "required": [ "last_name", @@ -1400,7 +1322,7 @@ } } }, - "repository.TokenPasswordReset": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset": { "type": "object", "required": [ "new_password" @@ -1411,7 +1333,7 @@ } } }, - "repository.TokenUpdate": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate": { "type": "object", "properties": { "last_name": { @@ -1422,7 +1344,7 @@ } } }, - "repository.TokenUpdatePermissions": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions": { "type": "object", "properties": { "is_active": { @@ -1439,7 +1361,7 @@ } } }, - "repository.Tokens": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens": { "type": "object", "properties": { "id": { @@ -1471,7 +1393,7 @@ } } }, - "storage.LogEntry": { + "gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry": { "type": "object", "properties": { "agent": { @@ -1490,6 +1412,89 @@ "type": "string" } } + }, + "internal_handlers.AgentInfo": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "services": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "string" + } + } + }, + "internal_handlers.InsertLogRequest": { + "type": "object", + "required": [ + "agent", + "level", + "message", + "service" + ], + "properties": { + "agent": { + "type": "string" + }, + "level": { + "type": "string" + }, + "message": { + "type": "string" + }, + "service": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "internal_handlers.InsertLogsRequest": { + "type": "object", + "required": [ + "logs" + ], + "properties": { + "logs": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_handlers.InsertLogRequest" + } + } + } + }, + "internal_handlers.RegisterRequest": { + "type": "object", + "required": [ + "csr", + "token" + ], + "properties": { + "csr": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, + "internal_handlers.RegisterResponse": { + "type": "object", + "properties": { + "ca_cert": { + "type": "string" + }, + "client_cert": { + "type": "string" + } + } } }, "securityDefinitions": { diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index e8d189b..01feea1 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -1,60 +1,5 @@ definitions: - handlers.AgentInfo: - properties: - label: - type: string - services: - items: - type: string - type: array - token: - type: string - type: object - handlers.InsertLogRequest: - properties: - agent: - type: string - level: - type: string - message: - type: string - service: - type: string - timestamp: - type: string - required: - - agent - - level - - message - - service - type: object - handlers.InsertLogsRequest: - properties: - logs: - items: - $ref: '#/definitions/handlers.InsertLogRequest' - type: array - required: - - logs - type: object - handlers.RegisterRequest: - properties: - csr: - type: string - token: - type: string - required: - - csr - - token - type: object - handlers.RegisterResponse: - properties: - ca_cert: - type: string - client_cert: - type: string - type: object - repository.AgentDeployConfig: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig: description: Configuration for deploying HellreigN agent to a single server properties: agentLabel: @@ -62,11 +7,11 @@ definitions: type: string authMethod: allOf: - - $ref: '#/definitions/repository.AuthMethod' + - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod' example: key deployType: allOf: - - $ref: '#/definitions/repository.DeployType' + - $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType' example: docker ip: example: 192.168.1.100 @@ -90,7 +35,7 @@ definitions: - ip - user type: object - repository.AuthMethod: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AuthMethod: description: 'SSH authentication method: key or password' enum: - key @@ -99,18 +44,18 @@ definitions: x-enum-varnames: - AuthMethodKey - AuthMethodPassword - repository.DeployAgentsRequest: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest: description: Request to deploy HellreigN agents to multiple servers properties: servers: items: - $ref: '#/definitions/repository.AgentDeployConfig' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.AgentDeployConfig' minItems: 1 type: array required: - servers type: object - repository.DeployResponse: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse: description: Response containing deployment results and registration tokens properties: message: @@ -118,10 +63,10 @@ definitions: type: string results: items: - $ref: '#/definitions/repository.DeployResult' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult' type: array type: object - repository.DeployResult: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResult: description: Result of deploying to a single server properties: agent_label: @@ -140,7 +85,7 @@ definitions: example: abc123... type: string type: object - repository.DeployType: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployType: description: 'Type of deployment: docker or binary' enum: - docker @@ -149,7 +94,7 @@ definitions: x-enum-varnames: - DeployTypeDocker - DeployTypeBinary - repository.LoginRequest: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest: properties: login: type: string @@ -159,7 +104,7 @@ definitions: - login - password type: object - repository.LoginResponse: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse: properties: is_active: type: boolean @@ -178,14 +123,14 @@ definitions: token: type: string type: object - repository.RegistrationRequest: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest: properties: label: type: string required: - label type: object - repository.TokenCreate: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate: properties: is_active: type: boolean @@ -209,21 +154,21 @@ definitions: - name - password type: object - repository.TokenPasswordReset: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset: properties: new_password: type: string required: - new_password type: object - repository.TokenUpdate: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate: properties: last_name: type: string name: type: string type: object - repository.TokenUpdatePermissions: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions: properties: is_active: type: boolean @@ -234,7 +179,7 @@ definitions: permission_view: type: boolean type: object - repository.Tokens: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens: properties: id: type: integer @@ -255,7 +200,7 @@ definitions: token: type: string type: object - storage.LogEntry: + gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry: properties: agent: type: string @@ -268,6 +213,61 @@ definitions: timestamp: type: string type: object + internal_handlers.AgentInfo: + properties: + label: + type: string + services: + items: + type: string + type: array + token: + type: string + type: object + internal_handlers.InsertLogRequest: + properties: + agent: + type: string + level: + type: string + message: + type: string + service: + type: string + timestamp: + type: string + required: + - agent + - level + - message + - service + type: object + internal_handlers.InsertLogsRequest: + properties: + logs: + items: + $ref: '#/definitions/internal_handlers.InsertLogRequest' + type: array + required: + - logs + type: object + internal_handlers.RegisterRequest: + properties: + csr: + type: string + token: + type: string + required: + - csr + - token + type: object + internal_handlers.RegisterResponse: + properties: + ca_cert: + type: string + client_cert: + type: string + type: object info: contact: {} paths: @@ -281,7 +281,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/handlers.AgentInfo' + $ref: '#/definitions/internal_handlers.AgentInfo' type: array summary: Get connected agents tags: @@ -298,14 +298,14 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.DeployAgentsRequest' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployAgentsRequest' produces: - application/json responses: "200": description: Deployment results with tokens for each server schema: - $ref: '#/definitions/repository.DeployResponse' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.DeployResponse' "400": description: Invalid request schema: @@ -333,14 +333,14 @@ paths: name: request required: true schema: - $ref: '#/definitions/handlers.RegisterRequest' + $ref: '#/definitions/internal_handlers.RegisterRequest' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/handlers.RegisterResponse' + $ref: '#/definitions/internal_handlers.RegisterResponse' summary: Register agent tags: - agents @@ -354,7 +354,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.RegistrationRequest' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.RegistrationRequest' produces: - application/json responses: @@ -380,12 +380,12 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.LoginRequest' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginRequest' responses: "200": description: OK schema: - $ref: '#/definitions/repository.LoginResponse' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.LoginResponse' "400": description: Bad Request schema: @@ -442,7 +442,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.TokenCreate' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenCreate' responses: "200": description: OK @@ -481,7 +481,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/repository.Tokens' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' type: array "500": description: Internal Server Error @@ -538,7 +538,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/repository.Tokens' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' "400": description: Bad Request schema: @@ -575,7 +575,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.TokenUpdate' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdate' responses: "200": description: OK @@ -694,7 +694,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.TokenPasswordReset' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenPasswordReset' responses: "200": description: OK @@ -739,7 +739,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/repository.TokenUpdatePermissions' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.TokenUpdatePermissions' responses: "200": description: OK @@ -778,7 +778,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/repository.Tokens' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' type: array "500": description: Internal Server Error @@ -798,7 +798,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/repository.Tokens' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_repository.Tokens' "401": description: Unauthorized schema: @@ -849,7 +849,7 @@ paths: description: OK schema: items: - $ref: '#/definitions/storage.LogEntry' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry' type: array summary: Search logs tags: @@ -864,7 +864,7 @@ paths: name: body required: true schema: - $ref: '#/definitions/handlers.InsertLogRequest' + $ref: '#/definitions/internal_handlers.InsertLogRequest' produces: - application/json responses: @@ -903,7 +903,7 @@ paths: name: body required: true schema: - $ref: '#/definitions/handlers.InsertLogsRequest' + $ref: '#/definitions/internal_handlers.InsertLogsRequest' produces: - application/json responses: @@ -965,8 +965,10 @@ paths: description: OK schema: items: - $ref: '#/definitions/storage.LogEntry' + $ref: '#/definitions/gitea_d3m0k1d_ru_d3m0k1d_HellreigN_backend_internal_storage.LogEntry' type: array + security: + - Bearer: [] summary: Get mock logs tags: - logs diff --git a/backend/internal/grpcsrv/collector/collector.go b/backend/internal/grpcsrv/collector/collector.go new file mode 100644 index 0000000..246d123 --- /dev/null +++ b/backend/internal/grpcsrv/collector/collector.go @@ -0,0 +1,180 @@ +package collector + +import ( + "fmt" + "io" + "log" + "sync" + "time" + + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/proto/proto" + "google.golang.org/grpc/metadata" +) + +type Collector struct { + proto.UnimplementedCollectorServer + logRepo *repository.LogRepository + agents map[string]*Agent + mu sync.RWMutex + batchSize int + flushInterval time.Duration +} + +type Agent struct { + ID string + Label string + Services []string + ConnectedAt time.Time +} + +func New(logRepo *repository.LogRepository) *Collector { + return &Collector{ + logRepo: logRepo, + agents: make(map[string]*Agent), + batchSize: 100, + flushInterval: 2 * time.Second, + } +} + +func (c *Collector) Stream(stream proto.Collector_StreamServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return fmt.Errorf("no metadata in context") + } + + whoamiVals := md["whoami"] + if len(whoamiVals) == 0 { + return fmt.Errorf("whoami metadata missing") + } + agentName := whoamiVals[0] + + serviceVals := md["service"] + if len(serviceVals) == 0 { + return fmt.Errorf("service metadata missing") + } + service := serviceVals[0] + + servicesVals := md["services"] + var services []string + if len(servicesVals) > 0 { + services = servicesVals + } + + // Register agent + c.mu.Lock() + c.agents[agentName] = &Agent{ + ID: agentName, + Label: agentName, + Services: services, + ConnectedAt: time.Now(), + } + c.mu.Unlock() + + defer func() { + c.mu.Lock() + delete(c.agents, agentName) + c.mu.Unlock() + }() + + log.Printf("Agent %s connected, streaming logs for service: %s", agentName, service) + + // If no ClickHouse, just consume the stream without storing + if c.logRepo == nil { + log.Printf("Warning: logRepo is nil, consuming logs without storing for agent %s", agentName) + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("failed to receive: %w", err) + } + } + } + + // Channels for communication with recv goroutine + recvCh := make(chan *proto.CollectorRequest, 1) + errCh := make(chan error, 1) + + // Goroutine that blocks on Recv + go func() { + for { + req, err := stream.Recv() + if err != nil { + errCh <- err + return + } + recvCh <- req + } + }() + + // Buffer for batch inserts + var batch []storage.LogEntry + ticker := time.NewTicker(c.flushInterval) + defer ticker.Stop() + + flush := func() error { + if len(batch) == 0 { + return nil + } + if err := c.logRepo.InsertBatch(stream.Context(), batch); err != nil { + log.Printf("Failed to insert batch for agent %s, service %s: %v", agentName, service, err) + return err + } + log.Printf("Flushed %d logs for agent %s, service %s", len(batch), agentName, service) + batch = batch[:0] + return nil + } + + for { + select { + case <-stream.Context().Done(): + // Context cancelled, flush remaining + _ = flush() + return stream.Context().Err() + case <-ticker.C: + if err := flush(); err != nil { + return err + } + case req := <-recvCh: + batch = append(batch, storage.LogEntry{ + Timestamp: time.Now(), + Level: "info", + Service: service, + Agent: agentName, + Message: req.Message, + }) + + if len(batch) >= c.batchSize { + if err := flush(); err != nil { + return err + } + } + case err := <-errCh: + if err == io.EOF { + // Client closed stream + return flush() + } + return fmt.Errorf("failed to receive: %w", err) + } + } +} + +func (c *Collector) GetAgent(name string) (*Agent, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + a, ok := c.agents[name] + return a, ok +} + +func (c *Collector) Agents() []*Agent { + c.mu.RLock() + defer c.mu.RUnlock() + result := make([]*Agent, 0, len(c.agents)) + for _, a := range c.agents { + result = append(result, a) + } + return result +} diff --git a/backend/internal/handlers/agents.go b/backend/internal/handlers/agents.go index 34c16a4..8c55c53 100644 --- a/backend/internal/handlers/agents.go +++ b/backend/internal/handlers/agents.go @@ -1,41 +1,44 @@ package handlers import ( - "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/commander" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/grpcsrv/collector" "github.com/gin-gonic/gin" "net/http" ) type AgentsGroup struct { *Handlers - cmder *commander.Commander + collector *collector.Collector } -func NewAgentsGroup(h *Handlers, cmder *commander.Commander) AgentsGroup { - return AgentsGroup{Handlers: h, cmder: cmder} +func NewAgentsGroup(h *Handlers, coll *collector.Collector) AgentsGroup { + return AgentsGroup{Handlers: h, collector: coll} } type AgentInfo struct { - Token string `json:"token"` - Label string `json:"label"` - Services []string `json:"services"` + Token string `json:"token"` + Label string `json:"label"` + Services []string `json:"services"` + ConnectedAt string `json:"connected_at"` } // @Summary Get connected agents -// @Description Returns a list of all agents currently connected via gRPC streaming +// @Description Returns a list of all agents currently connected via Collector (log streaming) // @Tags agents // @Produce json // @Success 200 {array} AgentInfo // @Router /agents [get] func (ag *AgentsGroup) List(c *gin.Context) { agents := make([]AgentInfo, 0) - // iterate over the commander's agents map - for _, agent := range ag.cmder.Agents() { + + for _, agent := range ag.collector.Agents() { agents = append(agents, AgentInfo{ - Token: agent.Token, - Label: agent.Label, - Services: agent.Services, + Token: agent.ID, + Label: agent.Label, + Services: agent.Services, + ConnectedAt: agent.ConnectedAt.Format("2006-01-02 15:04:05"), }) } + c.JSON(http.StatusOK, agents) } diff --git a/backend/internal/handlers/logs_mock.go b/backend/internal/handlers/logs_mock.go index 01e30f7..9dc5bc1 100644 --- a/backend/internal/handlers/logs_mock.go +++ b/backend/internal/handlers/logs_mock.go @@ -21,6 +21,7 @@ import ( // @Param limit query int false "Limit results" default(100) // @Param offset query int false "Offset results" default(0) // @Success 200 {array} storage.LogEntry +// @Security Bearer // @Router /logs/mock [get] func (lh *LogHandlers) GetMockLogs(c *gin.Context) { levelFilter := c.Query("level")