From d96f952d737a92722425297ee7746b68f66f72e9 Mon Sep 17 00:00:00 2001 From: d3m0k1d Date: Fri, 3 Apr 2026 23:23:43 +0300 Subject: [PATCH] chore: add clickhouse as db for logs on agent and search --- backend/cmd/main.go | 31 +++ backend/go.mod | 13 + backend/go.sum | 73 ++++++ backend/internal/config/types.go | 1 + backend/internal/handlers/logs.go | 229 ++++++++++++++++++ backend/internal/repository/log_repository.go | 178 ++++++++++++++ backend/internal/storage/clickhouse.go | 46 ++++ backend/internal/storage/log_entry.go | 11 + backend/internal/storage/migrations.go | 13 + 9 files changed, 595 insertions(+) create mode 100644 backend/internal/handlers/logs.go create mode 100644 backend/internal/repository/log_repository.go create mode 100644 backend/internal/storage/log_entry.go diff --git a/backend/cmd/main.go b/backend/cmd/main.go index e93101b..0a71109 100644 --- a/backend/cmd/main.go +++ b/backend/cmd/main.go @@ -1,12 +1,14 @@ package cmd import ( + "context" "log" "os" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/docs" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/config" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/handlers" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository" "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage" "github.com/gin-gonic/gin" swaggerFiles "github.com/swaggo/files" @@ -52,6 +54,35 @@ func main() { agentsGroup.GET("", agents.List) } + logsGroup := v1.Group("/logs") + { + if cfg.Database.Clickhouse_host != "" { + chConn, err := storage.OpenClickHouse(storage.ClickHouseConfig{ + Host: cfg.Database.Clickhouse_host, + User: cfg.Database.Clickhouse_user, + Password: cfg.Database.Clickhouse_password, + Database: cfg.Database.Clickhouse_database, + }) + if err != nil { + log.Printf("Warning: ClickHouse connection failed: %v", err) + } else { + defer chConn.Close() + + logRepo := repository.NewLogRepository(chConn) + if err := logRepo.Init(context.Background()); err != nil { + log.Printf("Warning: Failed to initialize logs table: %v", err) + } + + logHandlers := handlers.NewLogHandlers(logRepo) + logsGroup.POST("", logHandlers.Insert) + logsGroup.POST("/batch", logHandlers.InsertBatch) + logsGroup.GET("", logHandlers.Search) + logsGroup.GET("/services", logHandlers.GetServices) + logsGroup.GET("/agents", logHandlers.GetAgents) + logsGroup.GET("/levels", logHandlers.GetLevels) + } + } + } } log.Fatal(router.Run(":8080")) diff --git a/backend/go.mod b/backend/go.mod index 610e4e0..b7a62a4 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,18 +3,24 @@ module gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend go 1.26.1 require ( + github.com/ClickHouse/ch-go v0.71.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.44.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.2.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/bytedance/gopkg v0.1.4 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.1 // indirect github.com/gin-gonic/gin v1.12.0 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-openapi/jsonpointer v0.22.5 // indirect github.com/go-openapi/jsonreference v0.21.5 // indirect github.com/go-openapi/spec v0.22.4 // indirect @@ -34,6 +40,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.9.2 // indirect @@ -41,16 +48,22 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/paulmach/orb v0.12.0 // indirect github.com/pelletier/go-toml/v2 v2.3.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/swaggo/files v1.0.1 // indirect github.com/swaggo/gin-swagger v1.6.1 // indirect github.com/swaggo/swag v1.16.6 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 0e88875..037638b 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,9 +1,15 @@ +github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= +github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= +github.com/ClickHouse/clickhouse-go/v2 v2.44.0 h1:9pxs5pRwIvhni5BDRPn/n5A8DeUod5TnBaeulFBX8EQ= +github.com/ClickHouse/clickhouse-go/v2 v2.44.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/PuerkitoBio/purell v1.2.1 h1:QsZ4TjvwiMpat6gBCBxEQI0rcS9ehtkKtSpiUnd9N28= github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPngi8iYYv/Eo= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM= github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -12,6 +18,8 @@ github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uS github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= @@ -29,6 +37,10 @@ github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= @@ -61,6 +73,11 @@ github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -68,10 +85,18 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -84,10 +109,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM= github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= @@ -95,11 +127,16 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -112,37 +149,64 @@ github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs github.com/swaggo/gin-swagger v1.6.1/go.mod h1:LQ+hJStHakCWRiK/YNYtJOu4mR2FP+pxLnILT/qNiTw= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.25.0 h1:qnk6Ksugpi5Bz32947rkUgDt9/s5qvqDPl/gBKdMJLE= golang.org/x/arch v0.25.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -155,19 +219,28 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/internal/config/types.go b/backend/internal/config/types.go index c3928cf..be763eb 100644 --- a/backend/internal/config/types.go +++ b/backend/internal/config/types.go @@ -9,4 +9,5 @@ type Databases struct { Clickhouse_host string `yaml:"clickhouse_host"` Clickhouse_user string `yaml:"clickhouse_user"` Clickhouse_password string `yaml:"clickhouse_password"` + Clickhouse_database string `yaml:"clickhouse_database"` } diff --git a/backend/internal/handlers/logs.go b/backend/internal/handlers/logs.go new file mode 100644 index 0000000..487b5fc --- /dev/null +++ b/backend/internal/handlers/logs.go @@ -0,0 +1,229 @@ +package handlers + +import ( + "context" + "net/http" + "time" + + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/repository" + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage" + "github.com/gin-gonic/gin" +) + +type LogHandlers struct { + LogRepo *repository.LogRepository +} + +func NewLogHandlers(logRepo *repository.LogRepository) *LogHandlers { + return &LogHandlers{LogRepo: logRepo} +} + +type InsertLogRequest struct { + Timestamp time.Time `json:"timestamp"` + Level string `json:"level" binding:"required"` + Service string `json:"service" binding:"required"` + Agent string `json:"agent" binding:"required"` + Message string `json:"message" binding:"required"` +} + +// @Summary Insert log entry +// @Description Inserts a single log entry into ClickHouse +// @Tags logs +// @Accept json +// @Produce json +// @Param body body InsertLogRequest true "Log entry" +// @Success 201 {object} map[string]string +// @Router /logs [post] +func (lh *LogHandlers) Insert(c *gin.Context) { + var req InsertLogRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if req.Timestamp.IsZero() { + req.Timestamp = time.Now() + } + + log := storage.LogEntry{ + Timestamp: req.Timestamp, + Level: req.Level, + Service: req.Service, + Agent: req.Agent, + Message: req.Message, + } + + if err := lh.LogRepo.Insert(c.Request.Context(), log); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to insert log"}) + return + } + + c.JSON(http.StatusCreated, gin.H{"status": "ok"}) +} + +type InsertLogsRequest struct { + Logs []InsertLogRequest `json:"logs" binding:"required"` +} + +// @Summary Insert log entries (batch) +// @Description Inserts multiple log entries into ClickHouse +// @Tags logs +// @Accept json +// @Produce json +// @Param body body InsertLogsRequest true "Log entries" +// @Success 201 {object} map[string]string +// @Router /logs/batch [post] +func (lh *LogHandlers) InsertBatch(c *gin.Context) { + var req InsertLogsRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + logs := make([]storage.LogEntry, len(req.Logs)) + for i, l := range req.Logs { + if l.Timestamp.IsZero() { + l.Timestamp = time.Now() + } + logs[i] = storage.LogEntry{ + Timestamp: l.Timestamp, + Level: l.Level, + Service: l.Service, + Agent: l.Agent, + Message: l.Message, + } + } + + if err := lh.LogRepo.InsertBatch(c.Request.Context(), logs); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to insert logs"}) + return + } + + c.JSON(http.StatusCreated, gin.H{"status": "ok", "count": len(logs)}) +} + +type SearchLogsRequest struct { + Level string `form:"level"` + Service string `form:"service"` + Agent string `form:"agent"` + DateFrom string `form:"date_from"` + DateTo string `form:"date_to"` + Limit int `form:"limit"` + Offset int `form:"offset"` +} + +// @Summary Search logs +// @Description Searches logs with various filters +// @Tags logs +// @Produce json +// @Param level query string false "Log level (INFO, WARNING, ERROR, FATAL)" +// @Param service query string false "Service name" +// @Param agent query string false "Agent name" +// @Param date_from query string false "Date from (RFC3339)" +// @Param date_to query string false "Date to (RFC3339)" +// @Param limit query int false "Limit results" default(100) +// @Param offset query int false "Offset results" default(0) +// @Success 200 {array} storage.LogEntry +// @Router /logs [get] +func (lh *LogHandlers) Search(c *gin.Context) { + var req SearchLogsRequest + if err := c.ShouldBindQuery(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + filter := repository.LogFilter{ + Level: req.Level, + Service: req.Service, + Agent: req.Agent, + Limit: req.Limit, + Offset: req.Offset, + } + + if req.DateFrom != "" { + if t, err := time.Parse(time.RFC3339, req.DateFrom); err == nil { + filter.DateFrom = t + } + } + + if req.DateTo != "" { + if t, err := time.Parse(time.RFC3339, req.DateTo); err == nil { + filter.DateTo = t + } + } + + if filter.Limit <= 0 { + filter.Limit = 100 + } + + logs, err := lh.LogRepo.Search(c.Request.Context(), filter) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to search logs"}) + return + } + + c.JSON(http.StatusOK, logs) +} + +// @Summary Get distinct services +// @Description Returns list of all unique service names in logs +// @Tags logs +// @Produce json +// @Success 200 {array} string +// @Router /logs/services [get] +func (lh *LogHandlers) GetServices(c *gin.Context) { + services, err := lh.LogRepo.GetDistinctServices(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get services"}) + return + } + + if services == nil { + services = []string{} + } + + c.JSON(http.StatusOK, services) +} + +// @Summary Get distinct agents +// @Description Returns list of all unique agent names in logs +// @Tags logs +// @Produce json +// @Success 200 {array} string +// @Router /logs/agents [get] +func (lh *LogHandlers) GetAgents(c *gin.Context) { + agents, err := lh.LogRepo.GetDistinctAgents(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get agents"}) + return + } + + if agents == nil { + agents = []string{} + } + + c.JSON(http.StatusOK, agents) +} + +// @Summary Get distinct log levels +// @Description Returns list of all unique log levels in logs +// @Tags logs +// @Produce json +// @Success 200 {array} string +// @Router /logs/levels [get] +func (lh *LogHandlers) GetLevels(c *gin.Context) { + levels, err := lh.LogRepo.GetDistinctLevels(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get levels"}) + return + } + + if levels == nil { + levels = []string{} + } + + c.JSON(http.StatusOK, levels) +} + +// Ensure context is used +var _ = context.Background diff --git a/backend/internal/repository/log_repository.go b/backend/internal/repository/log_repository.go new file mode 100644 index 0000000..5d5ec92 --- /dev/null +++ b/backend/internal/repository/log_repository.go @@ -0,0 +1,178 @@ +package repository + +import ( + "context" + "time" + + "gitea.d3m0k1d.ru/d3m0k1d/HellreigN/backend/internal/storage" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" +) + +type LogRepository struct { + Conn driver.Conn +} + +func NewLogRepository(conn driver.Conn) *LogRepository { + return &LogRepository{Conn: conn} +} + +func (r *LogRepository) Init(ctx context.Context) error { + return r.Conn.Exec(ctx, storage.CreateLogsTable) +} + +func (r *LogRepository) Insert(ctx context.Context, log storage.LogEntry) error { + return r.Conn.Exec(ctx, ` + INSERT INTO logs (timestamp, level, service, agent, message) + VALUES ($1, $2, $3, $4, $5) + `, log.Timestamp, log.Level, log.Service, log.Agent, log.Message) +} + +func (r *LogRepository) InsertBatch(ctx context.Context, logs []storage.LogEntry) error { + batch, err := r.Conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, level, service, agent, message)") + if err != nil { + return err + } + + for _, log := range logs { + if err := batch.Append(log.Timestamp, log.Level, log.Service, log.Agent, log.Message); err != nil { + return err + } + } + + return batch.Send() +} + +type LogFilter struct { + Level string + Service string + Agent string + DateFrom time.Time + DateTo time.Time + Limit int + Offset int +} + +func (r *LogRepository) Search(ctx context.Context, filter LogFilter) ([]storage.LogEntry, error) { + query := "SELECT timestamp, level, service, agent, message FROM logs WHERE 1=1" + args := make([]interface{}, 0) + argIdx := 1 + + if filter.Level != "" { + query += " AND level = $" + string(rune('0'+argIdx)) + args = append(args, filter.Level) + argIdx++ + } + + if filter.Service != "" { + query += " AND service = $" + string(rune('0'+argIdx)) + args = append(args, filter.Service) + argIdx++ + } + + if filter.Agent != "" { + query += " AND agent = $" + string(rune('0'+argIdx)) + args = append(args, filter.Agent) + argIdx++ + } + + if !filter.DateFrom.IsZero() { + query += " AND timestamp >= $" + string(rune('0'+argIdx)) + args = append(args, filter.DateFrom) + argIdx++ + } + + if !filter.DateTo.IsZero() { + query += " AND timestamp <= $" + string(rune('0'+argIdx)) + args = append(args, filter.DateTo) + argIdx++ + } + + query += " ORDER BY timestamp DESC" + + if filter.Limit > 0 { + query += " LIMIT $" + string(rune('0'+argIdx)) + args = append(args, filter.Limit) + argIdx++ + } else { + query += " LIMIT 100" + } + + if filter.Offset > 0 { + query += " OFFSET $" + string(rune('0'+argIdx)) + args = append(args, filter.Offset) + } + + rows, err := r.Conn.Query(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var logs []storage.LogEntry + for rows.Next() { + var log storage.LogEntry + if err := rows.Scan(&log.Timestamp, &log.Level, &log.Service, &log.Agent, &log.Message); err != nil { + return nil, err + } + logs = append(logs, log) + } + + return logs, rows.Err() +} + +func (r *LogRepository) GetDistinctServices(ctx context.Context) ([]string, error) { + rows, err := r.Conn.Query(ctx, "SELECT DISTINCT service FROM logs ORDER BY service") + if err != nil { + return nil, err + } + defer rows.Close() + + var services []string + for rows.Next() { + var service string + if err := rows.Scan(&service); err != nil { + return nil, err + } + services = append(services, service) + } + + return services, rows.Err() +} + +func (r *LogRepository) GetDistinctAgents(ctx context.Context) ([]string, error) { + rows, err := r.Conn.Query(ctx, "SELECT DISTINCT agent FROM logs ORDER BY agent") + if err != nil { + return nil, err + } + defer rows.Close() + + var agents []string + for rows.Next() { + var agent string + if err := rows.Scan(&agent); err != nil { + return nil, err + } + agents = append(agents, agent) + } + + return agents, rows.Err() +} + +func (r *LogRepository) GetDistinctLevels(ctx context.Context) ([]string, error) { + rows, err := r.Conn.Query(ctx, "SELECT DISTINCT level FROM logs ORDER BY level") + if err != nil { + return nil, err + } + defer rows.Close() + + var levels []string + for rows.Next() { + var level string + if err := rows.Scan(&level); err != nil { + return nil, err + } + levels = append(levels, level) + } + + return levels, rows.Err() +} diff --git a/backend/internal/storage/clickhouse.go b/backend/internal/storage/clickhouse.go index 82be054..60917d3 100644 --- a/backend/internal/storage/clickhouse.go +++ b/backend/internal/storage/clickhouse.go @@ -1 +1,47 @@ package storage + +import ( + "context" + "fmt" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" +) + +type ClickHouseConfig struct { + Host string + User string + Password string + Database string +} + +func OpenClickHouse(cfg ClickHouseConfig) (driver.Conn, error) { + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{cfg.Host}, + Auth: clickhouse.Auth{ + Database: cfg.Database, + Username: cfg.User, + Password: cfg.Password, + }, + Settings: clickhouse.Settings{ + "max_execution_time": 60, + }, + Compression: &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }, + DialTimeout: 30, + MaxOpenConns: 10, + MaxIdleConns: 5, + ConnMaxLifetime: 3600, + ConnOpenStrategy: clickhouse.ConnOpenInOrder, + }) + if err != nil { + return nil, fmt.Errorf("clickhouse connect: %w", err) + } + + if err := conn.Ping(context.Background()); err != nil { + return nil, fmt.Errorf("clickhouse ping: %w", err) + } + + return conn, nil +} diff --git a/backend/internal/storage/log_entry.go b/backend/internal/storage/log_entry.go new file mode 100644 index 0000000..38d60bf --- /dev/null +++ b/backend/internal/storage/log_entry.go @@ -0,0 +1,11 @@ +package storage + +import "time" + +type LogEntry struct { + Timestamp time.Time `ch:"timestamp"` + Level string `ch:"level"` + Service string `ch:"service"` + Agent string `ch:"agent"` + Message string `ch:"message"` +} diff --git a/backend/internal/storage/migrations.go b/backend/internal/storage/migrations.go index f9623ec..9a04c98 100644 --- a/backend/internal/storage/migrations.go +++ b/backend/internal/storage/migrations.go @@ -1,3 +1,16 @@ package storage const CreateSqlite = `` + +const CreateLogsTable = ` +CREATE TABLE IF NOT EXISTS logs ( + timestamp DateTime64(3) DEFAULT now(), + level LowCardinality(String), + service LowCardinality(String), + agent LowCardinality(String), + message String +) ENGINE = MergeTree() +ORDER BY (timestamp, level, service, agent) +TTL timestamp + INTERVAL 30 DAY +SETTINGS index_granularity = 8192 +`