From 4396bebe80fc03010b894b2a077f2787aac07f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=B8=D1=82=D0=B0=20=D0=A8=D1=83=D0=B1?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Sat, 26 Jul 2025 14:15:40 +0000 Subject: [PATCH] [hw-7] add metrics, tracing --- cart/cmd/server/main.go | 8 +++ cart/configs/values_local.yaml | 2 +- cart/go.mod | 18 +++++-- cart/go.sum | 20 ++++++- cart/internal/app/app.go | 8 ++- cart/internal/clients/loms/client.go | 7 +++ .../domain/repository/in_memory_repository.go | 3 ++ cart/internal/domain/service/service.go | 16 ++++++ cart/internal/domain/service/service_test.go | 42 ++++++++------- cart/internal/infra/http/metrics/metrics.go | 52 +++++++++++++++++++ .../internal/infra/http/metrics/repository.go | 16 ++++++ .../infra/http/middlewares/metrics.go | 38 ++++++++++++++ .../infra/http/middlewares/tracing.go | 17 ++++++ .../infra/http/round_trippers/metrics.go | 37 +++++++++++++ cart/internal/infra/tracing/init.go | 48 +++++++++++++++++ cart/internal/infra/tracing/otel.go | 22 ++++++++ docker-compose.yaml | 29 +++++++++++ go.work.sum | 14 +++++ loms/cmd/server/main.go | 9 ++++ loms/go.mod | 14 +++-- loms/go.sum | 16 ++++++ loms/internal/app/app.go | 11 +++- .../repository/orders/sqlc/repository.go | 2 +- .../repository/outbox/sqlc/repository.go | 2 +- .../repository/stocks/sqlc/repository.go | 2 +- loms/internal/domain/service/service.go | 19 +++++++ .../infra/{ => db}/postgres/postgres.go | 4 ++ loms/internal/infra/db/postgres/tracer.go | 38 ++++++++++++++ loms/internal/infra/{ => db}/postgres/tx.go | 0 loms/internal/infra/grpc/metrics/db.go | 31 +++++++++++ loms/internal/infra/grpc/metrics/metrics.go | 31 +++++++++++ .../internal/infra/grpc/middleware/metrics.go | 43 +++++++++++++++ .../internal/infra/grpc/middleware/tracing.go | 38 ++++++++++++++ .../infra/messaging/kafka/producer.go | 7 +++ loms/internal/infra/tracing/init.go | 48 +++++++++++++++++ loms/internal/infra/tracing/otel.go | 22 ++++++++ .../integration/loms_integration_test.go | 2 +- prometheus/prometheus.yml | 17 ++++++ 38 files changed, 717 insertions(+), 36 deletions(-) create mode 100644 cart/internal/infra/http/metrics/metrics.go create mode 100644 cart/internal/infra/http/metrics/repository.go create mode 100644 cart/internal/infra/http/middlewares/metrics.go create mode 100644 cart/internal/infra/http/middlewares/tracing.go create mode 100644 cart/internal/infra/http/round_trippers/metrics.go create mode 100644 cart/internal/infra/tracing/init.go create mode 100644 cart/internal/infra/tracing/otel.go rename loms/internal/infra/{ => db}/postgres/postgres.go (93%) create mode 100644 loms/internal/infra/db/postgres/tracer.go rename loms/internal/infra/{ => db}/postgres/tx.go (100%) create mode 100644 loms/internal/infra/grpc/metrics/db.go create mode 100644 loms/internal/infra/grpc/metrics/metrics.go create mode 100644 loms/internal/infra/grpc/middleware/metrics.go create mode 100644 loms/internal/infra/grpc/middleware/tracing.go create mode 100644 loms/internal/infra/tracing/init.go create mode 100644 loms/internal/infra/tracing/otel.go create mode 100644 prometheus/prometheus.yml diff --git a/cart/cmd/server/main.go b/cart/cmd/server/main.go index 6728ab7..897ff0d 100644 --- a/cart/cmd/server/main.go +++ b/cart/cmd/server/main.go @@ -11,9 +11,17 @@ import ( "github.com/rs/zerolog/log" "route256/cart/internal/app" + "route256/cart/internal/infra/tracing" ) func main() { + shut, err := tracing.InitOTLP(context.Background(), "cart", "jaeger:4318") + if err != nil { + log.Fatal().Err(err).Msg("tracing init failed") + } + defer func() { + _ = shut(context.Background()) + }() srv, err := app.NewApp(os.Getenv("CONFIG_FILE")) if err != nil { log.Fatal().Err(err).Msg("failed creating app") diff --git a/cart/configs/values_local.yaml b/cart/configs/values_local.yaml index c8a99bb..148f1c6 100644 --- a/cart/configs/values_local.yaml +++ b/cart/configs/values_local.yaml @@ -15,5 +15,5 @@ product_service: burst: 10 loms_service: - host: localhost + host: loms port: 8083 diff --git a/cart/go.mod b/cart/go.mod index 56ddbfc..4d3da13 100644 --- a/cart/go.mod +++ b/cart/go.mod @@ -7,6 +7,9 @@ require ( github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.37.0 + go.opentelemetry.io/otel v1.37.0 + go.opentelemetry.io/otel/sdk v1.37.0 + go.opentelemetry.io/otel/trace v1.37.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 @@ -16,7 +19,9 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect @@ -35,7 +40,8 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect @@ -46,12 +52,16 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/ozontech/allure-go/pkg/allure v0.6.14 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -59,15 +69,13 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect golang.org/x/crypto v0.38.0 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/text v0.25.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/protobuf v1.36.6 // indirect ) @@ -77,6 +85,8 @@ require ( github.com/gojuno/minimock/v3 v3.4.5 github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/prometheus/client_golang v1.22.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 go.uber.org/goleak v1.3.0 golang.org/x/sys v0.33.0 // indirect ) diff --git a/cart/go.sum b/cart/go.sum index e70916c..7cdfae0 100644 --- a/cart/go.sum +++ b/cart/go.sum @@ -6,8 +6,12 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= @@ -65,12 +69,14 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5uk github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -96,6 +102,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -110,6 +118,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= diff --git a/cart/internal/app/app.go b/cart/internal/app/app.go index 003e1b1..aae0c94 100644 --- a/cart/internal/app/app.go +++ b/cart/internal/app/app.go @@ -8,6 +8,7 @@ import ( "os" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "google.golang.org/grpc" @@ -90,6 +91,7 @@ func (app *App) setupCartService() (*service.CartService, error) { // Product service client transport := http.DefaultTransport transport = round_trippers.NewLogRoundTripper(transport) + transport = round_trippers.NewMetricsRoundTripper(transport) transport = round_trippers.NewRetryRoundTripper(transport, productsRetryAttemptsDefault, productsInitialDelaySecDefault) httpClient := http.Client{ @@ -133,7 +135,11 @@ func (app *App) BootstrapHandlers(cartService *service.CartService) http.Handler mx.HandleFunc("DELETE /user/{user_id}/cart/{sku_id}", s.DeleteItemHandler) mx.HandleFunc("DELETE /user/{user_id}/cart", s.DeleteItemsByUserIDHandler) - h := middlewares.NewTimerMiddleware(mx) + mx.Handle("GET /metrics", promhttp.Handler()) + + h := middlewares.NewTracingMiddleware(mx) + h = middlewares.NewMetricsMiddleware(h) + h = middlewares.NewTimerMiddleware(h) return h } diff --git a/cart/internal/clients/loms/client.go b/cart/internal/clients/loms/client.go index dcfef2a..e141326 100644 --- a/cart/internal/clients/loms/client.go +++ b/cart/internal/clients/loms/client.go @@ -6,6 +6,7 @@ import ( "route256/cart/internal/domain/entity" "route256/cart/internal/domain/model" + "route256/cart/internal/infra/tracing" pbLoms "route256/pkg/api/loms/v1" ) @@ -21,6 +22,9 @@ func NewLomsService(grpcClient pbLoms.LOMSClient) *Service { } func (s *Service) OrderCreate(ctx context.Context, cart *model.Cart) (int64, error) { + ctx, span := tracing.Tracer().Start(ctx, "Loms.OrderCreate") + defer span.End() + items := make([]*pbLoms.OrderItem, len(cart.Items)) for i, item := range cart.Items { items[i] = &pbLoms.OrderItem{ @@ -43,6 +47,9 @@ func (s *Service) OrderCreate(ctx context.Context, cart *model.Cart) (int64, err } func (s *Service) StocksInfo(ctx context.Context, sku entity.Sku) (uint32, error) { + ctx, span := tracing.Tracer().Start(ctx, "Loms.StocksInfo") + defer span.End() + req := &pbLoms.StocksInfoRequest{ Sku: int64(sku), } diff --git a/cart/internal/domain/repository/in_memory_repository.go b/cart/internal/domain/repository/in_memory_repository.go index dcaf8c9..e2ea721 100644 --- a/cart/internal/domain/repository/in_memory_repository.go +++ b/cart/internal/domain/repository/in_memory_repository.go @@ -6,6 +6,7 @@ import ( "route256/cart/internal/domain/entity" "route256/cart/internal/domain/model" + "route256/cart/internal/infra/http/metrics" ) type storage = map[entity.UID]*entity.Cart @@ -34,6 +35,7 @@ func (r *InMemoryRepository) AddItem(_ context.Context, userID entity.UID, item } r.storage[userID] = cart + metrics.SetInMemoryObjects(len(r.storage)) } if _, ok := cart.ItemCount[item.Product.Sku]; !ok { @@ -112,6 +114,7 @@ func (r *InMemoryRepository) DeleteItemsByUserID(_ context.Context, userID entit _, ok := r.storage[userID] if ok { delete(r.storage, userID) + metrics.SetInMemoryObjects(len(r.storage)) } return nil diff --git a/cart/internal/domain/service/service.go b/cart/internal/domain/service/service.go index d18742c..dde65c7 100644 --- a/cart/internal/domain/service/service.go +++ b/cart/internal/domain/service/service.go @@ -7,6 +7,7 @@ import ( "route256/cart/internal/domain/entity" "route256/cart/internal/domain/model" + "route256/cart/internal/infra/tracing" ) //go:generate minimock -i Repository -o ./mock -s _mock.go @@ -43,6 +44,9 @@ func NewCartService(repository Repository, productService ProductService, lomsSe } func (s *CartService) AddItem(ctx context.Context, userID entity.UID, item *model.Item) error { + ctx, span := tracing.Tracer().Start(ctx, "AddItem") + defer span.End() + if err := item.Validate(); err != nil { return fmt.Errorf("invalid requested values: %w", err) } @@ -76,6 +80,9 @@ func (s *CartService) AddItem(ctx context.Context, userID entity.UID, item *mode // and return a list of the collected items. // In case of failed request to product-service, return nothing and error. func (s *CartService) GetItemsByUserID(ctx context.Context, userID entity.UID) (*model.Cart, error) { + ctx, span := tracing.Tracer().Start(ctx, "GetItemsByUserID") + defer span.End() + if userID <= 0 { return nil, fmt.Errorf("userID invalid") } @@ -122,6 +129,9 @@ func (s *CartService) GetItemsByUserID(ctx context.Context, userID entity.UID) ( } func (s *CartService) DeleteItem(ctx context.Context, userID entity.UID, sku entity.Sku) error { + ctx, span := tracing.Tracer().Start(ctx, "DeleteItem") + defer span.End() + if userID <= 0 { return fmt.Errorf("userID invalid") } @@ -138,6 +148,9 @@ func (s *CartService) DeleteItem(ctx context.Context, userID entity.UID, sku ent } func (s *CartService) DeleteItemsByUserID(ctx context.Context, userID entity.UID) error { + ctx, span := tracing.Tracer().Start(ctx, "DeleteItemsByUserID") + defer span.End() + if userID <= 0 { return fmt.Errorf("userID invalid") } @@ -150,6 +163,9 @@ func (s *CartService) DeleteItemsByUserID(ctx context.Context, userID entity.UID } func (s *CartService) CheckoutUserCart(ctx context.Context, userID entity.UID) (int64, error) { + ctx, span := tracing.Tracer().Start(ctx, "CheckoutUserCart") + defer span.End() + if userID <= 0 { return 0, fmt.Errorf("userID invalid") } diff --git a/cart/internal/domain/service/service_test.go b/cart/internal/domain/service/service_test.go index 7532698..3e648ba 100644 --- a/cart/internal/domain/service/service_test.go +++ b/cart/internal/domain/service/service_test.go @@ -119,7 +119,8 @@ func TestCartService_AddItem(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). AddItemMock. - Expect(ctx, 1337, &testItem). + ExpectUserIDParam2(1337). + ExpectItemParam3(&testItem). Return(nil), productService: &productServiceFake{}, lomsService: &lomsServiceFake{}, @@ -190,7 +191,8 @@ func TestCartService_AddItem(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). AddItemMock. - Expect(ctx, 1337, &testItem). + ExpectUserIDParam2(1337). + ExpectItemParam3(&testItem). Return(assert.AnError), productService: &productServiceFake{}, lomsService: &lomsServiceFake{}, @@ -304,7 +306,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(testEntityCart, nil), productService: &productServiceFake{}, }, @@ -320,7 +322,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{ UserID: testUID, Items: []entity.Sku{testSKU}, @@ -353,7 +355,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{ UserID: testUID, Items: []entity.Sku{testSKU, 1}, @@ -407,7 +409,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{}, assert.AnError), productService: nil, }, @@ -423,7 +425,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{}, nil), productService: nil, }, @@ -439,7 +441,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{ UserID: testUID, Items: []entity.Sku{2}, @@ -459,7 +461,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(entity.Cart{ UserID: testUID, Items: []entity.Sku{2}, @@ -522,7 +524,8 @@ func TestCartService_DeleteItem(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). DeleteItemMock. - Expect(ctx, testUID, testSKU). + ExpectUserIDParam2(testUID). + ExpectSkuParam3(testSKU). Return(nil), }, args: args{ @@ -561,7 +564,8 @@ func TestCartService_DeleteItem(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). DeleteItemMock. - Expect(ctx, testUID, testSKU). + ExpectUserIDParam2(testUID). + ExpectSkuParam3(testSKU). Return(assert.AnError), }, args: args{ @@ -613,7 +617,7 @@ func TestCartService_DeleteItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). DeleteItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(nil), }, args: args{ @@ -638,7 +642,7 @@ func TestCartService_DeleteItemsByUserID(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). DeleteItemsByUserIDMock. - Expect(ctx, testUID). + ExpectUserIDParam2(testUID). Return(assert.AnError), }, args: args{ @@ -697,10 +701,10 @@ func TestCartService_CheckoutUserCart(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, 1337). + ExpectUserIDParam2(1337). Return(testCart, nil). DeleteItemsByUserIDMock. - Expect(ctx, 1337). + ExpectUserIDParam2(1337). Return(nil), productService: &productServiceFake{}, lomsService: &lomsServiceFake{}, @@ -720,7 +724,7 @@ func TestCartService_CheckoutUserCart(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, 1337). + ExpectUserIDParam2(1337). Return(entity.Cart{}, assert.AnError), }, args: args{ctx: ctx, userID: 1337}, @@ -731,7 +735,7 @@ func TestCartService_CheckoutUserCart(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, 1111). + ExpectUserIDParam2(1111). Return(testCart, nil), productService: &productServiceFake{}, lomsService: &lomsServiceFake{}, @@ -744,10 +748,10 @@ func TestCartService_CheckoutUserCart(t *testing.T) { fields: fields{ repository: mock.NewRepositoryMock(mc). GetItemsByUserIDMock. - Expect(ctx, 1337). + ExpectUserIDParam2(1337). Return(testCart, nil). DeleteItemsByUserIDMock. - Expect(ctx, 1337). + ExpectUserIDParam2(1337). Return(assert.AnError), productService: &productServiceFake{}, lomsService: &lomsServiceFake{}, diff --git a/cart/internal/infra/http/metrics/metrics.go b/cart/internal/infra/http/metrics/metrics.go new file mode 100644 index 0000000..c8a5675 --- /dev/null +++ b/cart/internal/infra/http/metrics/metrics.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requestCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "app", + Name: "requests_total", + Help: "Total amount of request by handler", + }, []string{"method", "path", "status_code"}) + + requestDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "app", + Name: "request_duration_seconds", + Help: "Latency of handler processing, seconds", + Buckets: prometheus.DefBuckets, + }, []string{"method", "path", "status_code"}) + + outboundCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "app", + Name: "outbound_requests_total", + Help: "Total HTTP requests to external services", + }, []string{"method", "url", "status_code"}) + + outboundDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "app", + Name: "outbound_request_duration_seconds", + Help: "Latency of outbound HTTP requests, seconds", + Buckets: prometheus.DefBuckets, + }, []string{"method", "url", "status_code"}) +) + +func IncRequestHandlerCount(method, path, statusCode string) { + requestCounter.WithLabelValues(method, path, statusCode).Inc() +} + +func StoreHandlerRequestDuration(method, path, statusCode string, d time.Duration) { + requestDurationHistogram.WithLabelValues(method, path, statusCode).Observe(d.Seconds()) +} + +func IncOutboundRequestCount(method, url, status string) { + outboundCounter.WithLabelValues(method, url, status).Inc() +} + +func StoreOutboundRequestDuration(method, url, status string, d time.Duration) { + outboundDurationHistogram.WithLabelValues(method, url, status).Observe(d.Seconds()) +} diff --git a/cart/internal/infra/http/metrics/repository.go b/cart/internal/infra/http/metrics/repository.go new file mode 100644 index 0000000..0b1e639 --- /dev/null +++ b/cart/internal/infra/http/metrics/repository.go @@ -0,0 +1,16 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var inMemoryObjectsGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "app", + Name: "inmemory_repo_objects", + Help: "Current in-memory repository size", +}) + +func SetInMemoryObjects(n int) { + inMemoryObjectsGauge.Set(float64(n)) +} diff --git a/cart/internal/infra/http/middlewares/metrics.go b/cart/internal/infra/http/middlewares/metrics.go new file mode 100644 index 0000000..3c3310e --- /dev/null +++ b/cart/internal/infra/http/middlewares/metrics.go @@ -0,0 +1,38 @@ +package middlewares + +import ( + "net/http" + "strconv" + "time" + + "route256/cart/internal/infra/http/metrics" +) + +type statusWriter struct { + http.ResponseWriter + statusCode int +} + +func (w *statusWriter) WriteHeader(code int) { + w.statusCode = code + w.ResponseWriter.WriteHeader(code) +} + +func NewMetricsMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sw := &statusWriter{ + ResponseWriter: w, + statusCode: http.StatusOK, + } + + start := time.Now() + + h.ServeHTTP(sw, r) + + path := r.URL.Path + status := strconv.Itoa(sw.statusCode) + + metrics.IncRequestHandlerCount(r.Method, path, status) + metrics.StoreHandlerRequestDuration(r.Method, path, status, time.Since(start)) + }) +} diff --git a/cart/internal/infra/http/middlewares/tracing.go b/cart/internal/infra/http/middlewares/tracing.go new file mode 100644 index 0000000..4114d62 --- /dev/null +++ b/cart/internal/infra/http/middlewares/tracing.go @@ -0,0 +1,17 @@ +package middlewares + +import ( + "net/http" + + "route256/cart/internal/infra/tracing" +) + +func NewTracingMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.Tracer().Start(r.Context(), + r.Method+" "+r.URL.Path) + defer span.End() + + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} diff --git a/cart/internal/infra/http/round_trippers/metrics.go b/cart/internal/infra/http/round_trippers/metrics.go new file mode 100644 index 0000000..edbcbaa --- /dev/null +++ b/cart/internal/infra/http/round_trippers/metrics.go @@ -0,0 +1,37 @@ +package round_trippers + +import ( + "net/http" + "strconv" + "time" + + "route256/cart/internal/infra/http/metrics" +) + +type MetricsRoundTripper struct { + rt http.RoundTripper +} + +func NewMetricsRoundTripper(rt http.RoundTripper) http.RoundTripper { + return &MetricsRoundTripper{ + rt: rt, + } +} + +func (m *MetricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := m.rt.RoundTrip(r) + + status := "error" + + if resp != nil { + status = strconv.Itoa(resp.StatusCode) + } + + url := r.URL.Path + + metrics.IncOutboundRequestCount(r.Method, url, status) + metrics.StoreOutboundRequestDuration(r.Method, url, status, time.Since(start)) + + return resp, err +} diff --git a/cart/internal/infra/tracing/init.go b/cart/internal/infra/tracing/init.go new file mode 100644 index 0000000..dca822b --- /dev/null +++ b/cart/internal/infra/tracing/init.go @@ -0,0 +1,48 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + globalTracer oteltrace.Tracer + provider *trace.TracerProvider +) + +func NewTracer(serviceName string, option ...trace.TracerProviderOption) oteltrace.Tracer { + option = append([]trace.TracerProviderOption{ + trace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + )), + }, option...) + + provider = trace.NewTracerProvider(option...) + + otel.SetTracerProvider(provider) + + globalTracer = otel.GetTracerProvider().Tracer("cart") + + return globalTracer +} + +func Shutdown(ctx context.Context) error { + if provider != nil { + return provider.Shutdown(ctx) + } + return nil +} + +func Tracer() oteltrace.Tracer { + if globalTracer == nil { + return otel.Tracer("cart") + } + + return globalTracer +} diff --git a/cart/internal/infra/tracing/otel.go b/cart/internal/infra/tracing/otel.go new file mode 100644 index 0000000..778b586 --- /dev/null +++ b/cart/internal/infra/tracing/otel.go @@ -0,0 +1,22 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func InitOTLP(ctx context.Context, serviceName, endpoint string) (shutdown func(context.Context) error, err error) { + exp, err := otlptracehttp.New(ctx, + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, err + } + + NewTracer(serviceName, sdktrace.WithBatcher(exp)) + + return Shutdown, nil +} diff --git a/docker-compose.yaml b/docker-compose.yaml index ccc6c37..7bace30 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,3 +1,7 @@ +volumes: + prometheus_data: {} + grafana_data: {} + services: cart: build: @@ -119,3 +123,28 @@ services: kafka-topics --bootstrap-server kafka:29092 --list " + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + volumes: + - grafana_data:/var/lib/grafana + ports: + - "3000:3000" + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" + - "4318:4318" diff --git a/go.work.sum b/go.work.sum index aa9f049..dd56d29 100644 --- a/go.work.sum +++ b/go.work.sum @@ -4,6 +4,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/ClickHouse/ch-go v0.65.1/go.mod h1:bsodgURwmrkvkBe5jw1qnGDgyITsYErfONKAHn05nv4= github.com/ClickHouse/clickhouse-go/v2 v2.34.0/go.mod h1:yioSINoRLVZkLyDzdMXPLRIqhDvel8iLBlwh6Iefso8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= @@ -32,15 +34,24 @@ github.com/hexdigest/gowrap v1.4.2/go.mod h1:s+1hE6qakgdaaLqgdwPAj5qKYVBCSbPJhEb github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/lyft/protoc-gen-star/v2 v2.0.4-0.20230330145011-496ad1ac90a4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mfridman/xflag v0.1.0/go.mod h1:/483ywM5ZO5SuMVjrIGquYNE5CzLrj5Ux/LxWWnjRaE= github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= @@ -50,6 +61,7 @@ github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d/go.mod h1:l8xTsYB90uaVdMHXMCxKKLSgw5wLYBwBKKefNIUnm9s= github.com/vertica/vertica-sql-go v1.3.3/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk/v3 v3.108.1/go.mod h1:l5sSv153E18VvYcsmr51hok9Sjc16tEC8AXGbwrk+ho= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= @@ -69,6 +81,7 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= @@ -82,4 +95,5 @@ google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20/go.mod h1:Nr5H8+MlGWr5+xX/STzdoEqJrO+YteqFbMyCsrb6mH0= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= diff --git a/loms/cmd/server/main.go b/loms/cmd/server/main.go index 01a2bec..d48d4fb 100644 --- a/loms/cmd/server/main.go +++ b/loms/cmd/server/main.go @@ -11,9 +11,18 @@ import ( "github.com/rs/zerolog/log" "route256/loms/internal/app" + "route256/loms/internal/infra/tracing" ) func main() { + shut, err := tracing.InitOTLP(context.Background(), "cart", "jaeger:4318") + if err != nil { + log.Fatal().Err(err).Msg("tracing init failed") + } + defer func() { + _ = shut(context.Background()) + }() + srv, err := app.NewApp(os.Getenv("CONFIG_FILE")) if err != nil { log.Fatal().Err(err).Msg("failed creating app") diff --git a/loms/go.mod b/loms/go.mod index fedbbda..33dd08d 100644 --- a/loms/go.mod +++ b/loms/go.mod @@ -9,9 +9,13 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/ozontech/allure-go/pkg/framework v0.6.34 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.22.0 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.37.0 + go.opentelemetry.io/otel v1.36.0 + go.opentelemetry.io/otel/sdk v1.36.0 + go.opentelemetry.io/otel/trace v1.36.0 go.uber.org/goleak v1.3.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 @@ -22,7 +26,9 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect @@ -59,10 +65,14 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.1 // indirect @@ -72,11 +82,8 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect ) @@ -93,6 +100,7 @@ require ( github.com/ozontech/allure-go/pkg/allure v0.6.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pressly/goose/v3 v3.24.3 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 golang.org/x/crypto v0.38.0 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/sync v0.14.0 // indirect diff --git a/loms/go.sum b/loms/go.sum index 82e9e11..7b8555a 100644 --- a/loms/go.sum +++ b/loms/go.sum @@ -8,8 +8,12 @@ github.com/IBM/sarama v1.45.2 h1:8m8LcMCu3REcwpa7fCP6v2fuPuzVwXDAM2DOv3CBrKw= github.com/IBM/sarama v1.45.2/go.mod h1:ppaoTcVdGv186/z6MEKsMm70A5fwJfRTpstI37kVn3Y= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= @@ -104,6 +108,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= @@ -130,6 +136,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -154,6 +162,14 @@ github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwW github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= diff --git a/loms/internal/app/app.go b/loms/internal/app/app.go index 4f44c83..0ed1e79 100644 --- a/loms/internal/app/app.go +++ b/loms/internal/app/app.go @@ -11,6 +11,7 @@ import ( "github.com/IBM/sarama" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/jackc/pgx/v5/pgxpool" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "google.golang.org/grpc" @@ -22,9 +23,9 @@ import ( stocksRepository "route256/loms/internal/domain/repository/stocks/sqlc" "route256/loms/internal/domain/service" "route256/loms/internal/infra/config" + "route256/loms/internal/infra/db/postgres" mw "route256/loms/internal/infra/grpc/middleware" "route256/loms/internal/infra/messaging/kafka" - "route256/loms/internal/infra/postgres" pb "route256/pkg/api/loms/v1" ) @@ -134,7 +135,9 @@ func (app *App) ListenAndServe(ctx context.Context) error { app.grpcServer = grpc.NewServer( grpc.ChainUnaryInterceptor( + mw.WithTracing, mw.Logging, + mw.WithMetrics, mw.Validate, ), ) @@ -166,9 +169,13 @@ func (app *App) ListenAndServe(ctx context.Context) error { return fmt.Errorf("pb.RegisterLOMSHandler: %w", err) } + root := http.NewServeMux() + root.Handle("/metrics", promhttp.Handler()) + root.Handle("/", gwmux) + app.httpServer = &http.Server{ Addr: fmt.Sprintf("%s:%s", app.config.Service.Host, app.config.Service.HTTPPort), - Handler: gwmux, + Handler: root, ReadTimeout: 10 * time.Second, } diff --git a/loms/internal/domain/repository/orders/sqlc/repository.go b/loms/internal/domain/repository/orders/sqlc/repository.go index fda855c..ba4abc0 100644 --- a/loms/internal/domain/repository/orders/sqlc/repository.go +++ b/loms/internal/domain/repository/orders/sqlc/repository.go @@ -9,7 +9,7 @@ import ( "route256/loms/internal/domain/entity" "route256/loms/internal/domain/model" "route256/loms/internal/domain/service" - "route256/loms/internal/infra/postgres" + "route256/loms/internal/infra/db/postgres" ) type orderRepo struct { diff --git a/loms/internal/domain/repository/outbox/sqlc/repository.go b/loms/internal/domain/repository/outbox/sqlc/repository.go index 1821b94..e821273 100644 --- a/loms/internal/domain/repository/outbox/sqlc/repository.go +++ b/loms/internal/domain/repository/outbox/sqlc/repository.go @@ -7,7 +7,7 @@ import ( "route256/loms/internal/domain/entity" "route256/loms/internal/domain/repository/outbox" - "route256/loms/internal/infra/postgres" + "route256/loms/internal/infra/db/postgres" ) type outboxRepo struct { diff --git a/loms/internal/domain/repository/stocks/sqlc/repository.go b/loms/internal/domain/repository/stocks/sqlc/repository.go index 9332a00..e843951 100644 --- a/loms/internal/domain/repository/stocks/sqlc/repository.go +++ b/loms/internal/domain/repository/stocks/sqlc/repository.go @@ -11,7 +11,7 @@ import ( "route256/loms/internal/domain/entity" "route256/loms/internal/domain/model" "route256/loms/internal/domain/service" - "route256/loms/internal/infra/postgres" + "route256/loms/internal/infra/db/postgres" "route256/loms/internal/infra/tools" ) diff --git a/loms/internal/domain/service/service.go b/loms/internal/domain/service/service.go index ffb77eb..2a7eb49 100644 --- a/loms/internal/domain/service/service.go +++ b/loms/internal/domain/service/service.go @@ -7,6 +7,7 @@ import ( "route256/loms/internal/domain/entity" "route256/loms/internal/domain/model" + "route256/loms/internal/infra/tracing" pb "route256/pkg/api/loms/v1" @@ -56,6 +57,9 @@ func NewLomsService(orderRepo OrderRepository, stockRepo StockRepository, txMana } func (s *LomsService) rollbackStocks(ctx context.Context, stocks []*entity.Stock) { + ctx, span := tracing.Tracer().Start(ctx, "rollbackStocks") + defer span.End() + for _, stock := range stocks { if err := s.stocks.StockCancel(ctx, stock); err != nil { log.Error().Err(err).Msg("failed to rollback stock") @@ -117,6 +121,9 @@ func (s *LomsService) createInitial(ctx context.Context, orderReq *pb.OrderCreat } func (s *LomsService) OrderCreate(ctx context.Context, orderReq *pb.OrderCreateRequest) (entity.ID, error) { + ctx, span := tracing.Tracer().Start(ctx, "OrderCreate") + defer span.End() + if orderReq == nil || orderReq.UserId <= 0 || len(orderReq.Items) == 0 { return 0, model.ErrInvalidInput } @@ -178,6 +185,9 @@ func (s *LomsService) OrderCreate(ctx context.Context, orderReq *pb.OrderCreateR } func (s *LomsService) OrderInfo(ctx context.Context, orderID entity.ID) (*entity.Order, error) { + ctx, span := tracing.Tracer().Start(ctx, "OrderInfo") + defer span.End() + if orderID <= 0 { return nil, model.ErrInvalidInput } @@ -186,6 +196,9 @@ func (s *LomsService) OrderInfo(ctx context.Context, orderID entity.ID) (*entity } func (s *LomsService) OrderPay(ctx context.Context, orderID entity.ID) error { + ctx, span := tracing.Tracer().Start(ctx, "OrderPay") + defer span.End() + if orderID <= 0 { return model.ErrInvalidInput } @@ -217,6 +230,9 @@ func (s *LomsService) OrderPay(ctx context.Context, orderID entity.ID) error { } func (s *LomsService) OrderCancel(ctx context.Context, orderID entity.ID) error { + ctx, span := tracing.Tracer().Start(ctx, "OrderCancel") + defer span.End() + if orderID <= 0 { return model.ErrInvalidInput } @@ -249,6 +265,9 @@ func (s *LomsService) OrderCancel(ctx context.Context, orderID entity.ID) error } func (s *LomsService) StocksInfo(ctx context.Context, sku entity.Sku) (uint32, error) { + ctx, span := tracing.Tracer().Start(ctx, "StocksInfo") + defer span.End() + if sku <= 0 { return 0, model.ErrInvalidInput } diff --git a/loms/internal/infra/postgres/postgres.go b/loms/internal/infra/db/postgres/postgres.go similarity index 93% rename from loms/internal/infra/postgres/postgres.go rename to loms/internal/infra/db/postgres/postgres.go index e8902b6..4d5bde7 100644 --- a/loms/internal/infra/postgres/postgres.go +++ b/loms/internal/infra/db/postgres/postgres.go @@ -15,6 +15,8 @@ func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { return nil, errors.Wrap(err, "pgxpool.ParseConfig") } + config.ConnConfig.Tracer = promTracer{} + pool, err := pgxpool.NewWithConfig(ctx, config) if err != nil { return nil, errors.Wrap(err, "pgxpool.NewWithConfig") @@ -34,6 +36,8 @@ func NewPools(ctx context.Context, DSNs ...string) ([]*pgxpool.Pool, error) { return nil, errors.Wrap(err, "pgxpool.ParseConfig") } + cfg.ConnConfig.Tracer = promTracer{} + pool, err := pgxpool.NewWithConfig(ctx, cfg) if err != nil { closeOpened(pools[:i]) diff --git a/loms/internal/infra/db/postgres/tracer.go b/loms/internal/infra/db/postgres/tracer.go new file mode 100644 index 0000000..f9de8fe --- /dev/null +++ b/loms/internal/infra/db/postgres/tracer.go @@ -0,0 +1,38 @@ +package postgres + +import ( + "context" + "strings" + "time" + + "route256/loms/internal/infra/grpc/metrics" + + "github.com/jackc/pgx/v5" +) + +type startKey struct{} + +type promTracer struct{} + +func (promTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + return context.WithValue(ctx, startKey{}, time.Now()) +} + +func (promTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + start, _ := ctx.Value(startKey{}).(time.Time) + + sql := strings.TrimSpace(strings.ToLower(data.CommandTag.String())) + parts := strings.SplitN(sql, " ", 2) + category := "unknown" + if len(parts) > 0 { + category = parts[0] + } + + errLabel := "none" + if data.Err != nil { + errLabel = "error" + } + + metrics.IncDBQueryCount(category, errLabel) + metrics.StoreDBQueryDuration(category, errLabel, time.Since(start)) +} diff --git a/loms/internal/infra/postgres/tx.go b/loms/internal/infra/db/postgres/tx.go similarity index 100% rename from loms/internal/infra/postgres/tx.go rename to loms/internal/infra/db/postgres/tx.go diff --git a/loms/internal/infra/grpc/metrics/db.go b/loms/internal/infra/grpc/metrics/db.go new file mode 100644 index 0000000..e3efbb3 --- /dev/null +++ b/loms/internal/infra/grpc/metrics/db.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + dbQueryCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "app", + Name: "db_queries_total", + Help: "Total DB queries", + }, []string{"category", "error"}) + + dbQueryDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "app", + Name: "db_query_duration_seconds", + Help: "Latency of DB queries, seconds", + Buckets: prometheus.DefBuckets, + }, []string{"category", "error"}) +) + +func IncDBQueryCount(cat, errLabel string) { + dbQueryCounter.WithLabelValues(cat, errLabel).Inc() +} + +func StoreDBQueryDuration(cat, errLabel string, d time.Duration) { + dbQueryDurationHistogram.WithLabelValues(cat, errLabel).Observe(d.Seconds()) +} diff --git a/loms/internal/infra/grpc/metrics/metrics.go b/loms/internal/infra/grpc/metrics/metrics.go new file mode 100644 index 0000000..3902f90 --- /dev/null +++ b/loms/internal/infra/grpc/metrics/metrics.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requestCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "app", + Name: "requests_total", + Help: "Total amount of request by handler", + }, []string{"method", "path", "status_code"}) + + requestDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "app", + Name: "request_duration_seconds", + Help: "Latency of handler processing, seconds", + Buckets: prometheus.DefBuckets, + }, []string{"method", "path", "status_code"}) +) + +func IncRequestHandlerCount(method, path, statusCode string) { + requestCounter.WithLabelValues(method, path, statusCode).Inc() +} + +func StoreHandlerRequestDuration(method, path, statusCode string, d time.Duration) { + requestDurationHistogram.WithLabelValues(method, path, statusCode).Observe(d.Seconds()) +} diff --git a/loms/internal/infra/grpc/middleware/metrics.go b/loms/internal/infra/grpc/middleware/metrics.go new file mode 100644 index 0000000..1da6d10 --- /dev/null +++ b/loms/internal/infra/grpc/middleware/metrics.go @@ -0,0 +1,43 @@ +package mw + +import ( + "context" + "strings" + "time" + + "route256/loms/internal/infra/grpc/metrics" + + "google.golang.org/grpc" + "google.golang.org/grpc/status" +) + +const UNKNOWN = "unknown" + +func WithMetrics(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + start := time.Now() + + resp, err = handler(ctx, req) + + // "/pkg.Service/Method" + service, method := parseFullMethod(info.FullMethod) + + code := status.Code(err).String() + + metrics.IncRequestHandlerCount(service, method, code) + metrics.StoreHandlerRequestDuration(service, method, code, time.Since(start)) + + return resp, err +} + +func parseFullMethod(full string) (service, method string) { + if full == "" { + return UNKNOWN, UNKNOWN + } + full = strings.TrimPrefix(full, "/") + + slash := strings.Index(full, "/") + if slash < 0 { + return full, UNKNOWN + } + return full[:slash], full[slash+1:] +} diff --git a/loms/internal/infra/grpc/middleware/tracing.go b/loms/internal/infra/grpc/middleware/tracing.go new file mode 100644 index 0000000..375f2cb --- /dev/null +++ b/loms/internal/infra/grpc/middleware/tracing.go @@ -0,0 +1,38 @@ +package mw + +import ( + "context" + "strings" + + "go.opentelemetry.io/otel/codes" + "google.golang.org/grpc" + "google.golang.org/grpc/status" + + "route256/loms/internal/infra/tracing" +) + +func WithTracing(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + service, method := splitFullMethod(info.FullMethod) + ctx, span := tracing.Tracer().Start(ctx, service+"/"+method) + defer span.End() + + resp, err := handler(ctx, req) + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, status.Convert(err).Message()) + } else { + span.SetStatus(codes.Ok, "OK") + } + + return resp, err +} + +func splitFullMethod(full string) (service, method string) { + full = strings.TrimPrefix(full, "/") + if i := strings.Index(full, "/"); i >= 0 { + return full[:i], full[i+1:] + } + + return "unknown", full +} diff --git a/loms/internal/infra/messaging/kafka/producer.go b/loms/internal/infra/messaging/kafka/producer.go index 9688ae0..d9e307c 100644 --- a/loms/internal/infra/messaging/kafka/producer.go +++ b/loms/internal/infra/messaging/kafka/producer.go @@ -8,6 +8,7 @@ import ( "time" "route256/loms/internal/domain/entity" + "route256/loms/internal/infra/tracing" "github.com/IBM/sarama" "github.com/rs/zerolog/log" @@ -64,6 +65,9 @@ func (p *statusProducer) runCallbacks() { } func (p *statusProducer) Send(ctx context.Context, id entity.ID, status string) error { + ctx, span := tracing.Tracer().Start(ctx, "Producer.Send") + defer span.End() + log.Debug().Msgf("sending event for id: %d; status: %s", id, status) newStatus, err := mapOrderStatus(status) @@ -86,6 +90,9 @@ func (p *statusProducer) Send(ctx context.Context, id entity.ID, status string) } func (p *statusProducer) SendRaw(ctx context.Context, id entity.ID, value []byte) error { + ctx, span := tracing.Tracer().Start(ctx, "Producer.SendRaw") + defer span.End() + if len(value) == 0 { return fmt.Errorf("empty message value") } diff --git a/loms/internal/infra/tracing/init.go b/loms/internal/infra/tracing/init.go new file mode 100644 index 0000000..bb27cda --- /dev/null +++ b/loms/internal/infra/tracing/init.go @@ -0,0 +1,48 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + globalTracer oteltrace.Tracer + provider *trace.TracerProvider +) + +func NewTracer(serviceName string, option ...trace.TracerProviderOption) oteltrace.Tracer { + option = append([]trace.TracerProviderOption{ + trace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + )), + }, option...) + + provider = trace.NewTracerProvider(option...) + + otel.SetTracerProvider(provider) + + globalTracer = otel.GetTracerProvider().Tracer("loms") + + return globalTracer +} + +func Shutdown(ctx context.Context) error { + if provider != nil { + return provider.Shutdown(ctx) + } + return nil +} + +func Tracer() oteltrace.Tracer { + if globalTracer == nil { + return otel.Tracer("loms") + } + + return globalTracer +} diff --git a/loms/internal/infra/tracing/otel.go b/loms/internal/infra/tracing/otel.go new file mode 100644 index 0000000..778b586 --- /dev/null +++ b/loms/internal/infra/tracing/otel.go @@ -0,0 +1,22 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func InitOTLP(ctx context.Context, serviceName, endpoint string) (shutdown func(context.Context) error, err error) { + exp, err := otlptracehttp.New(ctx, + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, err + } + + NewTracer(serviceName, sdktrace.WithBatcher(exp)) + + return Shutdown, nil +} diff --git a/loms/tests/integration/loms_integration_test.go b/loms/tests/integration/loms_integration_test.go index 8799a4c..bf0dc37 100644 --- a/loms/tests/integration/loms_integration_test.go +++ b/loms/tests/integration/loms_integration_test.go @@ -29,7 +29,7 @@ import ( ordersRepository "route256/loms/internal/domain/repository/orders/sqlc" stocksRepository "route256/loms/internal/domain/repository/stocks/sqlc" lomsService "route256/loms/internal/domain/service" - "route256/loms/internal/infra/postgres" + "route256/loms/internal/infra/db/postgres" pb "route256/pkg/api/loms/v1" ) diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..d845eec --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,17 @@ +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'cart' + scrape_interval: 5s + static_configs: + - targets: ['cart:8080'] + + - job_name: 'loms' + scrape_interval: 5s + static_configs: + - targets: ['loms:8084']