[hw-7] add metrics, tracing

loms/cmd/server/main.go

@@ -11,9 +11,18 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"route256/loms/internal/app"
+	"route256/loms/internal/infra/tracing"
 )
 
 func main() {
+	shut, err := tracing.InitOTLP(context.Background(), "cart", "jaeger:4318")
+	if err != nil {
+		log.Fatal().Err(err).Msg("tracing init failed")
+	}
+	defer func() {
+		_ = shut(context.Background())
+	}()
+
 	srv, err := app.NewApp(os.Getenv("CONFIG_FILE"))
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating app")

loms/go.mod

@@ -9,9 +9,13 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/ozontech/allure-go/pkg/framework v0.6.34
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.22.0
 	github.com/rs/zerolog v1.34.0
 	github.com/stretchr/testify v1.10.0
 	github.com/testcontainers/testcontainers-go v0.37.0
+	go.opentelemetry.io/otel v1.36.0
+	go.opentelemetry.io/otel/sdk v1.36.0
+	go.opentelemetry.io/otel/trace v1.36.0
 	go.uber.org/goleak v1.3.0
 	google.golang.org/grpc v1.73.0
 	google.golang.org/protobuf v1.36.6
@@ -22,7 +26,9 @@ require (
 	dario.cat/mergo v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
@@ -59,10 +65,14 @@ require (
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/shirou/gopsutil/v4 v4.25.1 // indirect
@@ -72,11 +82,8 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.36.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect
 	go.opentelemetry.io/otel/metric v1.36.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
-	go.opentelemetry.io/otel/trace v1.36.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 )
@@ -93,6 +100,7 @@ require (
 	github.com/ozontech/allure-go/pkg/allure v0.6.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pressly/goose/v3 v3.24.3
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
 	golang.org/x/crypto v0.38.0 // indirect
 	golang.org/x/net v0.40.0 // indirect
 	golang.org/x/sync v0.14.0 // indirect

loms/go.sum

@@ -8,8 +8,12 @@ github.com/IBM/sarama v1.45.2 h1:8m8LcMCu3REcwpa7fCP6v2fuPuzVwXDAM2DOv3CBrKw=
 github.com/IBM/sarama v1.45.2/go.mod h1:ppaoTcVdGv186/z6MEKsMm70A5fwJfRTpstI37kVn3Y=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
@@ -104,6 +108,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
@@ -130,6 +136,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -154,6 +162,14 @@ github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwW
 github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

loms/internal/app/app.go

@@ -11,6 +11,7 @@ import (
 	"github.com/IBM/sarama"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"google.golang.org/grpc"
@@ -22,9 +23,9 @@ import (
 	stocksRepository "route256/loms/internal/domain/repository/stocks/sqlc"
 	"route256/loms/internal/domain/service"
 	"route256/loms/internal/infra/config"
+	"route256/loms/internal/infra/db/postgres"
 	mw "route256/loms/internal/infra/grpc/middleware"
 	"route256/loms/internal/infra/messaging/kafka"
-	"route256/loms/internal/infra/postgres"
 
 	pb "route256/pkg/api/loms/v1"
 )
@@ -134,7 +135,9 @@ func (app *App) ListenAndServe(ctx context.Context) error {
 
 	app.grpcServer = grpc.NewServer(
 		grpc.ChainUnaryInterceptor(
+			mw.WithTracing,
 			mw.Logging,
+			mw.WithMetrics,
 			mw.Validate,
 		),
 	)
@@ -166,9 +169,13 @@ func (app *App) ListenAndServe(ctx context.Context) error {
 		return fmt.Errorf("pb.RegisterLOMSHandler: %w", err)
 	}
 
+	root := http.NewServeMux()
+	root.Handle("/metrics", promhttp.Handler())
+	root.Handle("/", gwmux)
+
 	app.httpServer = &http.Server{
 		Addr:        fmt.Sprintf("%s:%s", app.config.Service.Host, app.config.Service.HTTPPort),
-		Handler:     gwmux,
+		Handler:     root,
 		ReadTimeout: 10 * time.Second,
 	}
 

loms/internal/domain/repository/orders/sqlc/repository.go

@@ -9,7 +9,7 @@ import (
 	"route256/loms/internal/domain/entity"
 	"route256/loms/internal/domain/model"
 	"route256/loms/internal/domain/service"
-	"route256/loms/internal/infra/postgres"
+	"route256/loms/internal/infra/db/postgres"
 )
 
 type orderRepo struct {

loms/internal/domain/repository/outbox/sqlc/repository.go

@@ -7,7 +7,7 @@ import (
 
 	"route256/loms/internal/domain/entity"
 	"route256/loms/internal/domain/repository/outbox"
-	"route256/loms/internal/infra/postgres"
+	"route256/loms/internal/infra/db/postgres"
 )
 
 type outboxRepo struct {

loms/internal/domain/repository/stocks/sqlc/repository.go

@@ -11,7 +11,7 @@ import (
 	"route256/loms/internal/domain/entity"
 	"route256/loms/internal/domain/model"
 	"route256/loms/internal/domain/service"
-	"route256/loms/internal/infra/postgres"
+	"route256/loms/internal/infra/db/postgres"
 	"route256/loms/internal/infra/tools"
 )
 

loms/internal/domain/service/service.go

@@ -7,6 +7,7 @@ import (
 
 	"route256/loms/internal/domain/entity"
 	"route256/loms/internal/domain/model"
+	"route256/loms/internal/infra/tracing"
 
 	pb "route256/pkg/api/loms/v1"
 
@@ -56,6 +57,9 @@ func NewLomsService(orderRepo OrderRepository, stockRepo StockRepository, txMana
 }
 
 func (s *LomsService) rollbackStocks(ctx context.Context, stocks []*entity.Stock) {
+	ctx, span := tracing.Tracer().Start(ctx, "rollbackStocks")
+	defer span.End()
+
 	for _, stock := range stocks {
 		if err := s.stocks.StockCancel(ctx, stock); err != nil {
 			log.Error().Err(err).Msg("failed to rollback stock")
@@ -117,6 +121,9 @@ func (s *LomsService) createInitial(ctx context.Context, orderReq *pb.OrderCreat
 }
 
 func (s *LomsService) OrderCreate(ctx context.Context, orderReq *pb.OrderCreateRequest) (entity.ID, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "OrderCreate")
+	defer span.End()
+
 	if orderReq == nil || orderReq.UserId <= 0 || len(orderReq.Items) == 0 {
 		return 0, model.ErrInvalidInput
 	}
@@ -178,6 +185,9 @@ func (s *LomsService) OrderCreate(ctx context.Context, orderReq *pb.OrderCreateR
 }
 
 func (s *LomsService) OrderInfo(ctx context.Context, orderID entity.ID) (*entity.Order, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "OrderInfo")
+	defer span.End()
+
 	if orderID <= 0 {
 		return nil, model.ErrInvalidInput
 	}
@@ -186,6 +196,9 @@ func (s *LomsService) OrderInfo(ctx context.Context, orderID entity.ID) (*entity
 }
 
 func (s *LomsService) OrderPay(ctx context.Context, orderID entity.ID) error {
+	ctx, span := tracing.Tracer().Start(ctx, "OrderPay")
+	defer span.End()
+
 	if orderID <= 0 {
 		return model.ErrInvalidInput
 	}
@@ -217,6 +230,9 @@ func (s *LomsService) OrderPay(ctx context.Context, orderID entity.ID) error {
 }
 
 func (s *LomsService) OrderCancel(ctx context.Context, orderID entity.ID) error {
+	ctx, span := tracing.Tracer().Start(ctx, "OrderCancel")
+	defer span.End()
+
 	if orderID <= 0 {
 		return model.ErrInvalidInput
 	}
@@ -249,6 +265,9 @@ func (s *LomsService) OrderCancel(ctx context.Context, orderID entity.ID) error
 }
 
 func (s *LomsService) StocksInfo(ctx context.Context, sku entity.Sku) (uint32, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "StocksInfo")
+	defer span.End()
+
 	if sku <= 0 {
 		return 0, model.ErrInvalidInput
 	}

loms/internal/infra/db/postgres/postgres.go

@@ -15,6 +15,8 @@ func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
 		return nil, errors.Wrap(err, "pgxpool.ParseConfig")
 	}
 
+	config.ConnConfig.Tracer = promTracer{}
+
 	pool, err := pgxpool.NewWithConfig(ctx, config)
 	if err != nil {
 		return nil, errors.Wrap(err, "pgxpool.NewWithConfig")
@@ -34,6 +36,8 @@ func NewPools(ctx context.Context, DSNs ...string) ([]*pgxpool.Pool, error) {
 			return nil, errors.Wrap(err, "pgxpool.ParseConfig")
 		}
 
+		cfg.ConnConfig.Tracer = promTracer{}
+
 		pool, err := pgxpool.NewWithConfig(ctx, cfg)
 		if err != nil {
 			closeOpened(pools[:i])

loms/internal/infra/db/postgres/tracer.go

@@ -0,0 +1,38 @@
+package postgres
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"route256/loms/internal/infra/grpc/metrics"
+
+	"github.com/jackc/pgx/v5"
+)
+
+type startKey struct{}
+
+type promTracer struct{}
+
+func (promTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
+	return context.WithValue(ctx, startKey{}, time.Now())
+}
+
+func (promTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
+	start, _ := ctx.Value(startKey{}).(time.Time)
+
+	sql := strings.TrimSpace(strings.ToLower(data.CommandTag.String()))
+	parts := strings.SplitN(sql, " ", 2)
+	category := "unknown"
+	if len(parts) > 0 {
+		category = parts[0]
+	}
+
+	errLabel := "none"
+	if data.Err != nil {
+		errLabel = "error"
+	}
+
+	metrics.IncDBQueryCount(category, errLabel)
+	metrics.StoreDBQueryDuration(category, errLabel, time.Since(start))
+}

loms/internal/infra/grpc/metrics/db.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	dbQueryCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "app",
+		Name:      "db_queries_total",
+		Help:      "Total DB queries",
+	}, []string{"category", "error"})
+
+	dbQueryDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "app",
+		Name:      "db_query_duration_seconds",
+		Help:      "Latency of DB queries, seconds",
+		Buckets:   prometheus.DefBuckets,
+	}, []string{"category", "error"})
+)
+
+func IncDBQueryCount(cat, errLabel string) {
+	dbQueryCounter.WithLabelValues(cat, errLabel).Inc()
+}
+
+func StoreDBQueryDuration(cat, errLabel string, d time.Duration) {
+	dbQueryDurationHistogram.WithLabelValues(cat, errLabel).Observe(d.Seconds())
+}

loms/internal/infra/grpc/metrics/metrics.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	requestCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "app",
+		Name:      "requests_total",
+		Help:      "Total amount of request by handler",
+	}, []string{"method", "path", "status_code"})
+
+	requestDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "app",
+		Name:      "request_duration_seconds",
+		Help:      "Latency of handler processing, seconds",
+		Buckets:   prometheus.DefBuckets,
+	}, []string{"method", "path", "status_code"})
+)
+
+func IncRequestHandlerCount(method, path, statusCode string) {
+	requestCounter.WithLabelValues(method, path, statusCode).Inc()
+}
+
+func StoreHandlerRequestDuration(method, path, statusCode string, d time.Duration) {
+	requestDurationHistogram.WithLabelValues(method, path, statusCode).Observe(d.Seconds())
+}

loms/internal/infra/grpc/middleware/metrics.go

@@ -0,0 +1,43 @@
+package mw
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"route256/loms/internal/infra/grpc/metrics"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+)
+
+const UNKNOWN = "unknown"
+
+func WithMetrics(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
+	start := time.Now()
+
+	resp, err = handler(ctx, req)
+
+	// "/pkg.Service/Method"
+	service, method := parseFullMethod(info.FullMethod)
+
+	code := status.Code(err).String()
+
+	metrics.IncRequestHandlerCount(service, method, code)
+	metrics.StoreHandlerRequestDuration(service, method, code, time.Since(start))
+
+	return resp, err
+}
+
+func parseFullMethod(full string) (service, method string) {
+	if full == "" {
+		return UNKNOWN, UNKNOWN
+	}
+	full = strings.TrimPrefix(full, "/")
+
+	slash := strings.Index(full, "/")
+	if slash < 0 {
+		return full, UNKNOWN
+	}
+	return full[:slash], full[slash+1:]
+}

loms/internal/infra/grpc/middleware/tracing.go

@@ -0,0 +1,38 @@
+package mw
+
+import (
+	"context"
+	"strings"
+
+	"go.opentelemetry.io/otel/codes"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+
+	"route256/loms/internal/infra/tracing"
+)
+
+func WithTracing(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+	service, method := splitFullMethod(info.FullMethod)
+	ctx, span := tracing.Tracer().Start(ctx, service+"/"+method)
+	defer span.End()
+
+	resp, err := handler(ctx, req)
+
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, status.Convert(err).Message())
+	} else {
+		span.SetStatus(codes.Ok, "OK")
+	}
+
+	return resp, err
+}
+
+func splitFullMethod(full string) (service, method string) {
+	full = strings.TrimPrefix(full, "/")
+	if i := strings.Index(full, "/"); i >= 0 {
+		return full[:i], full[i+1:]
+	}
+
+	return "unknown", full
+}

loms/internal/infra/messaging/kafka/producer.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"route256/loms/internal/domain/entity"
+	"route256/loms/internal/infra/tracing"
 
 	"github.com/IBM/sarama"
 	"github.com/rs/zerolog/log"
@@ -64,6 +65,9 @@ func (p *statusProducer) runCallbacks() {
 }
 
 func (p *statusProducer) Send(ctx context.Context, id entity.ID, status string) error {
+	ctx, span := tracing.Tracer().Start(ctx, "Producer.Send")
+	defer span.End()
+
 	log.Debug().Msgf("sending event for id: %d; status: %s", id, status)
 
 	newStatus, err := mapOrderStatus(status)
@@ -86,6 +90,9 @@ func (p *statusProducer) Send(ctx context.Context, id entity.ID, status string)
 }
 
 func (p *statusProducer) SendRaw(ctx context.Context, id entity.ID, value []byte) error {
+	ctx, span := tracing.Tracer().Start(ctx, "Producer.SendRaw")
+	defer span.End()
+
 	if len(value) == 0 {
 		return fmt.Errorf("empty message value")
 	}

loms/internal/infra/tracing/init.go

@@ -0,0 +1,48 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+var (
+	globalTracer oteltrace.Tracer
+	provider     *trace.TracerProvider
+)
+
+func NewTracer(serviceName string, option ...trace.TracerProviderOption) oteltrace.Tracer {
+	option = append([]trace.TracerProviderOption{
+		trace.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(serviceName),
+		)),
+	}, option...)
+
+	provider = trace.NewTracerProvider(option...)
+
+	otel.SetTracerProvider(provider)
+
+	globalTracer = otel.GetTracerProvider().Tracer("loms")
+
+	return globalTracer
+}
+
+func Shutdown(ctx context.Context) error {
+	if provider != nil {
+		return provider.Shutdown(ctx)
+	}
+	return nil
+}
+
+func Tracer() oteltrace.Tracer {
+	if globalTracer == nil {
+		return otel.Tracer("loms")
+	}
+
+	return globalTracer
+}

loms/internal/infra/tracing/otel.go

@@ -0,0 +1,22 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func InitOTLP(ctx context.Context, serviceName, endpoint string) (shutdown func(context.Context) error, err error) {
+	exp, err := otlptracehttp.New(ctx,
+		otlptracehttp.WithEndpoint(endpoint),
+		otlptracehttp.WithInsecure(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	NewTracer(serviceName, sdktrace.WithBatcher(exp))
+
+	return Shutdown, nil
+}

loms/tests/integration/loms_integration_test.go

@@ -29,7 +29,7 @@ import (
 	ordersRepository "route256/loms/internal/domain/repository/orders/sqlc"
 	stocksRepository "route256/loms/internal/domain/repository/stocks/sqlc"
 	lomsService "route256/loms/internal/domain/service"
-	"route256/loms/internal/infra/postgres"
+	"route256/loms/internal/infra/db/postgres"
 
 	pb "route256/pkg/api/loms/v1"
 )