[hw-7] add metrics, tracing

cart/internal/app/app.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"google.golang.org/grpc"
@@ -90,6 +91,7 @@ func (app *App) setupCartService() (*service.CartService, error) {
 	// Product service client
 	transport := http.DefaultTransport
 	transport = round_trippers.NewLogRoundTripper(transport)
+	transport = round_trippers.NewMetricsRoundTripper(transport)
 	transport = round_trippers.NewRetryRoundTripper(transport, productsRetryAttemptsDefault, productsInitialDelaySecDefault)
 
 	httpClient := http.Client{
@@ -133,7 +135,11 @@ func (app *App) BootstrapHandlers(cartService *service.CartService) http.Handler
 	mx.HandleFunc("DELETE /user/{user_id}/cart/{sku_id}", s.DeleteItemHandler)
 	mx.HandleFunc("DELETE /user/{user_id}/cart", s.DeleteItemsByUserIDHandler)
 
-	h := middlewares.NewTimerMiddleware(mx)
+	mx.Handle("GET /metrics", promhttp.Handler())
+
+	h := middlewares.NewTracingMiddleware(mx)
+	h = middlewares.NewMetricsMiddleware(h)
+	h = middlewares.NewTimerMiddleware(h)
 
 	return h
 }

cart/internal/clients/loms/client.go

@@ -6,6 +6,7 @@ import (
 
 	"route256/cart/internal/domain/entity"
 	"route256/cart/internal/domain/model"
+	"route256/cart/internal/infra/tracing"
 
 	pbLoms "route256/pkg/api/loms/v1"
 )
@@ -21,6 +22,9 @@ func NewLomsService(grpcClient pbLoms.LOMSClient) *Service {
 }
 
 func (s *Service) OrderCreate(ctx context.Context, cart *model.Cart) (int64, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "Loms.OrderCreate")
+	defer span.End()
+
 	items := make([]*pbLoms.OrderItem, len(cart.Items))
 	for i, item := range cart.Items {
 		items[i] = &pbLoms.OrderItem{
@@ -43,6 +47,9 @@ func (s *Service) OrderCreate(ctx context.Context, cart *model.Cart) (int64, err
 }
 
 func (s *Service) StocksInfo(ctx context.Context, sku entity.Sku) (uint32, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "Loms.StocksInfo")
+	defer span.End()
+
 	req := &pbLoms.StocksInfoRequest{
 		Sku: int64(sku),
 	}

cart/internal/domain/repository/in_memory_repository.go

@@ -6,6 +6,7 @@ import (
 
 	"route256/cart/internal/domain/entity"
 	"route256/cart/internal/domain/model"
+	"route256/cart/internal/infra/http/metrics"
 )
 
 type storage = map[entity.UID]*entity.Cart
@@ -34,6 +35,7 @@ func (r *InMemoryRepository) AddItem(_ context.Context, userID entity.UID, item
 		}
 
 		r.storage[userID] = cart
+		metrics.SetInMemoryObjects(len(r.storage))
 	}
 
 	if _, ok := cart.ItemCount[item.Product.Sku]; !ok {
@@ -112,6 +114,7 @@ func (r *InMemoryRepository) DeleteItemsByUserID(_ context.Context, userID entit
 	_, ok := r.storage[userID]
 	if ok {
 		delete(r.storage, userID)
+		metrics.SetInMemoryObjects(len(r.storage))
 	}
 
 	return nil

cart/internal/domain/service/service.go

@@ -7,6 +7,7 @@ import (
 
 	"route256/cart/internal/domain/entity"
 	"route256/cart/internal/domain/model"
+	"route256/cart/internal/infra/tracing"
 )
 
 //go:generate minimock -i Repository -o ./mock -s _mock.go
@@ -43,6 +44,9 @@ func NewCartService(repository Repository, productService ProductService, lomsSe
 }
 
 func (s *CartService) AddItem(ctx context.Context, userID entity.UID, item *model.Item) error {
+	ctx, span := tracing.Tracer().Start(ctx, "AddItem")
+	defer span.End()
+
 	if err := item.Validate(); err != nil {
 		return fmt.Errorf("invalid requested values: %w", err)
 	}
@@ -76,6 +80,9 @@ func (s *CartService) AddItem(ctx context.Context, userID entity.UID, item *mode
 // and return a list of the collected items.
 // In case of failed request to product-service, return nothing and error.
 func (s *CartService) GetItemsByUserID(ctx context.Context, userID entity.UID) (*model.Cart, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "GetItemsByUserID")
+	defer span.End()
+
 	if userID <= 0 {
 		return nil, fmt.Errorf("userID invalid")
 	}
@@ -122,6 +129,9 @@ func (s *CartService) GetItemsByUserID(ctx context.Context, userID entity.UID) (
 }
 
 func (s *CartService) DeleteItem(ctx context.Context, userID entity.UID, sku entity.Sku) error {
+	ctx, span := tracing.Tracer().Start(ctx, "DeleteItem")
+	defer span.End()
+
 	if userID <= 0 {
 		return fmt.Errorf("userID invalid")
 	}
@@ -138,6 +148,9 @@ func (s *CartService) DeleteItem(ctx context.Context, userID entity.UID, sku ent
 }
 
 func (s *CartService) DeleteItemsByUserID(ctx context.Context, userID entity.UID) error {
+	ctx, span := tracing.Tracer().Start(ctx, "DeleteItemsByUserID")
+	defer span.End()
+
 	if userID <= 0 {
 		return fmt.Errorf("userID invalid")
 	}
@@ -150,6 +163,9 @@ func (s *CartService) DeleteItemsByUserID(ctx context.Context, userID entity.UID
 }
 
 func (s *CartService) CheckoutUserCart(ctx context.Context, userID entity.UID) (int64, error) {
+	ctx, span := tracing.Tracer().Start(ctx, "CheckoutUserCart")
+	defer span.End()
+
 	if userID <= 0 {
 		return 0, fmt.Errorf("userID invalid")
 	}

cart/internal/domain/service/service_test.go

@@ -119,7 +119,8 @@ func TestCartService_AddItem(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					AddItemMock.
-					Expect(ctx, 1337, &testItem).
+					ExpectUserIDParam2(1337).
+					ExpectItemParam3(&testItem).
 					Return(nil),
 				productService: &productServiceFake{},
 				lomsService:    &lomsServiceFake{},
@@ -190,7 +191,8 @@ func TestCartService_AddItem(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					AddItemMock.
-					Expect(ctx, 1337, &testItem).
+					ExpectUserIDParam2(1337).
+					ExpectItemParam3(&testItem).
 					Return(assert.AnError),
 				productService: &productServiceFake{},
 				lomsService:    &lomsServiceFake{},
@@ -304,7 +306,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(testEntityCart, nil),
 				productService: &productServiceFake{},
 			},
@@ -320,7 +322,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{
 						UserID:    testUID,
 						Items:     []entity.Sku{testSKU},
@@ -353,7 +355,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{
 						UserID:    testUID,
 						Items:     []entity.Sku{testSKU, 1},
@@ -407,7 +409,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{}, assert.AnError),
 				productService: nil,
 			},
@@ -423,7 +425,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{}, nil),
 				productService: nil,
 			},
@@ -439,7 +441,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{
 						UserID:    testUID,
 						Items:     []entity.Sku{2},
@@ -459,7 +461,7 @@ func TestCartService_GetItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(entity.Cart{
 						UserID:    testUID,
 						Items:     []entity.Sku{2},
@@ -522,7 +524,8 @@ func TestCartService_DeleteItem(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					DeleteItemMock.
-					Expect(ctx, testUID, testSKU).
+					ExpectUserIDParam2(testUID).
+					ExpectSkuParam3(testSKU).
 					Return(nil),
 			},
 			args: args{
@@ -561,7 +564,8 @@ func TestCartService_DeleteItem(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					DeleteItemMock.
-					Expect(ctx, testUID, testSKU).
+					ExpectUserIDParam2(testUID).
+					ExpectSkuParam3(testSKU).
 					Return(assert.AnError),
 			},
 			args: args{
@@ -613,7 +617,7 @@ func TestCartService_DeleteItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					DeleteItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(nil),
 			},
 			args: args{
@@ -638,7 +642,7 @@ func TestCartService_DeleteItemsByUserID(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					DeleteItemsByUserIDMock.
-					Expect(ctx, testUID).
+					ExpectUserIDParam2(testUID).
 					Return(assert.AnError),
 			},
 			args: args{
@@ -697,10 +701,10 @@ func TestCartService_CheckoutUserCart(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, 1337).
+					ExpectUserIDParam2(1337).
 					Return(testCart, nil).
 					DeleteItemsByUserIDMock.
-					Expect(ctx, 1337).
+					ExpectUserIDParam2(1337).
 					Return(nil),
 				productService: &productServiceFake{},
 				lomsService:    &lomsServiceFake{},
@@ -720,7 +724,7 @@ func TestCartService_CheckoutUserCart(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, 1337).
+					ExpectUserIDParam2(1337).
 					Return(entity.Cart{}, assert.AnError),
 			},
 			args:    args{ctx: ctx, userID: 1337},
@@ -731,7 +735,7 @@ func TestCartService_CheckoutUserCart(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, 1111).
+					ExpectUserIDParam2(1111).
 					Return(testCart, nil),
 				productService: &productServiceFake{},
 				lomsService:    &lomsServiceFake{},
@@ -744,10 +748,10 @@ func TestCartService_CheckoutUserCart(t *testing.T) {
 			fields: fields{
 				repository: mock.NewRepositoryMock(mc).
 					GetItemsByUserIDMock.
-					Expect(ctx, 1337).
+					ExpectUserIDParam2(1337).
 					Return(testCart, nil).
 					DeleteItemsByUserIDMock.
-					Expect(ctx, 1337).
+					ExpectUserIDParam2(1337).
 					Return(assert.AnError),
 				productService: &productServiceFake{},
 				lomsService:    &lomsServiceFake{},

cart/internal/infra/http/metrics/metrics.go

@@ -0,0 +1,52 @@
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	requestCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "app",
+		Name:      "requests_total",
+		Help:      "Total amount of request by handler",
+	}, []string{"method", "path", "status_code"})
+
+	requestDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "app",
+		Name:      "request_duration_seconds",
+		Help:      "Latency of handler processing, seconds",
+		Buckets:   prometheus.DefBuckets,
+	}, []string{"method", "path", "status_code"})
+
+	outboundCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "app",
+		Name:      "outbound_requests_total",
+		Help:      "Total HTTP requests to external services",
+	}, []string{"method", "url", "status_code"})
+
+	outboundDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "app",
+		Name:      "outbound_request_duration_seconds",
+		Help:      "Latency of outbound HTTP requests, seconds",
+		Buckets:   prometheus.DefBuckets,
+	}, []string{"method", "url", "status_code"})
+)
+
+func IncRequestHandlerCount(method, path, statusCode string) {
+	requestCounter.WithLabelValues(method, path, statusCode).Inc()
+}
+
+func StoreHandlerRequestDuration(method, path, statusCode string, d time.Duration) {
+	requestDurationHistogram.WithLabelValues(method, path, statusCode).Observe(d.Seconds())
+}
+
+func IncOutboundRequestCount(method, url, status string) {
+	outboundCounter.WithLabelValues(method, url, status).Inc()
+}
+
+func StoreOutboundRequestDuration(method, url, status string, d time.Duration) {
+	outboundDurationHistogram.WithLabelValues(method, url, status).Observe(d.Seconds())
+}

cart/internal/infra/http/metrics/repository.go

@@ -0,0 +1,16 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var inMemoryObjectsGauge = promauto.NewGauge(prometheus.GaugeOpts{
+	Namespace: "app",
+	Name:      "inmemory_repo_objects",
+	Help:      "Current in-memory repository size",
+})
+
+func SetInMemoryObjects(n int) {
+	inMemoryObjectsGauge.Set(float64(n))
+}

cart/internal/infra/http/middlewares/metrics.go

@@ -0,0 +1,38 @@
+package middlewares
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"route256/cart/internal/infra/http/metrics"
+)
+
+type statusWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func (w *statusWriter) WriteHeader(code int) {
+	w.statusCode = code
+	w.ResponseWriter.WriteHeader(code)
+}
+
+func NewMetricsMiddleware(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sw := &statusWriter{
+			ResponseWriter: w,
+			statusCode:     http.StatusOK,
+		}
+
+		start := time.Now()
+
+		h.ServeHTTP(sw, r)
+
+		path := r.URL.Path
+		status := strconv.Itoa(sw.statusCode)
+
+		metrics.IncRequestHandlerCount(r.Method, path, status)
+		metrics.StoreHandlerRequestDuration(r.Method, path, status, time.Since(start))
+	})
+}

cart/internal/infra/http/middlewares/tracing.go

@@ -0,0 +1,17 @@
+package middlewares
+
+import (
+	"net/http"
+
+	"route256/cart/internal/infra/tracing"
+)
+
+func NewTracingMiddleware(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx, span := tracing.Tracer().Start(r.Context(),
+			r.Method+" "+r.URL.Path)
+		defer span.End()
+
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}

cart/internal/infra/http/round_trippers/metrics.go

@@ -0,0 +1,37 @@
+package round_trippers
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"route256/cart/internal/infra/http/metrics"
+)
+
+type MetricsRoundTripper struct {
+	rt http.RoundTripper
+}
+
+func NewMetricsRoundTripper(rt http.RoundTripper) http.RoundTripper {
+	return &MetricsRoundTripper{
+		rt: rt,
+	}
+}
+
+func (m *MetricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	start := time.Now()
+	resp, err := m.rt.RoundTrip(r)
+
+	status := "error"
+
+	if resp != nil {
+		status = strconv.Itoa(resp.StatusCode)
+	}
+
+	url := r.URL.Path
+
+	metrics.IncOutboundRequestCount(r.Method, url, status)
+	metrics.StoreOutboundRequestDuration(r.Method, url, status, time.Since(start))
+
+	return resp, err
+}

cart/internal/infra/tracing/init.go

@@ -0,0 +1,48 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+var (
+	globalTracer oteltrace.Tracer
+	provider     *trace.TracerProvider
+)
+
+func NewTracer(serviceName string, option ...trace.TracerProviderOption) oteltrace.Tracer {
+	option = append([]trace.TracerProviderOption{
+		trace.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(serviceName),
+		)),
+	}, option...)
+
+	provider = trace.NewTracerProvider(option...)
+
+	otel.SetTracerProvider(provider)
+
+	globalTracer = otel.GetTracerProvider().Tracer("cart")
+
+	return globalTracer
+}
+
+func Shutdown(ctx context.Context) error {
+	if provider != nil {
+		return provider.Shutdown(ctx)
+	}
+	return nil
+}
+
+func Tracer() oteltrace.Tracer {
+	if globalTracer == nil {
+		return otel.Tracer("cart")
+	}
+
+	return globalTracer
+}

cart/internal/infra/tracing/otel.go

@@ -0,0 +1,22 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func InitOTLP(ctx context.Context, serviceName, endpoint string) (shutdown func(context.Context) error, err error) {
+	exp, err := otlptracehttp.New(ctx,
+		otlptracehttp.WithEndpoint(endpoint),
+		otlptracehttp.WithInsecure(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	NewTracer(serviceName, sdktrace.WithBatcher(exp))
+
+	return Shutdown, nil
+}