foomo/contentserver
1
0
Fork 0
mirror of https://github.com/foomo/contentserver.git synced 2025-10-16 12:25:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added new metrics

Browse Source
This commit is contained in:
Philipp Mieden 2019-05-23 16:23:24 +02:00
parent 79d828bb23
commit 647853292b
7 changed files with 115 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -56,7 +56,7 @@ run-testserver:
bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json
run-contentserver:
contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level notice http://127.0.0.1:1234
contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234
clean-var:
rm var/contentserver-repo-2019*

6
repo/loader.go
View File

@ -9,6 +9,7 @@ import (
"github.com/foomo/contentserver/content"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/status"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
)
@ -172,8 +173,9 @@ func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) {
ur := <-c
return ur.repoRuntime, ur.jsonBytes, ur.err
default:
Log.Info("update request ignored, queue is full")
return 0, nil, errors.New("queue full")
Log.Info("update request rejected, queue is full")
status.M.UpdatesRejectedCounter.WithLabelValues().Inc()
return 0, nil, errors.New("update rejected: queue full")
}
}

11
repo/repo.go
View File

@ -6,6 +6,8 @@ import (
"strings"
"time"
"github.com/foomo/contentserver/status"
"github.com/foomo/contentserver/content"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/requests"
@ -62,13 +64,20 @@ func NewRepo(server string, varDir string) *Repo {
repoRuntime, jsonBytes, errUpdate := repo.update()
if errUpdate != nil {
status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc()
}
resChan <- updateResponse{
repoRuntime: repoRuntime,
jsonBytes: jsonBytes,
err: errUpdate,
}
Log.Info("update completed", zap.Duration("duration", time.Since(start)))
duration := time.Since(start)
Log.Info("update completed", zap.Duration("duration", duration))
status.M.UpdatesCompletedCounter.WithLabelValues().Inc()
status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds())
}
}
}()

19
server/handlerequest.go
View File

@ -10,7 +10,6 @@ import (
"github.com/foomo/contentserver/requests"
"github.com/foomo/contentserver/responses"
"github.com/foomo/contentserver/status"
"github.com/prometheus/client_golang/prometheus"
)
func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) {
@ -28,6 +27,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin
processingFunc()
}
)
status.M.ContentRequestCounter.WithLabelValues(source).Inc()
// handle and process
switch handler {
@ -59,7 +59,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin
default:
reply = responses.NewError(1, "unknown handler: "+string(handler))
}
addMetrics(metrics, handler, start, jsonErr, apiErr, source)
addMetrics(handler, start, jsonErr, apiErr, source)
// error handling
if jsonErr != nil {
@ -73,7 +73,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin
return encodeReply(reply)
}
func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, errJSON error, errAPI error, source string) {
func addMetrics(handlerName Handler, start time.Time, errJSON error, errAPI error, source string) {
var (
duration = time.Since(start)
@ -83,17 +83,8 @@ func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, e
s = "failed"
}
metrics.ServiceRequestCounter.With(prometheus.Labels{
status.MetricLabelHandler: string(handlerName),
status.MetricLabelStatus: s,
status.MetricLabelSource: source,
}).Inc()
metrics.ServiceRequestDuration.With(prometheus.Labels{
status.MetricLabelHandler: string(handlerName),
status.MetricLabelStatus: s,
status.MetricLabelSource: source,
}).Observe(float64(duration.Seconds()))
status.M.ServiceRequestCounter.WithLabelValues(string(handlerName), s, source).Inc()
status.M.ServiceRequestDuration.WithLabelValues(string(handlerName), s, source).Observe(float64(duration.Seconds()))
}
// encodeReply takes an interface and encodes it as JSON

4
server/server.go
View File

@ -9,14 +9,12 @@ import (
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo"
"github.com/foomo/contentserver/status"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
)
var (
json = jsoniter.ConfigCompatibleWithStandardLibrary
metrics = status.NewMetrics()
json = jsoniter.ConfigCompatibleWithStandardLibrary
)
// Handler type

4
server/socketserver.go
View File

@ -69,6 +69,7 @@ func (s *socketServer) writeResponse(conn net.Conn, reply []byte) {
func (s *socketServer) handleConnection(conn net.Conn) {
Log.Debug("socketServer.handleConnection")
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc()
var (
headerBuffer [1]byte
@ -82,6 +83,7 @@ func (s *socketServer) handleConnection(conn net.Conn) {
_, readErr := conn.Read(headerBuffer[0:])
if readErr != nil {
Log.Debug("looks like the client closed the connection", zap.Error(readErr))
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return
}
// read next byte
@ -121,6 +123,7 @@ func (s *socketServer) handleConnection(conn net.Conn) {
//@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong)
//@todo should we check for io.EOF here
Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr))
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return
}
jsonLengthCurrent += readLength
@ -138,6 +141,7 @@ func (s *socketServer) handleConnection(conn net.Conn) {
continue
}
Log.Error("can not read empty json")
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return
}
// adding to header byte by byte

102
status/metrics.go
View File

@ -4,22 +4,40 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
// M is the Metrics instance
var M = NewMetrics()
const (
MetricLabelHandler = "handler"
MetricLabelStatus = "status"
MetricLabelSource = "source"
namespace = "contentserver"
namespace = "contentserver"
metricLabelHandler = "handler"
metricLabelStatus = "status"
metricLabelSource = "source"
metricLabelRemote = "remote"
metricLabelError = "error"
)
type Metrics struct {
ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function
ServiceRequestDuration *prometheus.SummaryVec // count the duration of requests for each service function
ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function
ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function
UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates
UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates
UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error
UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call
ContentRequestCounter *prometheus.CounterVec // count the total number of content requests
NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets
}
func NewMetrics() *Metrics {
return &Metrics{
ServiceRequestCounter: serviceRequestCounter(),
ServiceRequestDuration: serviceRequestDuration(),
ServiceRequestCounter: serviceRequestCounter(),
ServiceRequestDuration: serviceRequestDuration(),
UpdatesRejectedCounter: updatesRejectedCounter(),
UpdatesCompletedCounter: updatesCompletedCounter(),
UpdatesFailedCounter: updatesFailedCounter(),
UpdateDuration: updateDuration(),
ContentRequestCounter: contentRequestCounter(),
NumSocketsGauge: numSocketsGauge(),
}
}
@ -28,8 +46,8 @@ func serviceRequestCounter() *prometheus.CounterVec {
prometheus.CounterOpts{
Namespace: namespace,
Name: "service_request_count",
Help: "count of requests per func",
}, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource})
Help: "Count of requests for each handler",
}, []string{metricLabelHandler, metricLabelStatus, metricLabelSource})
prometheus.MustRegister(vec)
return vec
}
@ -38,8 +56,68 @@ func serviceRequestDuration() *prometheus.SummaryVec {
vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: namespace,
Name: "service_request_duration_seconds",
Help: "seconds to unmarshal requests, execute a service function and marshal its reponses",
}, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource})
Help: "Seconds to unmarshal requests, execute a service function and marshal its reponses",
}, []string{metricLabelHandler, metricLabelStatus, metricLabelSource})
prometheus.MustRegister(vec)
return vec
}
func updatesRejectedCounter() *prometheus.CounterVec {
vec := prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "updates_rejected_count",
Help: "Number of updates that were rejected because the queue was full",
}, []string{})
prometheus.MustRegister(vec)
return vec
}
func updatesCompletedCounter() *prometheus.CounterVec {
vec := prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "updates_completed_count",
Help: "Number of updates that were successfully completed",
}, []string{})
prometheus.MustRegister(vec)
return vec
}
func updatesFailedCounter() *prometheus.CounterVec {
vec := prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "updates_failed_count",
Help: "Number of updates that failed due to an error",
}, []string{metricLabelError})
prometheus.MustRegister(vec)
return vec
}
func updateDuration() *prometheus.SummaryVec {
vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: namespace,
Name: "update_duration_seconds",
Help: "Duration in seconds for each successful repo.update() call",
}, []string{})
prometheus.MustRegister(vec)
return vec
}
func numSocketsGauge() *prometheus.GaugeVec {
vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "num_sockets_total",
Help: "Total number of currently open socket connections",
}, []string{metricLabelRemote})
prometheus.MustRegister(vec)
return vec
}
func contentRequestCounter() *prometheus.CounterVec {
vec := prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "content_request_count",
Help: "Number of requests for content",
}, []string{metricLabelSource})
prometheus.MustRegister(vec)
return vec
}