mirror of
https://github.com/foomo/sesamy-go.git
synced 2025-10-16 12:35:43 +00:00
feat: add loki
This commit is contained in:
Kevin Franklin Kim
parent
31f2ea9f8f
commit
ec7166f059
128
go.mod
128
go.mod
@ -6,21 +6,143 @@ require (
|
||||
github.com/ThreeDotsLabs/watermill v1.3.5
|
||||
github.com/foomo/go v0.0.3
|
||||
github.com/foomo/gostandards v0.1.0
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang/snappy v0.0.4
|
||||
github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3
|
||||
github.com/grafana/loki/v3 v3.0.0
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/common v0.49.1-0.20240306132007-4199f18c3e92
|
||||
github.com/stretchr/testify v1.9.0
|
||||
go.uber.org/zap v1.27.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.2.0 // indirect
|
||||
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/aws/aws-sdk-go v1.50.32 // indirect
|
||||
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dennwc/varint v1.0.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/edsrzf/mmap-go v1.1.0 // indirect
|
||||
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
|
||||
github.com/fatih/color v1.15.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.6.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5 // indirect
|
||||
github.com/gogo/googleapis v1.4.0 // indirect
|
||||
github.com/gogo/status v1.1.1 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect
|
||||
github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d // indirect
|
||||
github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect
|
||||
github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
|
||||
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
|
||||
github.com/hashicorp/consul/api v1.28.2 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.5.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.6 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/hashicorp/memberlist v0.5.0 // indirect
|
||||
github.com/hashicorp/serf v0.10.1 // indirect
|
||||
github.com/huandu/xstrings v1.3.3 // indirect
|
||||
github.com/imdario/mergo v0.3.16 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/backoff v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.17.7 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/miekg/dns v1.1.58 // indirect
|
||||
github.com/mitchellh/copystructure v1.0.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
|
||||
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pires/go-proxyproto v0.7.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.19.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.0 // indirect
|
||||
github.com/prometheus/common/sigv4 v0.1.0 // indirect
|
||||
github.com/prometheus/exporter-toolkit v0.11.0 // indirect
|
||||
github.com/prometheus/procfs v0.12.0 // indirect
|
||||
github.com/prometheus/prometheus v0.51.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
|
||||
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
|
||||
github.com/shopspring/decimal v1.2.0 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/spf13/cast v1.3.1 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.4 // indirect
|
||||
go.opentelemetry.io/otel v1.24.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.24.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.24.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/goleak v1.3.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect
|
||||
golang.org/x/crypto v0.21.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
|
||||
golang.org/x/mod v0.16.0 // indirect
|
||||
golang.org/x/net v0.22.0 // indirect
|
||||
golang.org/x/oauth2 v0.18.0 // indirect
|
||||
golang.org/x/sync v0.6.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.19.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
|
||||
google.golang.org/grpc v1.62.1 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/apimachinery v0.29.2 // indirect
|
||||
k8s.io/client-go v0.29.2 // indirect
|
||||
k8s.io/klog/v2 v2.120.1 // indirect
|
||||
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
|
||||
)
|
||||
|
||||
22
integration/loki/line.go
Normal file
22
integration/loki/line.go
Normal file
@ -0,0 +1,22 @@
|
||||
package loki
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/foomo/sesamy-go/pkg/encoding/mpv2"
|
||||
)
|
||||
|
||||
type Line struct {
|
||||
Params any `json:"params,omitempty"`
|
||||
ClientID string `json:"client_id,omitempty"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
UserProperties map[string]any `json:"user_properties,omitempty"`
|
||||
Consent *mpv2.Consent `json:"consent,omitempty"`
|
||||
NonPersonalizedAds bool `json:"non_personalized_ads,omitempty"`
|
||||
UserData *mpv2.UserData `json:"user_data,omitempty"`
|
||||
DebugMode bool `json:"debug_mode,omitempty"`
|
||||
}
|
||||
|
||||
func (l *Line) Marshal() ([]byte, error) {
|
||||
return json.Marshal(l)
|
||||
}
|
||||
253
integration/loki/loki.go
Normal file
253
integration/loki/loki.go
Normal file
@ -0,0 +1,253 @@
|
||||
package loki
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/sesamy-go/pkg/encoding/mpv2"
|
||||
"github.com/foomo/sesamy-go/pkg/utils"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/golang/snappy"
|
||||
"github.com/grafana/dskit/backoff"
|
||||
"github.com/grafana/loki/pkg/push"
|
||||
"github.com/grafana/loki/v3/pkg/logproto"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/common/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
pushEndpoint = "/loki/api/v1/push"
|
||||
defaultContentType = "application/x-protobuf"
|
||||
defaultMaxReponseBufferLen = 1024
|
||||
)
|
||||
|
||||
// Loki is a io.Writer, that writes given log entries by pushing
|
||||
// directly to the given loki server URL. Each `Push` instance handles for a single tenant.
|
||||
// No batching of log lines happens when sending to Loki.
|
||||
type (
|
||||
Loki struct {
|
||||
l *zap.Logger
|
||||
endpoint string
|
||||
|
||||
// channel for incoming logs
|
||||
entries chan logproto.Entry
|
||||
batchSize int
|
||||
bufferSize int
|
||||
|
||||
// shutdown channels
|
||||
cancel context.CancelFunc
|
||||
|
||||
userAgent string
|
||||
httpClient *http.Client
|
||||
backoff *backoff.Config
|
||||
}
|
||||
Option func(*Loki)
|
||||
)
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// ~ Options
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
func WithHTTPClient(v *http.Client) Option {
|
||||
return func(l *Loki) {
|
||||
l.httpClient = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithBackoffConfig(v *backoff.Config) Option {
|
||||
return func(l *Loki) {
|
||||
l.backoff = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithUserAgent(v string) Option {
|
||||
return func(l *Loki) {
|
||||
l.userAgent = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithBatchSize(v int) Option {
|
||||
return func(l *Loki) {
|
||||
l.batchSize = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithBufferSize(v int) Option {
|
||||
return func(l *Loki) {
|
||||
l.bufferSize = v
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// ~ Constructor
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
// New creates an instance of `Push` which writes logs directly to given `lokiAddr`
|
||||
func New(l *zap.Logger, addr string, opts ...Option) *Loki {
|
||||
inst := &Loki{
|
||||
l: l,
|
||||
endpoint: addr + pushEndpoint,
|
||||
httpClient: http.DefaultClient,
|
||||
userAgent: "sesamy",
|
||||
batchSize: 10,
|
||||
bufferSize: 50,
|
||||
backoff: &backoff.Config{
|
||||
MinBackoff: 500 * time.Millisecond,
|
||||
MaxBackoff: 5 * time.Minute,
|
||||
MaxRetries: 10,
|
||||
},
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(inst)
|
||||
}
|
||||
}
|
||||
|
||||
inst.entries = make(chan logproto.Entry, inst.bufferSize) // Use a buffered channel so we can retry failed pushes without blocking WriteEntry
|
||||
|
||||
return inst
|
||||
}
|
||||
|
||||
// Start pulls lines out of the channel and sends them to Loki
|
||||
func (p *Loki) Start(ctx context.Context) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p.cancel = cancel
|
||||
utils.Batch(ctx, p.entries, p.batchSize, p.process)
|
||||
}
|
||||
|
||||
func (p *Loki) Write(ts time.Time, payload mpv2.Payload[any]) {
|
||||
var metadata push.LabelsAdapter
|
||||
if payload.UserID != "" {
|
||||
metadata = append(metadata, push.LabelAdapter{
|
||||
Name: "user_id",
|
||||
Value: payload.UserID,
|
||||
})
|
||||
}
|
||||
for _, event := range payload.Events {
|
||||
metadata := append(metadata, push.LabelAdapter{
|
||||
Name: "name",
|
||||
Value: event.Name.String(),
|
||||
})
|
||||
|
||||
line := Line{
|
||||
Params: event.Params,
|
||||
ClientID: payload.ClientID,
|
||||
UserID: payload.UserID,
|
||||
UserProperties: payload.UserProperties,
|
||||
Consent: payload.Consent,
|
||||
NonPersonalizedAds: payload.NonPersonalizedAds,
|
||||
UserData: payload.UserData,
|
||||
DebugMode: payload.DebugMode,
|
||||
}
|
||||
|
||||
lineBytes, err := line.Marshal()
|
||||
if err != nil {
|
||||
p.l.Warn("failed to marshal line", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
p.entries <- logproto.Entry{
|
||||
Line: string(lineBytes),
|
||||
Timestamp: time.UnixMicro(payload.TimestampMicros),
|
||||
StructuredMetadata: metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop will cancel any ongoing requests and stop the goroutine listening for requests
|
||||
func (p *Loki) Stop() {
|
||||
if p.cancel != nil {
|
||||
p.cancel()
|
||||
p.cancel = nil
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// ~ Private methods
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
func (p *Loki) process(entries []logproto.Entry) {
|
||||
labels := model.LabelSet{
|
||||
"name": "events",
|
||||
"stream": "sesamy",
|
||||
}
|
||||
request, err := proto.Marshal(&logproto.PushRequest{
|
||||
Streams: []logproto.Stream{
|
||||
{
|
||||
Labels: labels.String(),
|
||||
Entries: entries,
|
||||
Hash: uint64(labels.Fingerprint()),
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
p.l.Error("failed to marshal payload to json", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
payload := snappy.Encode(nil, request)
|
||||
|
||||
// We will use a timeout within each attempt to send
|
||||
back := backoff.New(context.Background(), *p.backoff)
|
||||
|
||||
// send log with retry
|
||||
for {
|
||||
var status int
|
||||
status, err = p.send(context.Background(), payload)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
if status > 0 && status != 429 && status/100 != 5 {
|
||||
p.l.Error("failed to send entry, server rejected push with a non-retryable status code", zap.Error(err), zap.Int("status", status))
|
||||
break
|
||||
}
|
||||
|
||||
if !back.Ongoing() {
|
||||
p.l.Error("failed to send entry, retries exhausted, entry will be dropped", zap.Error(err), zap.Int("status", status))
|
||||
break
|
||||
}
|
||||
p.l.Warn("failed to send entry, retrying", zap.Error(err), zap.Int("status", status))
|
||||
back.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// send makes one attempt to send the payload to Loki
|
||||
func (p *Loki) send(ctx context.Context, payload []byte) (int, error) {
|
||||
// Set a timeout for the request
|
||||
ctx, cancel := context.WithTimeout(ctx, p.httpClient.Timeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return -1, errors.Wrap(err, "failed to create request")
|
||||
}
|
||||
req.Header.Set("Content-Type", defaultContentType)
|
||||
req.Header.Set("User-Agent", p.userAgent)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return -1, errors.Wrap(err, "failed to send payload")
|
||||
}
|
||||
status := resp.StatusCode
|
||||
if status/100 != 2 {
|
||||
scanner := bufio.NewScanner(io.LimitReader(resp.Body, defaultMaxReponseBufferLen))
|
||||
line := ""
|
||||
if scanner.Scan() {
|
||||
line = scanner.Text()
|
||||
}
|
||||
err = fmt.Errorf("server returned HTTP status %s (%d): %s", resp.Status, status, line)
|
||||
}
|
||||
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
p.l.Error("failed to close response body", zap.Error(err))
|
||||
}
|
||||
|
||||
return status, err
|
||||
}
|
||||
55
pkg/utils/batch.go
Normal file
55
pkg/utils/batch.go
Normal file
@ -0,0 +1,55 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// Batch reads from a channel and calls fn with a slice of batchSize.
|
||||
func Batch[T any](ctx context.Context, ch <-chan T, batchSize int, fn func([]T)) {
|
||||
if batchSize <= 1 { // sanity check,
|
||||
for v := range ch {
|
||||
fn([]T{v})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// batchSize > 1
|
||||
var batch = make([]T, 0, batchSize)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if len(batch) > 0 {
|
||||
fn(batch)
|
||||
}
|
||||
return
|
||||
case v, ok := <-ch:
|
||||
if !ok { // closed
|
||||
fn(batch)
|
||||
return
|
||||
}
|
||||
|
||||
batch = append(batch, v)
|
||||
if len(batch) == batchSize { // full
|
||||
fn(batch)
|
||||
batch = make([]T, 0, batchSize) // reset
|
||||
}
|
||||
default:
|
||||
if len(batch) > 0 { // partial
|
||||
fn(batch)
|
||||
batch = make([]T, 0, batchSize) // reset
|
||||
} else { // empty
|
||||
// wait for more
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case v, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
batch = append(batch, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user