foomo/sesamy-go
1
0
Fork 0
mirror of https://github.com/foomo/sesamy-go.git synced 2025-10-16 12:35:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add loki

Browse Source
This commit is contained in:
Kevin Franklin Kim 2024-05-29 21:37:46 +02:00
parent 31f2ea9f8f
commit ec7166f059
No known key found for this signature in database
5 changed files with 1432 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
go.mod
View File

@ -6,21 +6,143 @@ require (
github.com/ThreeDotsLabs/watermill v1.3.5 github.com/ThreeDotsLabs/watermill v1.3.5
github.com/foomo/go v0.0.3 github.com/foomo/go v0.0.3
github.com/foomo/gostandards v0.1.0 github.com/foomo/gostandards v0.1.0
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3
github.com/grafana/loki/v3 v3.0.0
github.com/json-iterator/go v1.1.12 github.com/json-iterator/go v1.1.12
github.com/mitchellh/mapstructure v1.5.0 github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/common v0.49.1-0.20240306132007-4199f18c3e92
github.com/stretchr/testify v1.9.0 github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.27.0 go.uber.org/zap v1.27.0
) )
require ( require (
github.com/davecgh/go-spew v1.1.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go v1.50.32 // indirect
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dennwc/varint v1.0.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/gogo/status v1.1.1 // indirect
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect
github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d // indirect
github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
github.com/hashicorp/consul/api v1.28.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.5.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.6 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/miekg/dns v1.1.58 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/reflectwalk v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/oklog/ulid v1.3.1 // indirect github.com/oklog/ulid v1.3.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pires/go-proxyproto v0.7.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common/sigv4 v0.1.0 // indirect
github.com/prometheus/exporter-toolkit v0.11.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/prometheus v0.51.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/v3 v3.5.4 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.19.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
google.golang.org/grpc v1.62.1 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apimachinery v0.29.2 // indirect
k8s.io/client-go v0.29.2 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
) )

981
go.sum
View File

File diff suppressed because it is too large Load Diff

22
integration/loki/line.go Normal file
View File

@ -0,0 +1,22 @@
package loki
import (
"encoding/json"
"github.com/foomo/sesamy-go/pkg/encoding/mpv2"
)
type Line struct {
Params any `json:"params,omitempty"`
ClientID string `json:"client_id,omitempty"`
UserID string `json:"user_id,omitempty"`
UserProperties map[string]any `json:"user_properties,omitempty"`
Consent *mpv2.Consent `json:"consent,omitempty"`
NonPersonalizedAds bool `json:"non_personalized_ads,omitempty"`
UserData *mpv2.UserData `json:"user_data,omitempty"`
DebugMode bool `json:"debug_mode,omitempty"`
}
func (l *Line) Marshal() ([]byte, error) {
return json.Marshal(l)
}

253
integration/loki/loki.go Normal file
View File

@ -0,0 +1,253 @@
package loki
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/foomo/sesamy-go/pkg/encoding/mpv2"
"github.com/foomo/sesamy-go/pkg/utils"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/grafana/dskit/backoff"
"github.com/grafana/loki/pkg/push"
"github.com/grafana/loki/v3/pkg/logproto"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"go.uber.org/zap"
)
const (
pushEndpoint = "/loki/api/v1/push"
defaultContentType = "application/x-protobuf"
defaultMaxReponseBufferLen = 1024
)
// Loki is a io.Writer, that writes given log entries by pushing
// directly to the given loki server URL. Each `Push` instance handles for a single tenant.
// No batching of log lines happens when sending to Loki.
type (
Loki struct {
l *zap.Logger
endpoint string
// channel for incoming logs
entries chan logproto.Entry
batchSize int
bufferSize int
// shutdown channels
cancel context.CancelFunc
userAgent string
httpClient *http.Client
backoff *backoff.Config
}
Option func(*Loki)
)
// ------------------------------------------------------------------------------------------------
// ~ Options
// ------------------------------------------------------------------------------------------------
func WithHTTPClient(v *http.Client) Option {
return func(l *Loki) {
l.httpClient = v
}
}
func WithBackoffConfig(v *backoff.Config) Option {
return func(l *Loki) {
l.backoff = v
}
}
func WithUserAgent(v string) Option {
return func(l *Loki) {
l.userAgent = v
}
}
func WithBatchSize(v int) Option {
return func(l *Loki) {
l.batchSize = v
}
}
func WithBufferSize(v int) Option {
return func(l *Loki) {
l.bufferSize = v
}
}
// ------------------------------------------------------------------------------------------------
// ~ Constructor
// ------------------------------------------------------------------------------------------------
// New creates an instance of `Push` which writes logs directly to given `lokiAddr`
func New(l *zap.Logger, addr string, opts ...Option) *Loki {
inst := &Loki{
l: l,
endpoint: addr + pushEndpoint,
httpClient: http.DefaultClient,
userAgent: "sesamy",
batchSize: 10,
bufferSize: 50,
backoff: &backoff.Config{
MinBackoff: 500 * time.Millisecond,
MaxBackoff: 5 * time.Minute,
MaxRetries: 10,
},
}
for _, opt := range opts {
if opt != nil {
opt(inst)
}
}
inst.entries = make(chan logproto.Entry, inst.bufferSize) // Use a buffered channel so we can retry failed pushes without blocking WriteEntry
return inst
}
// Start pulls lines out of the channel and sends them to Loki
func (p *Loki) Start(ctx context.Context) {
ctx, cancel := context.WithCancel(ctx)
p.cancel = cancel
utils.Batch(ctx, p.entries, p.batchSize, p.process)
}
func (p *Loki) Write(ts time.Time, payload mpv2.Payload[any]) {
var metadata push.LabelsAdapter
if payload.UserID != "" {
metadata = append(metadata, push.LabelAdapter{
Name: "user_id",
Value: payload.UserID,
})
}
for _, event := range payload.Events {
metadata := append(metadata, push.LabelAdapter{
Name: "name",
Value: event.Name.String(),
})
line := Line{
Params: event.Params,
ClientID: payload.ClientID,
UserID: payload.UserID,
UserProperties: payload.UserProperties,
Consent: payload.Consent,
NonPersonalizedAds: payload.NonPersonalizedAds,
UserData: payload.UserData,
DebugMode: payload.DebugMode,
}
lineBytes, err := line.Marshal()
if err != nil {
p.l.Warn("failed to marshal line", zap.Error(err))
continue
}
p.entries <- logproto.Entry{
Line: string(lineBytes),
Timestamp: time.UnixMicro(payload.TimestampMicros),
StructuredMetadata: metadata,
}
}
}
// Stop will cancel any ongoing requests and stop the goroutine listening for requests
func (p *Loki) Stop() {
if p.cancel != nil {
p.cancel()
p.cancel = nil
}
}
// ------------------------------------------------------------------------------------------------
// ~ Private methods
// ------------------------------------------------------------------------------------------------
func (p *Loki) process(entries []logproto.Entry) {
labels := model.LabelSet{
"name": "events",
"stream": "sesamy",
}
request, err := proto.Marshal(&logproto.PushRequest{
Streams: []logproto.Stream{
{
Labels: labels.String(),
Entries: entries,
Hash: uint64(labels.Fingerprint()),
},
},
})
if err != nil {
p.l.Error("failed to marshal payload to json", zap.Error(err))
return
}
payload := snappy.Encode(nil, request)
// We will use a timeout within each attempt to send
back := backoff.New(context.Background(), *p.backoff)
// send log with retry
for {
var status int
status, err = p.send(context.Background(), payload)
if err == nil {
break
}
if status > 0 && status != 429 && status/100 != 5 {
p.l.Error("failed to send entry, server rejected push with a non-retryable status code", zap.Error(err), zap.Int("status", status))
break
}
if !back.Ongoing() {
p.l.Error("failed to send entry, retries exhausted, entry will be dropped", zap.Error(err), zap.Int("status", status))
break
}
p.l.Warn("failed to send entry, retrying", zap.Error(err), zap.Int("status", status))
back.Wait()
}
}
// send makes one attempt to send the payload to Loki
func (p *Loki) send(ctx context.Context, payload []byte) (int, error) {
// Set a timeout for the request
ctx, cancel := context.WithTimeout(ctx, p.httpClient.Timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.endpoint, bytes.NewReader(payload))
if err != nil {
return -1, errors.Wrap(err, "failed to create request")
}
req.Header.Set("Content-Type", defaultContentType)
req.Header.Set("User-Agent", p.userAgent)
resp, err := p.httpClient.Do(req)
if err != nil {
return -1, errors.Wrap(err, "failed to send payload")
}
status := resp.StatusCode
if status/100 != 2 {
scanner := bufio.NewScanner(io.LimitReader(resp.Body, defaultMaxReponseBufferLen))
line := ""
if scanner.Scan() {
line = scanner.Text()
}
err = fmt.Errorf("server returned HTTP status %s (%d): %s", resp.Status, status, line)
}
if err := resp.Body.Close(); err != nil {
p.l.Error("failed to close response body", zap.Error(err))
}
return status, err
}

55
pkg/utils/batch.go Normal file
View File

@ -0,0 +1,55 @@
package utils
import (
"context"
)
// Batch reads from a channel and calls fn with a slice of batchSize.
func Batch[T any](ctx context.Context, ch <-chan T, batchSize int, fn func([]T)) {
if batchSize <= 1 { // sanity check,
for v := range ch {
fn([]T{v})
}
return
}
// batchSize > 1
var batch = make([]T, 0, batchSize)
for {
select {
case <-ctx.Done():
if len(batch) > 0 {
fn(batch)
}
return
case v, ok := <-ch:
if !ok { // closed
fn(batch)
return
}
batch = append(batch, v)
if len(batch) == batchSize { // full
fn(batch)
batch = make([]T, 0, batchSize) // reset
}
default:
if len(batch) > 0 { // partial
fn(batch)
batch = make([]T, 0, batchSize) // reset
} else { // empty
// wait for more
select {
case <-ctx.Done():
return
case v, ok := <-ch:
if !ok {
return
}
batch = append(batch, v)
}
}
}
}
}