foomo/squadron
1
0
Fork 0
mirror of https://github.com/foomo/squadron.git synced 2025-10-16 12:35:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: fix race conditions

Browse Source
This commit is contained in:
Kevin Franklin Kim 2023-09-27 10:02:55 +02:00
parent 7308826652
commit 80a9de238b
No known key found for this signature in database
15 changed files with 214 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
_examples/common/charts/backend/templates/deployment.yaml
View File

@ -7,8 +7,8 @@ metadata:
app.kubernetes.io/component: {{ .Chart.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
namespace: {{ .Release.Namespace }}
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: {{ .Release.Name }}

1
_examples/common/charts/backend/templates/service.yaml
View File

@ -7,6 +7,7 @@ metadata:
app.kubernetes.io/component: {{ .Chart.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
namespace: {{ .Release.Namespace }}
spec:
type: ClusterIP
selector:

2
_examples/common/charts/frontend/templates/deployment.yaml
View File

@ -7,8 +7,8 @@ metadata:
app.kubernetes.io/component: {{ .Chart.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
namespace: {{ .Release.Namespace }}
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: {{ .Release.Name }}

26
_examples/common/charts/frontend/templates/ingress.yaml Normal file
View File

@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ .Release.Name }}
labels:
app.kubernetes.io/name: {{ .Release.Name }}
app.kubernetes.io/component: {{ .Chart.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
namespace: {{ .Release.Namespace }}
spec:
tls:
- hosts: [ "foo.com" ]
secretName: foo-com-cert
rules:
- host: foo.com
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: {{ $.Release.Name }}
port:
name: http
number: 80

1
_examples/common/charts/frontend/templates/service.yaml
View File

@ -7,6 +7,7 @@ metadata:
app.kubernetes.io/component: {{ .Chart.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
namespace: {{ .Release.Namespace }}
spec:
type: ClusterIP
selector:

3
cmd/actions/template.go
View File

@ -9,6 +9,7 @@ import (
)
func init() {
templateCmd.Flags().IntVar(&flagParallel, "parallel", 1, "run command in parallel")
templateCmd.Flags().StringVarP(&flagNamespace, "namespace", "n", "default", "set the namespace name or template (default, squadron-{{.Squadron}}-{{.Unit}})")
templateCmd.Flags().StringSliceVar(&flagTags, "tags", nil, "list of tags to include or exclude (can specify multiple or separate values with commas: tag1,tag2,-tag3)")
}
@ -36,7 +37,7 @@ var templateCmd = &cobra.Command{
return err
}
out, err := sq.Template(cmd.Context(), helmArgs)
out, err := sq.Template(cmd.Context(), helmArgs, flagParallel)
if err != nil {
return err
}

3
go.mod
View File

@ -16,6 +16,7 @@ require (
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.28.2
k8s.io/apimachinery v0.28.2
)
require (
@ -54,11 +55,11 @@ require (
golang.org/x/term v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/apimachinery v0.28.2 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace github.com/miracl/conflate v1.2.1 => github.com/runz0rd/conflate v1.2.2-0.20210920145208-fa48576ef06d

5
go.sum
View File

@ -40,6 +40,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -163,6 +165,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
@ -205,6 +208,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

31
internal/config/unit.go
View File

@ -1,8 +1,11 @@
package config
import (
"bytes"
"context"
"fmt"
"os"
"path"
"strings"
"github.com/foomo/squadron/internal/helm"
@ -64,6 +67,34 @@ func (u *Unit) Push(ctx context.Context, squadron, unit string, args []string) (
return "", nil
}
func (u *Unit) Template(ctx context.Context, squadron, unit, namespace string, global map[string]interface{}, helmArgs []string) ([]byte, error) {
var ret bytes.Buffer
valueBytes, err := u.ValuesYAML(global)
if err != nil {
return nil, err
}
cmd := util.NewHelmCommand().Args("template", unit).
Stdin(bytes.NewReader(valueBytes)).
Stdout(&ret).
Args("--dependency-update").
Args("--namespace", namespace).
Args("--set", fmt.Sprintf("squadron=%s", squadron)).
Args("--set", fmt.Sprintf("unit=%s", unit)).
Args("--values", "-").
Args(helmArgs...)
if strings.HasPrefix(u.Chart.Repository, "file://") {
cmd.Args(path.Clean(strings.TrimPrefix(u.Chart.Repository, "file://")))
} else {
cmd.Args(u.Chart.Name, "--repo", u.Chart.Repository, "--version", u.Chart.Version)
}
if out, err := cmd.Run(ctx); err != nil {
return nil, errors.Wrap(err, out)
}
return ret.Bytes(), nil
}
func (u *Unit) DependencyUpdate(ctx context.Context) error {
// update local chart dependencies
// https://stackoverflow.com/questions/59210148/error-found-in-chart-yaml-but-missing-in-charts-directory-mysql

88
squadron.go
View File

@ -9,6 +9,7 @@ import (
"os/exec"
"slices"
"strings"
"sync"
"text/template"
"github.com/foomo/squadron/internal/config"
@ -49,6 +50,9 @@ func New(basePath, namespace string, files []string) *Squadron {
// ------------------------------------------------------------------------------------------------
func (sq *Squadron) Namespace(ctx context.Context, squadron, unit string) (string, error) {
if sq.namespace == "" {
return "default", nil
}
var out bytes.Buffer
t, err := template.New("namespace").Parse(sq.namespace)
if err != nil {
@ -317,7 +321,14 @@ func (sq *Squadron) Down(ctx context.Context, helmArgs []string, parallel int) e
}
func (sq *Squadron) Diff(ctx context.Context, helmArgs []string, parallel int) error {
var diff string
var m sync.Mutex
var ret string
write := func(b string) {
m.Lock()
defer m.Unlock()
ret += b
}
wg, ctx := errgroup.WithContext(ctx)
wg.SetLimit(parallel)
@ -360,7 +371,7 @@ func (sq *Squadron) Diff(ctx context.Context, helmArgs []string, parallel int) e
}
dmp := diffmatchpatch.New()
diff += dmp.DiffPrettyText(dmp.DiffMain(string(manifest), string(out), false))
write(dmp.DiffPrettyText(dmp.DiffMain(string(manifest), string(out), false)))
return nil
})
return nil
@ -371,16 +382,21 @@ func (sq *Squadron) Diff(ctx context.Context, helmArgs []string, parallel int) e
return err
}
fmt.Println(diff)
fmt.Println(ret)
return nil
}
func (sq *Squadron) Status(ctx context.Context, helmArgs []string, parallel int) error {
var m sync.Mutex
tbd := pterm.TableData{
{"Name", "Revision", "Status", "Deployed by", "Commit", "Branch", "Last deployed", "Notes"},
}
write := func(b []string) {
m.Lock()
defer m.Unlock()
tbd = append(tbd, b)
}
type statusType struct {
Name string `json:"name"`
Version int `json:"version"`
@ -435,7 +451,7 @@ func (sq *Squadron) Status(ctx context.Context, helmArgs []string, parallel int)
notes = append(notes, line)
}
}
tbd = append(tbd, []string{
write([]string{
status.Name,
fmt.Sprintf("%d", status.Version),
status.Info.Status,
@ -556,48 +572,42 @@ func (sq *Squadron) Up(ctx context.Context, helmArgs []string, username, version
return wg.Wait()
}
func (sq *Squadron) Template(ctx context.Context, helmArgs []string) (string, error) {
func (sq *Squadron) Template(ctx context.Context, helmArgs []string, parallel int) (string, error) {
var m sync.Mutex
var ret bytes.Buffer
write := func(b []byte) error {
m.Lock()
defer m.Unlock()
_, err := ret.Write(b)
return err
}
err := sq.Config().Squadrons.Iterate(func(key string, value config.Map[*config.Unit]) error {
wg, gctx := errgroup.WithContext(ctx)
wg.SetLimit(parallel)
_ = sq.Config().Squadrons.Iterate(func(key string, value config.Map[*config.Unit]) error {
return value.Iterate(func(k string, v *config.Unit) error {
name := fmt.Sprintf("%s-%s", key, k)
namespace, err := sq.Namespace(ctx, key, k)
if err != nil {
return err
}
valueBytes, err := v.ValuesYAML(sq.c.Global)
if err != nil {
return err
}
pterm.Debug.Printfln("running helm template for chart: %s", name)
cmd := util.NewHelmCommand().Args("template", k).
Stdin(bytes.NewReader(valueBytes)).
Stdout(&ret).
Args("--dependency-update").
Args("--namespace", namespace).
Args("--set", fmt.Sprintf("squadron=%s", key)).
Args("--set", fmt.Sprintf("unit=%s", k)).
Args("--values", "-").
Args(helmArgs...)
if strings.Contains(v.Chart.Repository, "file://") {
file := strings.TrimPrefix(v.Chart.Repository, "file://")
if !strings.HasPrefix(file, ".") {
file = "/" + file
wg.Go(func() error {
name := fmt.Sprintf("%s-%s", key, k)
namespace, err := sq.Namespace(ctx, key, k)
if err != nil {
return err
}
cmd.Args(file)
} else {
cmd.Args(v.Chart.Name, "--repo", v.Chart.Repository, "--version", v.Chart.Version)
}
if out, err := cmd.Run(ctx); err != nil {
return errors.Wrap(err, out)
}
pterm.Debug.Printfln("running helm template for chart: %s", name)
out, err := v.Template(gctx, key, k, namespace, sq.c.Global, helmArgs)
if err != nil {
return err
}
return write(out)
})
return nil
})
})
if err != nil {
if err := wg.Wait(); err != nil {
return "", err
}

2
squadron_test.go
View File

@ -89,7 +89,7 @@ func config(t *testing.T, name string, files []string, squadronName string, unit
}
{
out, err := sq.Template(ctx, nil)
out, err := sq.Template(ctx, nil, 1)
require.NoError(t, err, "failed to render template")
testutils.Snapshot(t, path.Join("testdata", name, "snapshop-template.yaml"), out)
}

31
testdata/override/snapshop-template.yaml vendored
View File

@ -9,6 +9,7 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
type: ClusterIP
selector:
@ -28,8 +29,8 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: frontend
@ -47,3 +48,31 @@ spec:
- name: http
protocol: TCP
containerPort: 80
---
# Source: frontend/templates/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: frontend
labels:
app.kubernetes.io/name: frontend
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
tls:
- hosts: [ "foo.com" ]
secretName: foo-com-cert
rules:
- host: foo.com
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: frontend
port:
name: http
number: 80

31
testdata/simple/snapshop-template.yaml vendored
View File

@ -9,6 +9,7 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
type: ClusterIP
selector:
@ -28,8 +29,8 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: frontend
@ -47,3 +48,31 @@ spec:
- name: http
protocol: TCP
containerPort: 80
---
# Source: frontend/templates/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: frontend
labels:
app.kubernetes.io/name: frontend
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
tls:
- hosts: [ "foo.com" ]
secretName: foo-com-cert
rules:
- host: foo.com
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: frontend
port:
name: http
number: 80

3
testdata/tags/snapshop-template.yaml vendored
View File

@ -9,6 +9,7 @@ metadata:
app.kubernetes.io/component: backend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: backend-0.0.1
namespace: default
spec:
type: ClusterIP
selector:
@ -28,8 +29,8 @@ metadata:
app.kubernetes.io/component: backend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: backend-0.0.1
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: backend

34
testdata/template/snapshop-template.yaml vendored
View File

@ -9,6 +9,7 @@ metadata:
app.kubernetes.io/component: backend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: backend-0.0.1
namespace: default
spec:
type: ClusterIP
selector:
@ -28,8 +29,8 @@ metadata:
app.kubernetes.io/component: backend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: backend-0.0.1
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: backend
@ -58,6 +59,7 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
type: ClusterIP
selector:
@ -77,8 +79,8 @@ metadata:
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: frontend
@ -96,3 +98,31 @@ spec:
- name: http
protocol: TCP
containerPort: 80
---
# Source: frontend/templates/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: frontend
labels:
app.kubernetes.io/name: frontend
app.kubernetes.io/component: frontend
app.kubernetes.io/managed-by: Helm
helm.sh/chart: frontend-0.0.1
namespace: default
spec:
tls:
- hosts: [ "foo.com" ]
secretName: foo-com-cert
rules:
- host: foo.com
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: frontend
port:
name: http
number: 80