foomo/squadron
1
0
Fork 0
mirror of https://github.com/foomo/squadron.git synced 2025-10-16 12:35:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add builds for dependencies

Browse Source
This commit is contained in:
Kevin Franklin Kim 2023-09-25 10:04:00 +02:00
parent bede368045
commit 442323c93f
No known key found for this signature in database
22 changed files with 201 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
_examples/common/docker/backend-base.Dockerfile Normal file
View File

@ -0,0 +1 @@
FROM golang:alpine

6
_examples/common/docker/backend.Dockerfile
View File

@ -1,4 +1,6 @@
FROM golang:alpine as builder
ARG BASE_IMAGE=golang
ARG BASE_IMAGE_TAG=alpine
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as builder
ENV CGO_ENABLED=0
@ -8,7 +10,7 @@ WORKDIR /src
RUN go build -ldflags "-w -s" -trimpath -o /go/bin/service .
FROM alpine:latest as development
FROM alpine:latest
COPY --from=builder /go/bin/service /usr/local/bin/service

1
_examples/common/docker/frontend-base.Dockerfile Normal file
View File

@ -0,0 +1 @@
FROM nginx:latest

4
_examples/common/docker/frontend.Dockerfile
View File

@ -1,3 +1,5 @@
FROM nginx:latest
ARG BASE_IMAGE=nginx
ARG BASE_IMAGE_TAG=latest
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG}
COPY ./index.html /etc/nginx/templates/.

2
_examples/monorepo/Makefile
View File

@ -9,7 +9,7 @@ files=$(shell find . -name 'squadron.yaml' | tail -r | xargs echo -n | tr " ", "
.PHONY: list
## Show config
list:
@go run ../../cmd/main.go -f ${files} list
@go run ../../cmd/main.go -f ${files} list --builds
.PHONY: config
## Show config

4
_examples/monorepo/squadrons/checkout/backend/go.mod
View File

@ -1,3 +1,3 @@
module github.com/foomo/squadron/exmpale/monorepo/storefinder/backend
module github.com/foomo/squadron/exmpale/monorepo/checkout/backend
go 1.16
go 1.21

4
_examples/monorepo/squadrons/checkout/backend/squadron.yaml
View File

@ -12,6 +12,10 @@ squadron:
image: <% .Global.docker.registry %>/checkout-backend
context: <% env "PROJECT_ROOT" %>/squadrons/checkout/backend
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/backend.Dockerfile
dependencies: [ "backend-base" ]
args:
- "BASE_IMAGE=<% .Global.docker.registry %>/backend-base"
- "BASE_IMAGE_TAG=<% .Global.docker.tag | quote %>"
values:
image:
tag: <% .Global.docker.tag | quote %>

2
_examples/monorepo/squadrons/checkout/frontend/index.html
View File

@ -7,6 +7,6 @@
<title>Document</title>
</head>
<body>
<h1>Storefinder</h1>
<h1>Checkout</h1>
</body>
</html>

4
_examples/monorepo/squadrons/checkout/frontend/squadron.yaml
View File

@ -12,6 +12,10 @@ squadron:
image: <% .Global.docker.registry %>/checkout-frontend
context: <% env "PROJECT_ROOT" %>/squadrons/checkout/frontend
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/frontend.Dockerfile
dependencies: [ "frontend-base" ]
args:
- "BASE_IMAGE=<% .Global.docker.registry %>/frontend-base"
- "BASE_IMAGE_TAG=<% .Global.docker.tag | quote %>"
values:
image:
tag: "<% .Global.docker.tag %>"

10
_examples/monorepo/squadrons/squadron.yaml
View File

@ -4,3 +4,13 @@ global:
docker:
tag: "230101.0"
registry: monorepo
builds:
backend-base:
tag: <% .Global.docker.tag | quote %>
image: <% .Global.docker.registry %>/backend-base
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/backend-base.Dockerfile
frontend-base:
tag: <% .Global.docker.tag | quote %>
image: <% .Global.docker.registry %>/frontend-base
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/frontend-base.Dockerfile

2
_examples/monorepo/squadrons/storefinder/backend/go.mod
View File

@ -1,3 +1,3 @@
module github.com/foomo/squadron/exmpale/monorepo/storefinder/backend
go 1.16
go 1.21

4
_examples/monorepo/squadrons/storefinder/backend/squadron.yaml
View File

@ -12,6 +12,10 @@ squadron:
image: <% .Global.docker.registry %>/storefinder-backend
context: <% env "PROJECT_ROOT" %>/squadrons/storefinder/backend
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/backend.Dockerfile
dependencies: [ "backend-base" ]
args:
- "BASE_IMAGE=<% .Global.docker.registry %>/backend-base"
- "BASE_IMAGE_TAG=<% .Global.docker.tag | quote %>"
values:
image:
tag: <% .Global.docker.tag | quote %>

4
_examples/monorepo/squadrons/storefinder/frontend/squadron.yaml
View File

@ -12,6 +12,10 @@ squadron:
image: <% .Global.docker.registry %>/storefinder-frontend
context: <% env "PROJECT_ROOT" %>/squadrons/storefinder/frontend
dockerfile: <% env "PROJECT_ROOT" %>/../common/docker/frontend.Dockerfile
dependencies: [ "frontend-base" ]
args:
- "BASE_IMAGE=<% .Global.docker.registry %>/frontend-base"
- "BASE_IMAGE_TAG=<% .Global.docker.tag | quote %>"
values:
image:
tag: "<% .Global.docker.tag %>"

20
cmd/actions/list.go
View File

@ -8,10 +8,13 @@ import (
"github.com/spf13/cobra"
)
var flagPrefixSquadron bool
var (
flagBuilds bool
)
func init() {
listCmd.Flags().StringSliceVar(&flagTags, "tags", nil, "list of tags to include or exclude (can specify multiple or separate values with commas: tag1,tag2,-tag3)")
listCmd.Flags().BoolVar(&flagBuilds, "builds", false, "include builds")
}
var listCmd = &cobra.Command{
@ -31,12 +34,23 @@ var listCmd = &cobra.Command{
return err
}
return sq.Config().Squadrons.Iterate(func(key string, value config.Map[*config.Unit]) error {
// List squadrons
_ = sq.Config().Squadrons.Iterate(func(key string, value config.Map[*config.Unit]) error {
fmt.Println("Squadron:", key)
return value.Iterate(func(k string, v *config.Unit) error {
fmt.Println(" ", k)
fmt.Println(" " + k)
if flagBuilds {
for name, build := range v.Builds {
fmt.Println(" " + name)
for _, dependency := range build.Dependencies {
fmt.Println(" " + dependency)
}
}
}
return nil
})
})
return nil
},
}

102
internal/config/build.go
View File

@ -5,34 +5,71 @@ import (
"fmt"
"github.com/foomo/squadron/internal/util"
"github.com/sirupsen/logrus"
"github.com/pterm/pterm"
"gopkg.in/yaml.v3"
)
type Build struct {
Args []string `yaml:"args,omitempty"`
Builder string `yaml:"builder,omitempty"`
CacheFrom string `yaml:"cache_from,omitempty"`
CacheTo string `yaml:"cache_to,omitempty"`
Context string `yaml:"context,omitempty"`
Dockerfile string `yaml:"dockerfile,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Image string `yaml:"image,omitempty"`
IIDFile string `yaml:"iidfile,omitempty"`
Labels []string `yaml:"labels,omitempty"`
Load bool `yaml:"load,omitempty"`
MetadataFile string `yaml:"metadata_file,omitempty"`
Network string `yaml:"network,omitempty"`
NoCache bool `yaml:"no_cache,omitempty"`
Output string `yaml:"output,omitempty"`
Platform string `yaml:"platform,omitempty"`
Platforms []string `yaml:"platforms,omitempty"`
Secrets []string `yaml:"secrets,omitempty"`
ShmSize string `yaml:"shm_size,omitempty"`
SSH string `yaml:"ssh,omitempty"`
Tag string `yaml:"tag,omitempty"`
Target string `yaml:"target,omitempty"`
ULimit string `yaml:"ulimit,omitempty"`
Context string `yaml:"context,omitempty"`
// AddHost add a custom host-to-IP mapping (format: "host:ip")
AddHost []string `yaml:"add_host,omitempty"`
ExtraHost []string `yaml:"extra_hosts,omitempty"` // TODO deprecate use AddHost
// Allow extra privileged entitlement (e.g., "network.host", "security.insecure")
Allow []string `yaml:"allow,omitempty"`
// Attest parameters (format: "type=sbom,generator=image")
Attest []string `yaml:"attest,omitempty"`
// BuildArg set build-time variables
BuildArg []string `yaml:"build_arg,omitempty"`
Args []string `yaml:"args,omitempty"` // TODO deprecate use BuildArg
// BuildContext additional build contexts (e.g., name=path)
BuildContext []string `yaml:"build_context,omitempty"`
// Builder override the configured builder instance
Builder string `yaml:"builder,omitempty"`
// CacheFrom external cache sources (e.g., "user/app:cache", "type=local,src=path/to/dir")
CacheFrom string `yaml:"cache_from,omitempty"`
// CacheTo cache export destinations (e.g., "user/app:cache", "type=local,dest=path/to/dir")
CacheTo string `yaml:"cache_to,omitempty"`
// CGroupParent optional parent cgroup for the container
CGroupParent string `yaml:"cgroup_parent,omitempty"`
// File name of the Dockerfile (default: "PATH/Dockerfile")
File string `yaml:"file,omitempty"`
Dockerfile string `yaml:"dockerfile,omitempty"` // TODO deprecate use File
// IIDFile write the image ID to the file
IIDFile string `yaml:"iidfile,omitempty"`
// Label wet metadata for an image
Label []string `yaml:"label,omitempty"`
Labels []string `yaml:"labels,omitempty"` // TODO deprecate use Label
// Load shorthand for "--output=type=docker"
Load bool `yaml:"load,omitempty"`
// MetadataFile write build result metadata to the file
MetadataFile string `yaml:"metadata_file,omitempty"`
// Network set the networking mode for the "RUN" instructions during build (default "default")
Network string `yaml:"network,omitempty"`
// NoCache do not use cache when building the image
NoCache bool `yaml:"no_cache,omitempty"`
// NoCacheFilter do not cache specified stages
NoCacheFilter []string `yaml:"no_cache_filter,omitempty"`
// Output destination (format: "type=local,dest=path")
Output string `yaml:"output,omitempty"`
// Platform set target platform for build
Platform string `yaml:"platform,omitempty"`
Platforms []string `yaml:"platforms,omitempty"` // TODO deprecate use Platform
// Secret to expose to the build (format: "id=mysecret[,src=/local/secret]")
Secret []string `yaml:"secret,omitempty"`
Secrets []string `yaml:"secrets,omitempty"` // TODO deprecate use Secret
// ShmSize size of "/dev/shm"
ShmSize string `yaml:"shm_size,omitempty"`
// SSH agent socket or keys to expose to the build (format: "default|<id>[=<socket>|<key>[,<key>]]")
SSH string `yaml:"ssh,omitempty"`
// Tag name and optionally a tag (format: "name:tag")
Tag string `yaml:"tag,omitempty"`
Image string `yaml:"image,omitempty"`
// Target set the target build stage to build
Target string `yaml:"target,omitempty"`
// ULimit ulimit options (default []
ULimit string `yaml:"ulimit,omitempty"`
// Dependencies list of build names defined in the squadron configuration
Dependencies []string `yaml:"dependencies,omitempty"`
}
// ------------------------------------------------------------------------------------------------
@ -40,27 +77,38 @@ type Build struct {
// ------------------------------------------------------------------------------------------------
func (b *Build) Build(ctx context.Context, args []string) (string, error) {
logrus.Debugf("running docker build for %q", b.Context)
pterm.Debug.Printfln("running docker build for %q", b.Context)
return util.NewDockerCommand().Build(b.Context).
ListArg("--add-host", b.ExtraHosts).
ListArg("--add-host", b.ExtraHost).
ListArg("--add-host", b.AddHost).
ListArg("--allow", b.Allow).
ListArg("--attest", b.Attest).
ListArg("--build-arg", b.Args).
ListArg("--build-arg", b.BuildArg).
ListArg("--build-contet", b.BuildContext).
Arg("--builder", b.Builder).
Arg("--cache-from", b.CacheFrom).
Arg("--cache-to", b.CacheTo).
Arg("--cgroup-parent", b.Dockerfile).
Arg("--file", b.File).
Arg("--file", b.Dockerfile).
Arg("--iidfile", b.IIDFile).
ListArg("--label", b.Label).
ListArg("--label", b.Labels).
BoolArg("--load", b.Load).
Arg("--metadata-file", b.MetadataFile).
Arg("--network", b.Network).
BoolArg("--no-cache", b.NoCache).
ListArg("--noe-cache-filter", b.NoCacheFilter).
Arg("--output", b.Output).
Arg("--platform", b.Platform).
ListArg("--platform", b.Platforms).
// Arg("--progress", xxx).
// Arg("--provenance", xxx).
// Arg("--push", xxx).
// Arg("--pull", xxx).
// Arg("--quiet", xxx).
ListArg("--secret", b.Secret).
ListArg("--secret", b.Secrets).
Arg("--shm-size", b.ShmSize).
Arg("--ssh", b.SSH).
@ -72,7 +120,7 @@ func (b *Build) Build(ctx context.Context, args []string) (string, error) {
}
func (b *Build) Push(ctx context.Context, args []string) (string, error) {
logrus.Debugf("running docker push for %s:%s", b.Image, b.Tag)
pterm.Debug.Printfln("running docker push for %s:%s", b.Image, b.Tag)
return util.NewDockerCommand().Push(b.Image, b.Tag).Args(args...).Run(ctx)
}

37
internal/config/config.go
View File

@ -3,15 +3,45 @@ package config
import (
"fmt"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type Config struct {
Version string `yaml:"version,omitempty"`
Global map[string]interface{} `yaml:"global,omitempty"`
Squadrons Map[Map[*Unit]] `yaml:"squadron,omitempty"`
// Version of the schema
Version string `yaml:"version,omitempty"`
// Global values to be injected into all squadron values
Global map[string]interface{} `yaml:"global,omitempty"`
// Global builds that can be referenced as dependencies
Builds map[string]Build `yaml:"builds,omitempty"`
// Squadron definitions
Squadrons Map[Map[*Unit]] `yaml:"squadron,omitempty"`
}
// BuildDependencies returns a map of requested build dependencies
func (c *Config) BuildDependencies() map[string]Build {
ret := map[string]Build{}
_ = c.Squadrons.Iterate(func(key string, value Map[*Unit]) error {
return value.Iterate(func(k string, v *Unit) error {
for _, build := range v.Builds {
for _, dependency := range build.Dependencies {
b, ok := c.Builds[dependency]
if !ok {
return errors.Errorf("missing build dependency `%s`", dependency)
}
ret[dependency] = b
}
}
return nil
})
})
if len(ret) > 0 {
return ret
}
return nil
}
// Trim delete empty squadron recursively
func (c *Config) Trim() {
_ = c.Squadrons.Iterate(func(key string, value Map[*Unit]) error {
value.Trim()
@ -20,6 +50,7 @@ func (c *Config) Trim() {
c.Squadrons.Trim()
}
// UnmarshalYAML interface method
func (c *Config) UnmarshalYAML(value *yaml.Node) error {
switch value.Tag {
case "!!map":

9
internal/util/cmd.go
View File

@ -35,15 +35,6 @@ func NewCommand(name string) *Cmd {
}
}
func (c Cmd) Base() *Cmd {
c.command = []string{c.command[0]}
return &c
}
func (c Cmd) Command() []string {
return c.command
}
func (c *Cmd) Args(args ...string) *Cmd {
for _, arg := range args {
if arg == "" {

29
squadron.go
View File

@ -235,7 +235,36 @@ func (sq *Squadron) Push(ctx context.Context, pushArgs []string, parallel int) e
return wg.Wait()
}
func (sq *Squadron) BuildDependencies(ctx context.Context, buildArgs []string, parallel int) error {
wg, gctx := errgroup.WithContext(ctx)
wg.SetLimit(parallel)
var i int
dependencies := sq.c.BuildDependencies()
for name, dependency := range dependencies {
i := i + 1
name := name
dependency := dependency
wg.Go(func() error {
pterm.Info.Printfln("[%d/%d] Building dependency `%s`", i, len(dependencies), name)
pterm.FgGray.Printfln("└ %s:%s", dependency.Image, dependency.Tag)
if out, err := dependency.Build(gctx, buildArgs); err != nil {
pterm.Error.Printfln("[%d/%d] Failed to build dependency `%s`", i, len(dependencies), name)
pterm.FgGray.Printfln("└ %s:%s", dependency.Image, dependency.Tag)
return errors.Wrap(err, out)
}
return nil
})
}
return wg.Wait()
}
func (sq *Squadron) Build(ctx context.Context, buildArgs []string, parallel int) error {
if err := sq.BuildDependencies(ctx, buildArgs, parallel); err != nil {
return err
}
wg, gctx := errgroup.WithContext(ctx)
wg.SetLimit(parallel)

2
testdata/override/snapshop-config-norender.yaml vendored
View File

@ -14,8 +14,8 @@ squadron:
- bar=baz
- baz=baz
dockerfile: Dockerfile
image: docker.mycompany.com/mycomapny/frontend
tag: nightly
image: docker.mycompany.com/mycomapny/frontend
values:
image:
repository: <% .Squadron.storefinder.frontend.builds.default.image %>

2
testdata/override/snapshop-config.yaml vendored
View File

@ -14,8 +14,8 @@ squadron:
- bar=baz
- baz=baz
dockerfile: Dockerfile
image: docker.mycompany.com/mycomapny/frontend
tag: nightly
image: docker.mycompany.com/mycomapny/frontend
values:
image:
repository: docker.mycompany.com/mycomapny/frontend

4
testdata/template/snapshop-config-norender.yaml vendored
View File

@ -10,8 +10,8 @@ squadron:
version: 0.0.1
builds:
default:
image: docker.mycompany.com/mycomapny/frontend-admin
tag: latest
image: docker.mycompany.com/mycomapny/frontend-admin
values:
image:
repository: <% .Squadron.storefinder.backend.builds.default.image %>
@ -23,8 +23,8 @@ squadron:
version: 0.0.1
builds:
default:
image: docker.mycompany.com/mycomapny/frontend
tag: latest
image: docker.mycompany.com/mycomapny/frontend
values:
env:
BASE64: <% base64 "1234567890" %>

4
testdata/template/snapshop-config.yaml vendored
View File

@ -10,8 +10,8 @@ squadron:
version: 0.0.1
builds:
default:
image: docker.mycompany.com/mycomapny/frontend-admin
tag: latest
image: docker.mycompany.com/mycomapny/frontend-admin
values:
image:
repository: docker.mycompany.com/mycomapny/frontend-admin
@ -23,8 +23,8 @@ squadron:
version: 0.0.1
builds:
default:
image: docker.mycompany.com/mycomapny/frontend
tag: latest
image: docker.mycompany.com/mycomapny/frontend
values:
env:
BASE64: MTIzNDU2Nzg5MA==