foomo/squadron
1
0
Fork 0
mirror of https://github.com/foomo/squadron.git synced 2026-10-04 14:46:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/templating' into develop

Browse Source
This commit is contained in:
Milos Pejanovic
2020-06-25 15:21:23 +02:00
parent 54a27fb408 5a794e155b
commit d06ec12bbf
8 changed files with 190 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/actions/build.go
+2 -4
View File
@@ -19,16 +19,14 @@ var buildCmd = &cobra.Command{
Short: "Build a service with a given tag",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
out, err := build(args[0], flagTag, flagDir, flagPush, flagVerbose)
out, err := build(args[0], flagPush)
if err != nil {
log.WithError(err).WithField("output", out).Fatal("Build failed")
}
},
}
func build(service, tag, dir string, push, verbose bool) (string, error) {
logger := newLogger(verbose)
cnf := mustNewConfigurd(logger, tag, dir)
func build(service string, push bool) (string, error) {
svc, err := cnf.Service(service)
if err != nil {
return "", fmt.Errorf("could not find service: %w", err)
cmd/actions/install.go
+8 -23
View File
@@ -25,7 +25,7 @@ var (
Long: "installs a group of services with given namespace and tag version",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
_, err := install(args[0], flagNamespace, flagTag, flagDir, flagOutputDir, flagService, flagBuild, flagVerbose)
_, err := install(args[0], flagNamespace, flagTag, flagDir, flagOutputDir, flagService, flagBuild, templateVars)
if err != nil {
log.WithError(err).Fatalf("Installation failed")
}
@@ -33,10 +33,7 @@ var (
}
)
func install(group, namespace, tag, workDir, outputDir, service string, buildService, verbose bool) (string, error) {
log := newLogger(verbose)
cnf := mustNewConfigurd(log, tag, workDir)
func install(group, namespace, tag, workDir, outputDir, service string, buildService bool, tv configurd.TemplateVars) (string, error) {
ns, err := cnf.Namespace(namespace)
if err != nil {
return "", err
@@ -45,37 +42,25 @@ func install(group, namespace, tag, workDir, outputDir, service string, buildSer
if err != nil {
return "", err
}
sos, err := g.ServiceOverrides()
overrides, err := g.Overrides(workDir, namespace, tv)
if err != nil {
return "", err
}
if service != "" {
so, err := g.ServiceOverride(service)
if err != nil {
return "", err
}
sos = map[string]configurd.Override{
service: so,
overrides = map[string]configurd.Override{
service: overrides[service],
}
}
if buildService {
log.Printf("Building services")
for name := range sos {
output, err := build(name, tag, workDir, false, verbose)
for name := range overrides {
output, err := build(name, true)
if err != nil {
return output, err
}
}
}
return cnf.Install(configurd.InstallConfiguration{
ServiceOverrides: sos,
BasePath: workDir,
OutputDir: outputDir,
Tag: tag,
Verbose: verbose,
Group: group,
Namespace: namespace,
})
return cnf.Install(overrides, workDir, outputDir, namespace, group, tag)
}
cmd/actions/root.go
+29 -19
View File
@@ -13,50 +13,60 @@ var (
log = logrus.New()
rootCmd = &cobra.Command{
Use: "configurd",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if cmd.Name() == "help" || cmd.Name() == "init" {
return nil
}
// flagDir
wdir, err := os.Getwd()
if err != nil {
log.Fatal(err)
return err
}
if flagDir != "" {
flagDir = path.Join(wdir, flagDir)
} else {
flagDir = wdir
}
if cmd.Name() == "help" || cmd.Name() == "init" {
return
// templateVars
templateVars, err = configurd.NewTemplateVars(flagDir, flagTemplateSlice, flagTemplateFile)
if err != nil {
return err
}
// cnf
cnf, err = newConfigurd(newLogger(flagVerbose), flagTag, flagDir)
if err != nil {
return err
}
return nil
},
}
flagTag string
flagDir string
flagVerbose bool
flagNamespace string
cnf configurd.Configurd
templateVars configurd.TemplateVars
flagTag string
flagDir string
flagVerbose bool
flagNamespace string
flagTemplateSlice []string
flagTemplateFile string
)
func newConfigurd(log *logrus.Entry, tag, basePath string) (configurd.Configurd, error) {
config := configurd.Config{
Tag: tag,
BasePath: basePath,
Log: log,
Tag: tag,
BasePath: basePath,
Log: log,
}
return configurd.New(config)
}
func mustNewConfigurd(log *logrus.Entry, tag, basePath string) configurd.Configurd {
cnf, err := newConfigurd(log, tag, basePath)
if err != nil {
log.Fatal(err)
}
return cnf
}
func init() {
rootCmd.PersistentFlags().StringVarP(&flagTag, "tag", "t", "latest", "Specifies the image tag")
rootCmd.PersistentFlags().StringVarP(&flagDir, "dir", "d", "", "Specifies working directory")
rootCmd.PersistentFlags().BoolVarP(&flagVerbose, "verbose", "v", false, "Specifies should command output be displayed")
rootCmd.PersistentFlags().StringSliceVar(&flagTemplateSlice, "template-vars", nil, "Specifies template vars x=y")
rootCmd.PersistentFlags().StringVar(&flagTemplateFile, "template-file", "", "Specifies the template file with vars")
rootCmd.AddCommand(buildCmd, installCmd, uninstallCmd, initCmd, versionCmd)
}
cmd/actions/uninstall.go
+3 -6
View File
@@ -15,7 +15,7 @@ var (
Long: "uninstalls a group with given namespace and tag version",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
_, err := uninstall(args[0], flagNamespace, flagTag, flagDir, flagVerbose)
_, err := uninstall(args[0], flagNamespace)
if err != nil {
log.WithError(err).Fatalf("Uninstallation failed")
}
@@ -23,10 +23,7 @@ var (
}
)
func uninstall(group, namespace, tag, dir string, verbose bool) (string, error) {
log := newLogger(verbose)
cnf := mustNewConfigurd(log, tag, dir)
func uninstall(group, namespace string) (string, error) {
ns, err := cnf.Namespace(namespace)
if err != nil {
return "", err
@@ -36,5 +33,5 @@ func uninstall(group, namespace, tag, dir string, verbose bool) (string, error)
return "", err
}
return cnf.Uninstall(group, namespace, verbose)
return cnf.Uninstall(group, namespace)
}
configurd.go
+116 -22
View File
@@ -2,15 +2,20 @@ package configurd
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"text/template"
"github.com/foomo/config-bob/builder"
"github.com/foomo/configurd/exampledata"
"github.com/sirupsen/logrus"
@@ -168,19 +173,23 @@ func loadGroups(log *logrus.Entry, sl serviceLoader, basePath, namespace string)
}
func loadGroup(log *logrus.Entry, sl serviceLoader, path, namespace, group string) (Group, error) {
file, err := os.Open(path)
if err != nil {
return Group{}, err
}
defer file.Close()
var wrapper struct {
Group Group `yaml:"group"`
}
if err := yaml.NewDecoder(file).Decode(&wrapper); err != nil {
return Group{}, fmt.Errorf("could not decode group: %w", err)
bs, err := parseTemplate(path, nil, false)
if err != nil {
return wrapper.Group, err
}
if err := yaml.Unmarshal(bs, &wrapper); err != nil {
return wrapper.Group, err
}
wrapper.Group.name = group
for name := range wrapper.Group.Services {
// the overrides have not been parsed with templates
// we only need this on install
// so use nil instead of wrong values
wrapper.Group.Services[name] = nil
}
return wrapper.Group, nil
}
@@ -217,22 +226,19 @@ func (ns Namespace) Group(name string) (Group, error) {
return Group{}, errResourceNotFound(name, "group", available)
}
func (g Group) ServiceOverride(name string) (Override, error) {
var available []string
for n, v := range g.Services {
if name == n {
return v, nil
}
available = append(available, name)
func (g Group) Overrides(basePath, namespace string, tv TemplateVars) (map[string]Override, error) {
path := path.Join(basePath, defaultNamespaceDir, namespace, g.name+defaultConfigFileExt)
var wrapper struct {
Group Group `yaml:"group"`
}
return nil, errResourceNotFound(name, "serviceOverride", available)
}
func (g Group) ServiceOverrides() (map[string]Override, error) {
if len(g.Services) == 0 {
return nil, fmt.Errorf("could not find any service for group: %v", g.name)
bs, err := parseTemplate(path, tv, true)
if err != nil {
return nil, err
}
return g.Services, nil
if err := yaml.Unmarshal(bs, &wrapper); err != nil {
return nil, err
}
return wrapper.Group.Services, nil
}
func (c Configurd) Build(s Service) (string, error) {
@@ -327,3 +333,91 @@ func errResourceNotFound(name, resource string, available []string) error {
}
return fmt.Errorf("%s '%s' not found. Available: %s", resource, name, strings.Join(available, ", "))
}
func stringInSlice(str string, slice []string) bool {
for _, s := range slice {
if s == str {
return true
}
}
return false
}
func isYaml(file string) bool {
return stringInSlice(filepath.Ext(file), []string{".yml, .yaml"})
}
func isJson(file string) bool {
return filepath.Ext(file) == ".json"
}
type TemplateVars map[string]interface{}
func (tv TemplateVars) supportedFileExt() []string {
return []string{"yml", "yaml", "json"}
}
func NewTemplateVars(workDir string, sourceSlice []string, sourceFile string) (TemplateVars, error) {
tv := TemplateVars{}
if err := tv.parseFile(workDir, sourceFile); err != nil {
return nil, err
}
if err := tv.parseSlice(sourceSlice); err != nil {
return nil, err
}
tv["cwd"] = workDir
return tv, nil
}
func (tv TemplateVars) parseSlice(source []string) error {
for _, item := range source {
pieces := strings.Split(item, "=")
if len(pieces) != 2 || pieces[0] == "" {
return fmt.Errorf("Invalid format for template var %q, use x=y", item)
}
tv[pieces[0]] = pieces[1]
}
return nil
}
func (tv TemplateVars) parseFile(workDir, source string) error {
if source == "" {
return nil
}
if !filepath.IsAbs(source) {
source = path.Join(workDir, source)
}
if !isYaml(source) && !isJson(source) {
return fmt.Errorf("Unable to parse %q, supported: %v", source, strings.Join(tv.supportedFileExt(), ", "))
}
file, err := ioutil.ReadFile(source)
if err != nil {
return fmt.Errorf("Error while opening template file: %s", err)
}
if isYaml(source) {
if err := yaml.Unmarshal(file, &tv); err != nil {
return fmt.Errorf("Error while unmarshalling template file: %s", err)
}
}
if isJson(source) {
if err := json.Unmarshal(file, &tv); err != nil {
return fmt.Errorf("Error while unmarshalling template file: %s", err)
}
return nil
}
return nil
}
func parseTemplate(file string, templateVars interface{}, errOnMissing bool) ([]byte, error) {
tmp, err := template.ParseFiles(file)
if err != nil {
return nil, err
}
out := bytes.NewBuffer([]byte{})
if errOnMissing {
tmp = tmp.Option("missingkey=error")
}
if err := tmp.Funcs(builder.TemplateFuncs).Execute(out, templateVars); err != nil {
return nil, err
}
return out.Bytes(), nil
}
example/configurd/namespaces/local/hello-group.yml
+3 -4
View File
@@ -8,11 +8,11 @@ group:
port: 80
ingress:
hosts:
- name: localhost
- name: {{ .host }}
path: /services/hello
volumes:
- name: config
host: ./application
host: {{ .cwd }}/application
mount: /config/
hi-service:
args:
@@ -22,6 +22,5 @@ group:
port: 80
ingress:
hosts:
- name: localhost
- name: {{ .host }}
path: /services/hi
jobs:
go.mod
+2 -2
View File
@@ -3,8 +3,8 @@ module github.com/foomo/configurd
go 1.14
require (
github.com/mitchellh/mapstructure v1.1.2
github.com/otiai10/copy v1.0.2
github.com/foomo/config-bob v0.0.0-20180509121310-96db3832d519
github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c // indirect
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.6
github.com/stretchr/testify v1.4.0 // indirect
helm.go
+27 -58
View File
@@ -4,9 +4,7 @@ import (
"fmt"
"os"
"path"
"strings"
"github.com/mitchellh/mapstructure"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
@@ -39,15 +37,12 @@ func NewChart(name string) *Chart {
}
}
type Image struct {
Repository string
Tag string
}
type Volume struct {
Name string
Host string
Mount string
type JobItem struct {
Name string
Overrides interface{}
namespace string
group string
chart string
}
func generateYaml(_ *logrus.Entry, path string, data interface{}) error {
@@ -67,29 +62,6 @@ func generateYaml(_ *logrus.Entry, path string, data interface{}) error {
return nil
}
func updateVolumeOverride(basePath string, override Override) (Override, error) {
if _, ok := override["volumes"]; ok {
// if there are volume overrides, make relative paths absolute
var vs []Volume
if err := mapstructure.Decode(override["volumes"], &vs); err != nil {
return nil, err
}
for i := range vs {
vs[i].Host = strings.Replace(vs[i].Host, "./", fmt.Sprintf("%v/", basePath), 1)
}
override["volumes"] = vs
}
return override, nil
}
func updateImageOverride(image, tag string, override Override) (Override, error) {
if _, ok := override["image"]; !ok {
// if repository and tag are not in overrides, add them
override["image"] = Image{image, tag}
}
return override, nil
}
func helmUpdateDependency(log *logrus.Entry, group, groupChartPath string) (string, error) {
log.Infof("Running helm dependency update for group: %v", group)
cmd := []string{"helm", "dependency", "update", groupChartPath}
@@ -117,24 +89,25 @@ func helmUninstall(log *logrus.Entry, group, namespace string) (string, error) {
return runCommand(log, "", nil, cmd...)
}
type InstallConfiguration struct {
ServiceOverrides map[string]Override
BasePath string
OutputDir string
Tag string
Verbose bool
Group string
Namespace string
func updateImageOverride(image, tag string, override Override) (Override, error) {
if _, ok := override["image"]; !ok {
// if repository and tag are not in overrides, add them
override["image"] = map[string]string{
"repository": image,
"tag": tag,
}
}
return override, nil
}
func (c Configurd) Install(cnf InstallConfiguration) (string, error) {
func (c Configurd) Install(ors map[string]Override, basePath, outputDir, namespace, group, tag string) (string, error) {
logger := c.config.Log
logger.Infof("Installing services")
groupChartPath := path.Join(cnf.BasePath, defaultOutputDir, cnf.OutputDir, cnf.Group)
groupChartPath := path.Join(basePath, defaultOutputDir, outputDir, group)
logger.Infof("Entering dir: %q", path.Join(cnf.BasePath, defaultOutputDir))
logger.Printf("Creating dir: %q", cnf.OutputDir)
logger.Infof("Entering dir: %q", path.Join(basePath, defaultOutputDir))
logger.Printf("Creating dir: %q", outputDir)
if _, err := os.Stat(groupChartPath); os.IsNotExist(err) {
if err := os.MkdirAll(groupChartPath, 0744); err != nil {
return "", fmt.Errorf("could not create a workdir directory: %w", err)
@@ -152,39 +125,35 @@ func (c Configurd) Install(cnf InstallConfiguration) (string, error) {
return "", fmt.Errorf("could not clean workdir directory: %w", err)
}
groupChart := NewChart(cnf.Group)
for name, override := range cnf.ServiceOverrides {
groupChart := NewChart(group)
for name, override := range ors {
s, err := c.Service(name)
if err != nil {
return "", err
}
override, err = updateVolumeOverride(cnf.BasePath, override)
ors[name], err = updateImageOverride(s.Image, s.Tag, override)
if err != nil {
return "", err
}
override, err = updateImageOverride(s.Image, s.Tag, override)
if err != nil {
return "", err
}
cnf.ServiceOverrides[name] = override
groupChart.Dependencies = append(groupChart.Dependencies, s.Chart)
}
if err := generateYaml(logger, path.Join(groupChartPath, chartFile), groupChart); err != nil {
return "", err
}
if err := generateYaml(logger, path.Join(groupChartPath, valuesFile), cnf.ServiceOverrides); err != nil {
if err := generateYaml(logger, path.Join(groupChartPath, valuesFile), ors); err != nil {
return "", err
}
output, err := helmUpdateDependency(logger, cnf.Group, groupChartPath)
output, err := helmUpdateDependency(logger, group, groupChartPath)
if err != nil {
return output, err
}
return helmInstall(logger, cnf.Group, cnf.Namespace, groupChartPath)
return helmInstall(logger, group, namespace, groupChartPath)
}
func (c Configurd) Uninstall(group, namespace string, verbose bool) (string, error) {
func (c Configurd) Uninstall(group, namespace string) (string, error) {
logger := c.config.Log
output, err := helmUninstall(logger, group, namespace)