diff --git a/cmd/actions/build.go b/cmd/actions/build.go index 03b07a6..8c18054 100644 --- a/cmd/actions/build.go +++ b/cmd/actions/build.go @@ -19,16 +19,14 @@ var buildCmd = &cobra.Command{ Short: "Build a service with a given tag", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - out, err := build(args[0], flagTag, flagDir, flagPush, flagVerbose) + out, err := build(args[0], flagPush) if err != nil { log.WithError(err).WithField("output", out).Fatal("Build failed") } }, } -func build(service, tag, dir string, push, verbose bool) (string, error) { - logger := newLogger(verbose) - cnf := mustNewConfigurd(logger, tag, dir) +func build(service string, push bool) (string, error) { svc, err := cnf.Service(service) if err != nil { return "", fmt.Errorf("could not find service: %w", err) diff --git a/cmd/actions/install.go b/cmd/actions/install.go index fbb7b3e..c773614 100644 --- a/cmd/actions/install.go +++ b/cmd/actions/install.go @@ -25,7 +25,7 @@ var ( Long: "installs a group of services with given namespace and tag version", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - _, err := install(args[0], flagNamespace, flagTag, flagDir, flagOutputDir, flagService, flagBuild, flagVerbose) + _, err := install(args[0], flagNamespace, flagTag, flagDir, flagOutputDir, flagService, flagBuild, templateVars) if err != nil { log.WithError(err).Fatalf("Installation failed") } @@ -33,10 +33,7 @@ var ( } ) -func install(group, namespace, tag, workDir, outputDir, service string, buildService, verbose bool) (string, error) { - log := newLogger(verbose) - cnf := mustNewConfigurd(log, tag, workDir) - +func install(group, namespace, tag, workDir, outputDir, service string, buildService bool, tv configurd.TemplateVars) (string, error) { ns, err := cnf.Namespace(namespace) if err != nil { return "", err @@ -45,37 +42,25 @@ func install(group, namespace, tag, workDir, outputDir, service string, buildSer if err != nil { return "", err } - sos, err := g.ServiceOverrides() + overrides, err := g.Overrides(workDir, namespace, tv) if err != nil { return "", err } if service != "" { - so, err := g.ServiceOverride(service) - if err != nil { - return "", err - } - sos = map[string]configurd.Override{ - service: so, + overrides = map[string]configurd.Override{ + service: overrides[service], } } if buildService { log.Printf("Building services") - for name := range sos { - output, err := build(name, tag, workDir, false, verbose) + for name := range overrides { + output, err := build(name, true) if err != nil { return output, err } } } - return cnf.Install(configurd.InstallConfiguration{ - ServiceOverrides: sos, - BasePath: workDir, - OutputDir: outputDir, - Tag: tag, - Verbose: verbose, - Group: group, - Namespace: namespace, - }) + return cnf.Install(overrides, workDir, outputDir, namespace, group, tag) } diff --git a/cmd/actions/root.go b/cmd/actions/root.go index d5e018e..852edcf 100644 --- a/cmd/actions/root.go +++ b/cmd/actions/root.go @@ -13,50 +13,60 @@ var ( log = logrus.New() rootCmd = &cobra.Command{ Use: "configurd", - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if cmd.Name() == "help" || cmd.Name() == "init" { + return nil + } + // flagDir wdir, err := os.Getwd() if err != nil { - log.Fatal(err) + return err } if flagDir != "" { flagDir = path.Join(wdir, flagDir) } else { flagDir = wdir } - if cmd.Name() == "help" || cmd.Name() == "init" { - return + // templateVars + templateVars, err = configurd.NewTemplateVars(flagDir, flagTemplateSlice, flagTemplateFile) + if err != nil { + return err } + // cnf + cnf, err = newConfigurd(newLogger(flagVerbose), flagTag, flagDir) + if err != nil { + return err + } + return nil }, } - flagTag string - flagDir string - flagVerbose bool - flagNamespace string + cnf configurd.Configurd + templateVars configurd.TemplateVars + flagTag string + flagDir string + flagVerbose bool + flagNamespace string + flagTemplateSlice []string + flagTemplateFile string ) func newConfigurd(log *logrus.Entry, tag, basePath string) (configurd.Configurd, error) { config := configurd.Config{ - Tag: tag, - BasePath: basePath, - Log: log, + Tag: tag, + BasePath: basePath, + Log: log, } return configurd.New(config) } -func mustNewConfigurd(log *logrus.Entry, tag, basePath string) configurd.Configurd { - cnf, err := newConfigurd(log, tag, basePath) - if err != nil { - log.Fatal(err) - } - return cnf -} - func init() { rootCmd.PersistentFlags().StringVarP(&flagTag, "tag", "t", "latest", "Specifies the image tag") rootCmd.PersistentFlags().StringVarP(&flagDir, "dir", "d", "", "Specifies working directory") rootCmd.PersistentFlags().BoolVarP(&flagVerbose, "verbose", "v", false, "Specifies should command output be displayed") + rootCmd.PersistentFlags().StringSliceVar(&flagTemplateSlice, "template-vars", nil, "Specifies template vars x=y") + rootCmd.PersistentFlags().StringVar(&flagTemplateFile, "template-file", "", "Specifies the template file with vars") rootCmd.AddCommand(buildCmd, installCmd, uninstallCmd, initCmd, versionCmd) } diff --git a/cmd/actions/uninstall.go b/cmd/actions/uninstall.go index a5bbae0..e7ad016 100644 --- a/cmd/actions/uninstall.go +++ b/cmd/actions/uninstall.go @@ -15,7 +15,7 @@ var ( Long: "uninstalls a group with given namespace and tag version", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - _, err := uninstall(args[0], flagNamespace, flagTag, flagDir, flagVerbose) + _, err := uninstall(args[0], flagNamespace) if err != nil { log.WithError(err).Fatalf("Uninstallation failed") } @@ -23,10 +23,7 @@ var ( } ) -func uninstall(group, namespace, tag, dir string, verbose bool) (string, error) { - log := newLogger(verbose) - cnf := mustNewConfigurd(log, tag, dir) - +func uninstall(group, namespace string) (string, error) { ns, err := cnf.Namespace(namespace) if err != nil { return "", err @@ -36,5 +33,5 @@ func uninstall(group, namespace, tag, dir string, verbose bool) (string, error) return "", err } - return cnf.Uninstall(group, namespace, verbose) + return cnf.Uninstall(group, namespace) } diff --git a/configurd.go b/configurd.go index f9027fe..da1d41e 100644 --- a/configurd.go +++ b/configurd.go @@ -2,15 +2,20 @@ package configurd import ( "bufio" + "bytes" + "encoding/json" "errors" "fmt" "io" + "io/ioutil" "os" "os/exec" "path" "path/filepath" "strings" + "text/template" + "github.com/foomo/config-bob/builder" "github.com/foomo/configurd/exampledata" "github.com/sirupsen/logrus" @@ -168,19 +173,23 @@ func loadGroups(log *logrus.Entry, sl serviceLoader, basePath, namespace string) } func loadGroup(log *logrus.Entry, sl serviceLoader, path, namespace, group string) (Group, error) { - file, err := os.Open(path) - if err != nil { - return Group{}, err - } - defer file.Close() - var wrapper struct { Group Group `yaml:"group"` } - if err := yaml.NewDecoder(file).Decode(&wrapper); err != nil { - return Group{}, fmt.Errorf("could not decode group: %w", err) + bs, err := parseTemplate(path, nil, false) + if err != nil { + return wrapper.Group, err + } + if err := yaml.Unmarshal(bs, &wrapper); err != nil { + return wrapper.Group, err } wrapper.Group.name = group + for name := range wrapper.Group.Services { + // the overrides have not been parsed with templates + // we only need this on install + // so use nil instead of wrong values + wrapper.Group.Services[name] = nil + } return wrapper.Group, nil } @@ -217,22 +226,19 @@ func (ns Namespace) Group(name string) (Group, error) { return Group{}, errResourceNotFound(name, "group", available) } -func (g Group) ServiceOverride(name string) (Override, error) { - var available []string - for n, v := range g.Services { - if name == n { - return v, nil - } - available = append(available, name) +func (g Group) Overrides(basePath, namespace string, tv TemplateVars) (map[string]Override, error) { + path := path.Join(basePath, defaultNamespaceDir, namespace, g.name+defaultConfigFileExt) + var wrapper struct { + Group Group `yaml:"group"` } - return nil, errResourceNotFound(name, "serviceOverride", available) -} - -func (g Group) ServiceOverrides() (map[string]Override, error) { - if len(g.Services) == 0 { - return nil, fmt.Errorf("could not find any service for group: %v", g.name) + bs, err := parseTemplate(path, tv, true) + if err != nil { + return nil, err } - return g.Services, nil + if err := yaml.Unmarshal(bs, &wrapper); err != nil { + return nil, err + } + return wrapper.Group.Services, nil } func (c Configurd) Build(s Service) (string, error) { @@ -327,3 +333,91 @@ func errResourceNotFound(name, resource string, available []string) error { } return fmt.Errorf("%s '%s' not found. Available: %s", resource, name, strings.Join(available, ", ")) } + +func stringInSlice(str string, slice []string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} + +func isYaml(file string) bool { + return stringInSlice(filepath.Ext(file), []string{".yml, .yaml"}) +} + +func isJson(file string) bool { + return filepath.Ext(file) == ".json" +} + +type TemplateVars map[string]interface{} + +func (tv TemplateVars) supportedFileExt() []string { + return []string{"yml", "yaml", "json"} +} + +func NewTemplateVars(workDir string, sourceSlice []string, sourceFile string) (TemplateVars, error) { + tv := TemplateVars{} + if err := tv.parseFile(workDir, sourceFile); err != nil { + return nil, err + } + if err := tv.parseSlice(sourceSlice); err != nil { + return nil, err + } + tv["cwd"] = workDir + return tv, nil +} + +func (tv TemplateVars) parseSlice(source []string) error { + for _, item := range source { + pieces := strings.Split(item, "=") + if len(pieces) != 2 || pieces[0] == "" { + return fmt.Errorf("Invalid format for template var %q, use x=y", item) + } + tv[pieces[0]] = pieces[1] + } + return nil +} +func (tv TemplateVars) parseFile(workDir, source string) error { + if source == "" { + return nil + } + if !filepath.IsAbs(source) { + source = path.Join(workDir, source) + } + if !isYaml(source) && !isJson(source) { + return fmt.Errorf("Unable to parse %q, supported: %v", source, strings.Join(tv.supportedFileExt(), ", ")) + } + file, err := ioutil.ReadFile(source) + if err != nil { + return fmt.Errorf("Error while opening template file: %s", err) + } + if isYaml(source) { + if err := yaml.Unmarshal(file, &tv); err != nil { + return fmt.Errorf("Error while unmarshalling template file: %s", err) + } + } + if isJson(source) { + if err := json.Unmarshal(file, &tv); err != nil { + return fmt.Errorf("Error while unmarshalling template file: %s", err) + } + return nil + } + return nil +} + +func parseTemplate(file string, templateVars interface{}, errOnMissing bool) ([]byte, error) { + tmp, err := template.ParseFiles(file) + if err != nil { + return nil, err + } + out := bytes.NewBuffer([]byte{}) + if errOnMissing { + tmp = tmp.Option("missingkey=error") + } + if err := tmp.Funcs(builder.TemplateFuncs).Execute(out, templateVars); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/example/configurd/namespaces/local/hello-group.yml b/example/configurd/namespaces/local/hello-group.yml index 381e91a..0fb64fb 100644 --- a/example/configurd/namespaces/local/hello-group.yml +++ b/example/configurd/namespaces/local/hello-group.yml @@ -8,11 +8,11 @@ group: port: 80 ingress: hosts: - - name: localhost + - name: {{ .host }} path: /services/hello volumes: - name: config - host: ./application + host: {{ .cwd }}/application mount: /config/ hi-service: args: @@ -22,6 +22,5 @@ group: port: 80 ingress: hosts: - - name: localhost + - name: {{ .host }} path: /services/hi -jobs: diff --git a/go.mod b/go.mod index 8683b7e..fc6c7f6 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/foomo/configurd go 1.14 require ( - github.com/mitchellh/mapstructure v1.1.2 - github.com/otiai10/copy v1.0.2 + github.com/foomo/config-bob v0.0.0-20180509121310-96db3832d519 + github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c // indirect github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.6 github.com/stretchr/testify v1.4.0 // indirect diff --git a/helm.go b/helm.go index 74ae76f..3e61597 100644 --- a/helm.go +++ b/helm.go @@ -4,9 +4,7 @@ import ( "fmt" "os" "path" - "strings" - "github.com/mitchellh/mapstructure" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -39,15 +37,12 @@ func NewChart(name string) *Chart { } } -type Image struct { - Repository string - Tag string -} - -type Volume struct { - Name string - Host string - Mount string +type JobItem struct { + Name string + Overrides interface{} + namespace string + group string + chart string } func generateYaml(_ *logrus.Entry, path string, data interface{}) error { @@ -67,29 +62,6 @@ func generateYaml(_ *logrus.Entry, path string, data interface{}) error { return nil } -func updateVolumeOverride(basePath string, override Override) (Override, error) { - if _, ok := override["volumes"]; ok { - // if there are volume overrides, make relative paths absolute - var vs []Volume - if err := mapstructure.Decode(override["volumes"], &vs); err != nil { - return nil, err - } - for i := range vs { - vs[i].Host = strings.Replace(vs[i].Host, "./", fmt.Sprintf("%v/", basePath), 1) - } - override["volumes"] = vs - } - return override, nil -} - -func updateImageOverride(image, tag string, override Override) (Override, error) { - if _, ok := override["image"]; !ok { - // if repository and tag are not in overrides, add them - override["image"] = Image{image, tag} - } - return override, nil -} - func helmUpdateDependency(log *logrus.Entry, group, groupChartPath string) (string, error) { log.Infof("Running helm dependency update for group: %v", group) cmd := []string{"helm", "dependency", "update", groupChartPath} @@ -117,24 +89,25 @@ func helmUninstall(log *logrus.Entry, group, namespace string) (string, error) { return runCommand(log, "", nil, cmd...) } -type InstallConfiguration struct { - ServiceOverrides map[string]Override - BasePath string - OutputDir string - Tag string - Verbose bool - Group string - Namespace string +func updateImageOverride(image, tag string, override Override) (Override, error) { + if _, ok := override["image"]; !ok { + // if repository and tag are not in overrides, add them + override["image"] = map[string]string{ + "repository": image, + "tag": tag, + } + } + return override, nil } -func (c Configurd) Install(cnf InstallConfiguration) (string, error) { +func (c Configurd) Install(ors map[string]Override, basePath, outputDir, namespace, group, tag string) (string, error) { logger := c.config.Log logger.Infof("Installing services") - groupChartPath := path.Join(cnf.BasePath, defaultOutputDir, cnf.OutputDir, cnf.Group) + groupChartPath := path.Join(basePath, defaultOutputDir, outputDir, group) - logger.Infof("Entering dir: %q", path.Join(cnf.BasePath, defaultOutputDir)) - logger.Printf("Creating dir: %q", cnf.OutputDir) + logger.Infof("Entering dir: %q", path.Join(basePath, defaultOutputDir)) + logger.Printf("Creating dir: %q", outputDir) if _, err := os.Stat(groupChartPath); os.IsNotExist(err) { if err := os.MkdirAll(groupChartPath, 0744); err != nil { return "", fmt.Errorf("could not create a workdir directory: %w", err) @@ -152,39 +125,35 @@ func (c Configurd) Install(cnf InstallConfiguration) (string, error) { return "", fmt.Errorf("could not clean workdir directory: %w", err) } - groupChart := NewChart(cnf.Group) - for name, override := range cnf.ServiceOverrides { + groupChart := NewChart(group) + for name, override := range ors { s, err := c.Service(name) if err != nil { return "", err } - override, err = updateVolumeOverride(cnf.BasePath, override) + ors[name], err = updateImageOverride(s.Image, s.Tag, override) if err != nil { return "", err } - override, err = updateImageOverride(s.Image, s.Tag, override) - if err != nil { - return "", err - } - cnf.ServiceOverrides[name] = override groupChart.Dependencies = append(groupChart.Dependencies, s.Chart) } if err := generateYaml(logger, path.Join(groupChartPath, chartFile), groupChart); err != nil { return "", err } - if err := generateYaml(logger, path.Join(groupChartPath, valuesFile), cnf.ServiceOverrides); err != nil { + if err := generateYaml(logger, path.Join(groupChartPath, valuesFile), ors); err != nil { return "", err } - output, err := helmUpdateDependency(logger, cnf.Group, groupChartPath) + output, err := helmUpdateDependency(logger, group, groupChartPath) if err != nil { return output, err } - return helmInstall(logger, cnf.Group, cnf.Namespace, groupChartPath) + + return helmInstall(logger, group, namespace, groupChartPath) } -func (c Configurd) Uninstall(group, namespace string, verbose bool) (string, error) { +func (c Configurd) Uninstall(group, namespace string) (string, error) { logger := c.config.Log output, err := helmUninstall(logger, group, namespace)