add multiple template base path functionality

jaypipes · jaypipes · commit 907b08508f5a · 2021-03-08T19:17:37.000-05:00
We want different teams to have the ability to specify template
overrides for their specific controller or generated codebase.

There are a number of use cases where having the ability to specify
multiple "template search paths" will be useful.

Imagine that the ElastiCache ACK controller team wants to change their
boilerplate.txt content to mention something specific to ElastiCache.
Instead of adding a bunch of conditional statements to the code
generator or new fields to the generator configuration object (which is
already complicated), what if the maintainer team could add their custom
boilerplate text to a `templates/boilerplate.txt` file in the
elasticache-controller source repository and call `ack-generate`
commands like this:

```bash
CODE_GEN_REPO=$GOPATH/src/github.com/aws-controllers-k8s/code-generator
CONTROLLER_REPO=$GOPATH/src/github.com/aws-controllers-k8s/elasticache-controller
ack-generate apis --template-dirs $CONTROLLER_REPO/templates,$CODE_GEN_REPO/templates`
```

and the code generator would simply pick up the boilerplate.txt file in
the elasticache-controller repository and use that instead of the one in
the code-generator repo.

Another use case might be for a service controller team to experiment
with some Go code changes to the standard template files without needing
to modify any code in the code-generator repo itself.

For example, let's say that the S3 controller requires a radically different
set of Go code for its update code path than all the other controllers
and we want to test some ideas but not update the code-generator repo
itself. We could add a `templates/pkg/resource/sdk_update.go.tpl` file
to the s3-controller repository, rebuild the controller and iterate more
quickly.

Finally, there is the use case of having templates containing "hook
code" that is specific to one service controller. Imagine the following
generator config snippet:

```yaml
resources:
  Broker:
    hooks:
      set_update_pre_build_update_request:
        templatePath: hooks/sdk_update_pre_build_update_request.go.tpl
```

And in the service controller, we had a file
`templates/hooks/sdk_update_pre_build_update_request.go.tpl` with the
contents:

```go
if err := rm.requeueIfNotRunning; err != nil {
        return nil, err
}
```

We could call the `ack-generate` passing in multiple template search
base paths like shown above and in this way, we could have
service-specific code placed inside the service controller source
repositories and have our code generation be much more flexible.
diff --git a/cmd/ack-generate/command/apis.go b/cmd/ack-generate/command/apis.go
@@ -85,7 +85,7 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	ts, err := ackgenerate.APIs(g, optTemplatesDir)
+	ts, err := ackgenerate.APIs(g, optTemplateDirs)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/ack-generate/command/controller.go b/cmd/ack-generate/command/controller.go
@@ -82,7 +82,7 @@ func generateController(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	ts, err := ackgenerate.Controller(g, optTemplatesDir)
+	ts, err := ackgenerate.Controller(g, optTemplateDirs)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/ack-generate/command/crossplane.go b/cmd/ack-generate/command/crossplane.go
@@ -53,7 +53,6 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 	if err := ensureSDKRepo(optCacheDir); err != nil {
 		return err
 	}
-	optTemplatesDir = filepath.Join(optTemplatesDir, "crossplane")
 	svcAlias := strings.ToLower(args[0])
 	sdkHelper := model.NewSDKHelper(sdkDir)
 	sdkHelper.APIGroupSuffix = "aws.crossplane.io"
@@ -83,7 +82,7 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 		return err
 	}
 
-	ts, err := cpgenerate.Crossplane(g, optTemplatesDir)
+	ts, err := cpgenerate.Crossplane(g, optTemplateDirs)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/ack-generate/command/olm.go b/cmd/ack-generate/command/olm.go
@@ -138,7 +138,7 @@ func generateOLMAssets(cmd *cobra.Command, args []string) error {
 	}
 
 	// generate templates
-	ts, err := olmgenerate.BundleAssets(g, commonMeta, svcConf, version, optTemplatesDir)
+	ts, err := olmgenerate.BundleAssets(g, commonMeta, svcConf, version, optTemplateDirs)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/ack-generate/command/release.go b/cmd/ack-generate/command/release.go
@@ -87,7 +87,7 @@ func generateRelease(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	ts, err := ackgenerate.Release(
-		g, optTemplatesDir,
+		g, optTemplateDirs,
 		releaseVersion, optImageRepository, optServiceAccountName,
 	)
 	if err != nil {
diff --git a/cmd/ack-generate/command/root.go b/cmd/ack-generate/command/root.go
@@ -38,8 +38,8 @@ var (
 	optCacheDir            string
 	optRefreshCache        bool
 	optAWSSDKGoVersion     string
-	defaultTemplatesDir    string
-	optTemplatesDir        string
+	defaultTemplateDirs    []string
+	optTemplateDirs        []string
 	defaultServicesDir     string
 	optServicesDir         string
 	optDryRun              bool
@@ -80,7 +80,7 @@ func init() {
 	for _, tryPath := range tryPaths {
 		if fi, err := os.Stat(tryPath); err == nil {
 			if fi.IsDir() {
-				defaultTemplatesDir = tryPath
+				defaultTemplateDirs = append(defaultTemplateDirs, tryPath)
 				break
 			}
 		}
@@ -100,8 +100,8 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(
 		&optDryRun, "dry-run", false, "If true, outputs all files to stdout",
 	)
-	rootCmd.PersistentFlags().StringVar(
-		&optTemplatesDir, "templates-dir", defaultTemplatesDir, "Path to directory with templates to use in code generation",
+	rootCmd.PersistentFlags().StringSliceVar(
+		&optTemplateDirs, "template-dirs", defaultTemplateDirs, "Paths to directories with templates to use in code generation. Note that the order in which directories is specified will be used to provide override functionality.",
 	)
 	rootCmd.PersistentFlags().StringVar(
 		&optServicesDir, "services-dir", defaultServicesDir, "Path to directory to output service-specific code",
diff --git a/pkg/generate/ack/apis.go b/pkg/generate/ack/apis.go
@@ -46,7 +46,7 @@ var (
 // generating ACK service controller's apis/ contents
 func APIs(
 	g *generate.Generator,
-	templateBasePath string,
+	templateBasePaths []string,
 ) (*templateset.TemplateSet, error) {
 	enumDefs, err := g.GetEnumDefs()
 	if err != nil {
@@ -62,7 +62,7 @@ func APIs(
 	}
 
 	ts := templateset.New(
-		templateBasePath,
+		templateBasePaths,
 		apisIncludePaths,
 		apisCopyPaths,
 		apisFuncMap,
diff --git a/pkg/generate/ack/controller.go b/pkg/generate/ack/controller.go
@@ -112,15 +112,15 @@ var (
 // for generating ACK service controller implementations
 func Controller(
 	g *generate.Generator,
-	templateBasePath string,
+	templateBasePaths []string,
 ) (*templateset.TemplateSet, error) {
 	crds, err := g.GetCRDs()
 	if err != nil {
 		return nil, err
 	}
 
 	ts := templateset.New(
-		templateBasePath,
+		templateBasePaths,
 		controllerIncludePaths,
 		controllerCopyPaths,
 		controllerFuncMap,
diff --git a/pkg/generate/ack/release.go b/pkg/generate/ack/release.go
@@ -47,7 +47,7 @@ var (
 // generating an ACK service controller release (Helm artifacts, etc)
 func Release(
 	g *generate.Generator,
-	templateBasePath string,
+	templateBasePaths []string,
 	// releaseVersion is the SemVer string describing the release that the Helm
 	// chart will install
 	releaseVersion string,
@@ -59,7 +59,7 @@ func Release(
 	serviceAccountName string,
 ) (*templateset.TemplateSet, error) {
 	ts := templateset.New(
-		templateBasePath,
+		templateBasePaths,
 		releaseIncludePaths,
 		releaseCopyPaths,
 		releaseFuncMap,
diff --git a/pkg/generate/crossplane/crossplane.go b/pkg/generate/crossplane/crossplane.go
@@ -27,18 +27,18 @@ import (
 
 var (
 	apisTemplatePaths = []string{
-		"apis/doc.go.tpl",
-		"apis/enums.go.tpl",
-		"apis/groupversion_info.go.tpl",
-		"apis/types.go.tpl",
+		"crossplane/apis/doc.go.tpl",
+		"crossplane/apis/enums.go.tpl",
+		"crossplane/apis/groupversion_info.go.tpl",
+		"crossplane/apis/types.go.tpl",
 	}
 	includePaths = []string{
-		"boilerplate.go.tpl",
-		"apis/enum_def.go.tpl",
-		"apis/type_def.go.tpl",
-		"pkg/sdk_find_read_one.go.tpl",
-		"pkg/sdk_find_read_many.go.tpl",
-		"pkg/sdk_find_get_attributes.go.tpl",
+		"crossplane/boilerplate.go.tpl",
+		"crossplane/apis/enum_def.go.tpl",
+		"crossplane/apis/type_def.go.tpl",
+		"crossplane/pkg/sdk_find_read_one.go.tpl",
+		"crossplane/pkg/sdk_find_read_many.go.tpl",
+		"crossplane/pkg/sdk_find_get_attributes.go.tpl",
 	}
 	copyPaths = []string{}
 	funcMap   = ttpl.FuncMap{
@@ -111,7 +111,7 @@ type templateCRDVars struct {
 // generating Crossplane API types and controller code for an AWS service API
 func Crossplane(
 	g *generate.Generator,
-	templateBasePath string,
+	templateBasePaths []string,
 ) (*templateset.TemplateSet, error) {
 	enumDefs, err := g.GetEnumDefs()
 	if err != nil {
@@ -127,7 +127,7 @@ func Crossplane(
 	}
 
 	ts := templateset.New(
-		templateBasePath,
+		templateBasePaths,
 		includePaths,
 		copyPaths,
 		funcMap,
diff --git a/pkg/generate/olm/olm.go b/pkg/generate/olm/olm.go
@@ -43,11 +43,11 @@ func BundleAssets(
 	commonMeta CommonMetadata,
 	serviceConfig ServiceConfig,
 	vers string,
-	templateBasePath string,
+	templateBasePaths []string,
 ) (*templateset.TemplateSet, error) {
 
 	ts := templateset.New(
-		templateBasePath,
+		templateBasePaths,
 		csvIncludePaths,
 		csvCopyPaths,
 		csvFuncMap,
diff --git a/pkg/generate/templateset/templateset.go b/pkg/generate/templateset/templateset.go
@@ -15,12 +15,29 @@ package templateset
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	ttpl "text/template"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// ErrTemplateNotFound is returned when a template path couldn't be located
+	// in any base search path
+	ErrTemplateNotFound    = errors.New("Template Not Found")
+	errMsgTemplateNotFound = "template %s not found in any base search path"
 )
 
+func errTemplateNotFound(templatePath string) error {
+	return errors.WithMessage(
+		ErrTemplateNotFound,
+		fmt.Sprintf(errMsgTemplateNotFound, templatePath),
+	)
+}
+
 // templateWithVars contains a template and the variables injected during execution
 type templateWithVars struct {
 	t *ttpl.Template
@@ -30,28 +47,30 @@ type templateWithVars struct {
 // TemplateSet contains a set of templates and copy files for a particular
 // target
 type TemplateSet struct {
-	basePath     string
-	includePaths []string
-	copyPaths    []string
-	templates    map[string]templateWithVars
-	funcMap      ttpl.FuncMap
-	executed     map[string]*bytes.Buffer
+	// baseSearchPaths is a set of directories from which templates will be
+	// searched
+	baseSearchPaths []string
+	includePaths    []string
+	copyPaths       []string
+	templates       map[string]templateWithVars
+	funcMap         ttpl.FuncMap
+	executed        map[string]*bytes.Buffer
 }
 
 // New returns a pointer to a TemplateSet
 func New(
-	templateBasePath string,
+	baseSearchPaths []string,
 	includePaths []string,
 	copyPaths []string,
 	funcMap ttpl.FuncMap,
 ) *TemplateSet {
 	return &TemplateSet{
-		basePath:     templateBasePath,
-		includePaths: includePaths,
-		copyPaths:    copyPaths,
-		funcMap:      funcMap,
-		templates:    map[string]templateWithVars{},
-		executed:     map[string]*bytes.Buffer{},
+		baseSearchPaths: baseSearchPaths,
+		includePaths:    includePaths,
+		copyPaths:       copyPaths,
+		funcMap:         funcMap,
+		templates:       map[string]templateWithVars{},
+		executed:        map[string]*bytes.Buffer{},
 	}
 }
 
@@ -61,27 +80,53 @@ func (ts *TemplateSet) Add(
 	templatePath string,
 	vars interface{},
 ) error {
-	path := filepath.Join(ts.basePath, templatePath)
-	tplContents, err := ioutil.ReadFile(path)
+	var foundPath string
+	for _, basePath := range ts.baseSearchPaths {
+		path := filepath.Join(basePath, templatePath)
+		if fileExists(path) {
+			foundPath = path
+			break
+		}
+	}
+
+	if foundPath == "" {
+		return errTemplateNotFound(templatePath)
+	}
+
+	tplContents, err := ioutil.ReadFile(foundPath)
 	if err != nil {
 		return err
 	}
-	t := ttpl.New(path)
+	t := ttpl.New(foundPath)
 	t = t.Funcs(ts.funcMap)
 	t, err = t.Parse(string(tplContents))
 	if err != nil {
 		return err
 	}
-	for _, includePath := range ts.includePaths {
-		tplPath := filepath.Join(ts.basePath, includePath)
-		if t, err = includeTemplate(t, tplPath); err != nil {
-			return err
-		}
+	if err = ts.joinIncludes(t); err != nil {
+		return err
 	}
 	ts.templates[outPath] = templateWithVars{t, vars}
 	return nil
 }
 
+// joinIncludes adds all include templates to the supplied template
+func (ts *TemplateSet) joinIncludes(t *ttpl.Template) error {
+	var err error
+	for _, basePath := range ts.baseSearchPaths {
+		for _, includePath := range ts.includePaths {
+			tplPath := filepath.Join(basePath, includePath)
+			if !fileExists(tplPath) {
+				continue
+			}
+			if t, err = includeTemplate(t, tplPath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // Execute runs all of the template and copy files in our TemplateSet and
 // returns whether any error occurred executing any of the templates. Once
 // Execute() is run, `TemplateSet.Executed()` can be used to iterate over a set
@@ -94,13 +139,18 @@ func (ts *TemplateSet) Execute() error {
 		}
 		ts.executed[path] = &b
 	}
-	for _, path := range ts.copyPaths {
-		copyPath := filepath.Join(ts.basePath, path)
-		b, err := byteBufferFromFile(copyPath)
-		if err != nil {
-			return err
+	for _, basePath := range ts.baseSearchPaths {
+		for _, path := range ts.copyPaths {
+			copyPath := filepath.Join(basePath, path)
+			if !fileExists(copyPath) {
+				continue
+			}
+			b, err := byteBufferFromFile(copyPath)
+			if err != nil {
+				return err
+			}
+			ts.executed[path] = b
 		}
-		ts.executed[path] = b
 	}
 	return nil
 }
@@ -144,3 +194,9 @@ func includeTemplate(t *ttpl.Template, tplPath string) (*ttpl.Template, error) {
 	}
 	return t, nil
 }
+
+// fileExists returns tTrue if the supplied file path exists, false otherwise
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return !os.IsNotExist(err)
+}
diff --git a/scripts/build-controller.sh b/scripts/build-controller.sh

Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,7 @@ func generateAPIs(cmd *cobra.Command, args []string) error {`
`85`	`85`	`if err != nil {`
`86`	`86`	`return err`
`87`	`87`	`}`
`88`		`- ts, err := ackgenerate.APIs(g, optTemplatesDir)`
	`88`	`+ ts, err := ackgenerate.APIs(g, optTemplateDirs)`
`89`	`89`	`if err != nil {`
`90`	`90`	`return err`
`91`	`91`	`}`
Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,7 @@ func generateController(cmd *cobra.Command, args []string) error {`
`82`	`82`	`if err != nil {`
`83`	`83`	`return err`
`84`	`84`	`}`
`85`		`- ts, err := ackgenerate.Controller(g, optTemplatesDir)`
	`85`	`+ ts, err := ackgenerate.Controller(g, optTemplateDirs)`
`86`	`86`	`if err != nil {`
`87`	`87`	`return err`
`88`	`88`	`}`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,6 @@ func generateCrossplane(_ *cobra.Command, args []string) error {`
`53`	`53`	`if err := ensureSDKRepo(optCacheDir); err != nil {`
`54`	`54`	`return err`
`55`	`55`	`}`
`56`		`- optTemplatesDir = filepath.Join(optTemplatesDir, "crossplane")`
`57`	`56`	`svcAlias := strings.ToLower(args[0])`
`58`	`57`	`sdkHelper := model.NewSDKHelper(sdkDir)`
`59`	`58`	`sdkHelper.APIGroupSuffix = "aws.crossplane.io"`
`@@ -83,7 +82,7 @@ func generateCrossplane(_ *cobra.Command, args []string) error {`
`83`	`82`	`return err`
`84`	`83`	`}`
`85`	`84`
`86`		`- ts, err := cpgenerate.Crossplane(g, optTemplatesDir)`
	`85`	`+ ts, err := cpgenerate.Crossplane(g, optTemplateDirs)`
`87`	`86`	`if err != nil {`
`88`	`87`	`return err`
`89`	`88`	`}`
Original file line number	Diff line number	Diff line change
`@@ -138,7 +138,7 @@ func generateOLMAssets(cmd *cobra.Command, args []string) error {`
`138`	`138`	`}`
`139`	`139`
`140`	`140`	`// generate templates`
`141`		`- ts, err := olmgenerate.BundleAssets(g, commonMeta, svcConf, version, optTemplatesDir)`
	`141`	`+ ts, err := olmgenerate.BundleAssets(g, commonMeta, svcConf, version, optTemplateDirs)`
`142`	`142`	`if err != nil {`
`143`	`143`	`return err`
`144`	`144`	`}`
Original file line number	Diff line number	Diff line change
`@@ -87,7 +87,7 @@ func generateRelease(cmd *cobra.Command, args []string) error {`
`87`	`87`	`return err`
`88`	`88`	`}`
`89`	`89`	`ts, err := ackgenerate.Release(`
`90`		`- g, optTemplatesDir,`
	`90`	`+ g, optTemplateDirs,`
`91`	`91`	`releaseVersion, optImageRepository, optServiceAccountName,`
`92`	`92`	`)`
`93`	`93`	`if err != nil {`