Support streaming logs for remote builds.

builder/build.go

@@ -4,24 +4,26 @@
 package builder
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strings"
 
 	v2execute "github.com/alexellis/go-execute/v2"
 	"github.com/openfaas/faas-cli/schema"
 	"github.com/openfaas/faas-cli/stack"
 	vcs "github.com/openfaas/faas-cli/versioncontrol"
+	"github.com/openfaas/go-sdk/builder"
 )
 
 // AdditionalPackageBuildArg holds the special build-arg keyname for use with build-opts.
@@ -52,13 +54,18 @@ func BuildImage(image string, handler string, functionName string, language stri
 			return fmt.Errorf("building %s, %s is an invalid path", functionName, handler)
 		}
 
-		tempPath, err := createBuildContext(functionName, handler, language, isLanguageTemplate(language), langTemplate.HandlerFolder, copyExtraPaths)
+		opts := []builder.BuildContextOption{}
+		if len(langTemplate.HandlerFolder) > 0 {
+			opts = append(opts, builder.WithHandlerOverlay(langTemplate.HandlerFolder))
+		}
+
+		buildContext, err := builder.CreateBuildContext(functionName, handler, language, copyExtraPaths, opts...)
 		if err != nil {
 			return err
 		}
 
 		if shrinkwrap {
-			fmt.Printf("%s shrink-wrapped to %s\n", functionName, tempPath)
+			fmt.Printf("%s shrink-wrapped to %s\n", functionName, buildContext)
 			return nil
 		}
 
@@ -69,65 +76,80 @@ func BuildImage(image string, handler string, functionName string, language stri
 
 		imageName := schema.BuildImageName(tagFormat, image, version, branch)
 
+		buildOptPackages, err := getBuildOptionPackages(buildOptions, language, langTemplate.BuildOptions)
+		if err != nil {
+			return err
+
+		}
+		buildArgMap = appendAdditionalPackages(buildArgMap, buildOptPackages)
+
 		fmt.Printf("Building: %s with %s template. Please wait..\n", imageName, language)
 
 		if remoteBuilder != "" {
-			tempDir, err := os.MkdirTemp(os.TempDir(), "builder-*")
+			tempDir, err := os.MkdirTemp(os.TempDir(), "openfaas-build-*")
 			if err != nil {
-				return fmt.Errorf("failed to create temporary directory for %s, error: %w", functionName, err)
+				return fmt.Errorf("failed to create temporary directory: %w", err)
 			}
 			defer os.RemoveAll(tempDir)
 
 			tarPath := path.Join(tempDir, "req.tar")
 
-			if err := makeTar(buildConfig{Image: imageName, BuildArgs: buildArgMap}, path.Join("build", functionName), tarPath); err != nil {
+			buildConfig := builder.BuildConfig{
+				Image:     imageName,
+				BuildArgs: buildArgMap,
+			}
+
+			// Prepare a tar archive that contains the build config and build context.
+			if err := builder.MakeTar(tarPath, path.Join("build", functionName), &buildConfig); err != nil {
 				return fmt.Errorf("failed to create tar file for %s, error: %w", functionName, err)
 			}
 
-			res, err := callBuilder(tarPath, remoteBuilder, functionName, payloadSecretPath)
+			// Get the HMAC secret used for payload authentication with the builder API.
+			payloadSecret, err := os.ReadFile(payloadSecretPath)
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to read payload secret: %w", err)
 			}
-			defer res.Body.Close()
+			payloadSecret = bytes.TrimSpace(payloadSecret)
 
-			data, _ := io.ReadAll(res.Body)
+			// Initialize a new builder client.
+			u, _ := url.Parse(remoteBuilder)
+			builderURL := &url.URL{
+				Scheme: u.Scheme,
+				Host:   u.Host,
+			}
+			b := builder.NewFunctionBuilder(builderURL, http.DefaultClient, builder.WithHmacAuth(string(payloadSecret)))
 
-			result := builderResult{}
-			if err := json.Unmarshal(data, &result); err != nil {
-				return err
+			stream, err := b.BuildWithStream(tarPath)
+			if err != nil {
+				return fmt.Errorf("failed to invoke builder: %w", err)
 			}
+			defer stream.Close()
 
-			if !quietBuild {
-				for _, logMsg := range result.Log {
-					fmt.Printf("%s\n", logMsg)
+			for result := range stream.Results() {
+				if !quietBuild {
+					for _, logMsg := range result.Log {
+						fmt.Printf("%s\n", logMsg)
+					}
 				}
-			}
 
-			if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted {
-				fmt.Println(res.StatusCode)
-				return fmt.Errorf("%s failure while building or pushing image %s: %s", functionName, imageName, result.Status)
+				switch result.Status {
+				case builder.BuildSuccess:
+					log.Printf("%s success building and pushing image: %s", functionName, result.Image)
+				case builder.BuildFailed:
+					return fmt.Errorf("%s failure while building or pushing image %s: %s", functionName, imageName, result.Error)
+				}
 			}
 
-			log.Printf("%s success building and pushing image: %s", functionName, result.Image)
-
 		} else {
-
-			buildOptPackages, err := getBuildOptionPackages(buildOptions, language, langTemplate.BuildOptions)
-			if err != nil {
-				return err
-
-			}
-
 			dockerBuildVal := dockerBuild{
-				Image:            imageName,
-				NoCache:          nocache,
-				Squash:           squash,
-				HTTPProxy:        os.Getenv("http_proxy"),
-				HTTPSProxy:       os.Getenv("https_proxy"),
-				BuildArgMap:      buildArgMap,
-				BuildOptPackages: buildOptPackages,
-				BuildLabelMap:    buildLabelMap,
-				ForcePull:        forcePull,
+				Image:         imageName,
+				NoCache:       nocache,
+				Squash:        squash,
+				HTTPProxy:     os.Getenv("http_proxy"),
+				HTTPSProxy:    os.Getenv("https_proxy"),
+				BuildArgMap:   buildArgMap,
+				BuildLabelMap: buildLabelMap,
+				ForcePull:     forcePull,
 			}
 
 			command, args := getDockerBuildCommand(dockerBuildVal)
@@ -139,7 +161,7 @@ func BuildImage(image string, handler string, functionName string, language stri
 			log.Printf("Build flags: %+v\n", args)
 
 			task := v2execute.ExecTask{
-				Cwd:         tempPath,
+				Cwd:         buildContext,
 				Command:     command,
 				Args:        args,
 				StreamStdio: !quietBuild,
@@ -262,7 +284,7 @@ func hashFolder(contextPath string) (string, error) {
 }
 
 func getDockerBuildCommand(build dockerBuild) (string, []string) {
-	flagSlice := buildFlagSlice(build.NoCache, build.Squash, build.HTTPProxy, build.HTTPSProxy, build.BuildArgMap, build.BuildOptPackages, build.BuildLabelMap, build.ForcePull)
+	flagSlice := buildFlagSlice(build.NoCache, build.Squash, build.HTTPProxy, build.HTTPSProxy, build.BuildArgMap, build.BuildLabelMap, build.ForcePull)
 	args := []string{"build"}
 	args = append(args, flagSlice...)
 
@@ -274,15 +296,14 @@ func getDockerBuildCommand(build dockerBuild) (string, []string) {
 }
 
 type dockerBuild struct {
-	Image            string
-	Version          string
-	NoCache          bool
-	Squash           bool
-	HTTPProxy        string
-	HTTPSProxy       string
-	BuildArgMap      map[string]string
-	BuildOptPackages []string
-	BuildLabelMap    map[string]string
+	Image         string
+	Version       string
+	NoCache       bool
+	Squash        bool
+	HTTPProxy     string
+	HTTPSProxy    string
+	BuildArgMap   map[string]string
+	BuildLabelMap map[string]string
 
 	// Platforms for use with buildx and publish command
 	Platforms string
@@ -293,101 +314,6 @@ type dockerBuild struct {
 	ForcePull bool
 }
 
-var defaultDirPermissions os.FileMode = 0700
-
-const defaultHandlerFolder string = "function"
-
-// isRunningInCI checks the ENV var CI and returns true if it's set to true or 1
-func isRunningInCI() bool {
-	if env, ok := os.LookupEnv("CI"); ok {
-		if env == "true" || env == "1" {
-			return true
-		}
-	}
-	return false
-}
-
-// createBuildContext creates temporary build folder to perform a Docker build with language template
-func createBuildContext(functionName string, handler string, language string, useFunction bool, handlerFolder string, copyExtraPaths []string) (string, error) {
-	tempPath := fmt.Sprintf("./build/%s/", functionName)
-
-	if err := os.RemoveAll(tempPath); err != nil {
-		return tempPath, fmt.Errorf("unable to clear temporary build folder: %s", tempPath)
-	}
-
-	functionPath := tempPath
-
-	if useFunction {
-		if handlerFolder == "" {
-			functionPath = path.Join(functionPath, defaultHandlerFolder)
-		} else {
-			functionPath = path.Join(functionPath, handlerFolder)
-		}
-	}
-
-	// fmt.Printf("Preparing: %s %s\n", handler+"/", functionPath)
-
-	if isRunningInCI() {
-		defaultDirPermissions = 0777
-	}
-
-	mkdirErr := os.MkdirAll(functionPath, defaultDirPermissions)
-	if mkdirErr != nil {
-		fmt.Printf("Error creating path: %s - %s.\n", functionPath, mkdirErr.Error())
-		return tempPath, mkdirErr
-	}
-
-	if useFunction {
-		if err := CopyFiles(path.Join("./template/", language), tempPath); err != nil {
-			fmt.Printf("Error copying template directory: %s.\n", err.Error())
-			return tempPath, err
-		}
-	}
-
-	// Overlay in user-function
-	// CopyFiles(handler, functionPath)
-	infos, err := os.ReadDir(handler)
-	if err != nil {
-		fmt.Printf("Error reading the handler: %s - %s.\n", handler, err.Error())
-		return tempPath, err
-	}
-
-	for _, info := range infos {
-		switch info.Name() {
-		case "build", "template":
-			fmt.Printf("Skipping \"%s\" folder\n", info.Name())
-			continue
-		default:
-			if err := CopyFiles(
-				filepath.Clean(path.Join(handler, info.Name())),
-				filepath.Clean(path.Join(functionPath, info.Name())),
-			); err != nil {
-				return tempPath, err
-			}
-		}
-	}
-
-	for _, extraPath := range copyExtraPaths {
-		extraPathAbs, err := pathInScope(extraPath, ".")
-		if err != nil {
-			return tempPath, err
-		}
-		// Note that if useFunction is false, ie is a `dockerfile` template, then
-		// functionPath == tempPath, the docker build context, not the `function` handler folder
-		// inside the docker build context
-		copyErr := CopyFiles(
-			extraPathAbs,
-			filepath.Clean(path.Join(functionPath, extraPath)),
-		)
-
-		if copyErr != nil {
-			return tempPath, copyErr
-		}
-	}
-
-	return tempPath, nil
-}
-
 // pathInScope returns the absolute path to `path` and ensures that it is located within the
 // provided scope. An error will be returned, if the path is outside of the provided scope.
 func pathInScope(path string, scope string) (string, error) {
@@ -413,7 +339,7 @@ func pathInScope(path string, scope string) (string, error) {
 	return "", fmt.Errorf("forbidden path appears to be outside of the build context: %s (%s)", path, abs)
 }
 
-func buildFlagSlice(nocache bool, squash bool, httpProxy string, httpsProxy string, buildArgMap map[string]string, buildOptionPackages []string, buildLabelMap map[string]string, forcePull bool) []string {
+func buildFlagSlice(nocache bool, squash bool, httpProxy string, httpsProxy string, buildArgMap map[string]string, buildLabelMap map[string]string, forcePull bool) []string {
 
 	var spaceSafeBuildFlags []string
 
@@ -433,16 +359,7 @@ func buildFlagSlice(nocache bool, squash bool, httpProxy string, httpsProxy stri
 	}
 
 	for k, v := range buildArgMap {
-
-		if k != AdditionalPackageBuildArg {
-			spaceSafeBuildFlags = append(spaceSafeBuildFlags, "--build-arg", fmt.Sprintf("%s=%s", k, v))
-		} else {
-			buildOptionPackages = append(buildOptionPackages, strings.Split(v, " ")...)
-		}
-	}
-	if len(buildOptionPackages) > 0 {
-		buildOptionPackages = deDuplicate(buildOptionPackages)
-		spaceSafeBuildFlags = append(spaceSafeBuildFlags, "--build-arg", fmt.Sprintf("%s=%s", AdditionalPackageBuildArg, strings.Join(buildOptionPackages, " ")))
+		spaceSafeBuildFlags = append(spaceSafeBuildFlags, "--build-arg", fmt.Sprintf("%s=%s", k, v))
 	}
 
 	for k, v := range buildLabelMap {
@@ -456,6 +373,32 @@ func buildFlagSlice(nocache bool, squash bool, httpProxy string, httpsProxy stri
 	return spaceSafeBuildFlags
 }
 
+// appendAdditionalPackages appends additional packages  to the ADDITIONAL_PACKAGE build arg.
+// If the ADDITIONAL_PACKAGE build arg is not present, it is created.
+// If the ADDITIONAL_PACKAGE build arg is present, the packages are appended to the list.
+func appendAdditionalPackages(buildArgs map[string]string, additionalPackages []string) map[string]string {
+	for k, v := range buildArgs {
+		if k == AdditionalPackageBuildArg {
+			packages := strings.Split(v, " ")
+			for i := range packages {
+				packages[i] = strings.TrimSpace(packages[i])
+			}
+
+			// Remove empty strings
+			packages = slices.DeleteFunc(packages, func(s string) bool {
+				return len(s) == 0
+			})
+
+			additionalPackages = append(additionalPackages, packages...)
+		}
+	}
+	additionalPackages = deDuplicate(additionalPackages)
+	sort.Strings(additionalPackages)
+	buildArgs[AdditionalPackageBuildArg] = strings.Join(additionalPackages, " ")
+
+	return buildArgs
+}
+
 func ensureHandlerPath(handler string) error {
 	if _, err := os.Stat(handler); err != nil {
 		return err
@@ -509,7 +452,6 @@ func getPackages(availableBuildOptions []stack.BuildOption, requestedBuildOption
 }
 
 func deDuplicate(buildOptPackages []string) []string {
-
 	seenPackages := map[string]bool{}
 	retPackages := []string{}
 
@@ -523,7 +465,3 @@ func deDuplicate(buildOptPackages []string) []string {
 	}
 	return retPackages
 }
-
-func isLanguageTemplate(language string) bool {
-	return strings.ToLower(language) != "dockerfile"
-}