Merge pull request #1409 from lexcao/feat/image/registry

Support custom registry on pushing image

cmd/convert.go

@@ -47,6 +47,7 @@ var (
 	ConvertReplicas              int
 	ConvertController            string
 	ConvertPushImage             bool
+	ConvertPushImageRegistry     string
 	ConvertOpt                   kobject.ConvertOptions
 	ConvertYAMLIndent            int
 
@@ -88,6 +89,7 @@ var convertCmd = &cobra.Command{
 			BuildRepo:                   ConvertBuildRepo,
 			BuildBranch:                 ConvertBuildBranch,
 			PushImage:                   ConvertPushImage,
+			PushImageRegistry:           ConvertPushImageRegistry,
 			CreateDeploymentConfig:      ConvertDeploymentConfig,
 			EmptyVols:                   ConvertEmptyVols,
 			Volumes:                     ConvertVolumes,
@@ -147,6 +149,7 @@ func init() {
 	// Standard between the two
 	convertCmd.Flags().StringVar(&ConvertBuild, "build", "none", `Set the type of build ("local"|"build-config"(OpenShift only)|"none")`)
 	convertCmd.Flags().BoolVar(&ConvertPushImage, "push-image", false, "If we should push the docker image we built")
+	convertCmd.Flags().StringVar(&ConvertPushImageRegistry, "push-image-registry", "", "Specify registry for pushing image, which will override registry from image name.")
 	convertCmd.Flags().BoolVarP(&ConvertYaml, "yaml", "y", false, "Generate resource files into YAML format")
 	convertCmd.Flags().MarkDeprecated("yaml", "YAML is the default format now.")
 	convertCmd.Flags().MarkShorthandDeprecated("y", "YAML is the default format now.")

docs/conversion.md

@@ -15,7 +15,7 @@ __Glossary:__
 
 | Keys                   | V1 | V2 | V3 | Kubernetes / OpenShift                                      | Notes                                                                                                          |
 |------------------------|----|----|----|-------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|
-| build                  | ✓  | ✓  | ✓  |                                                             | Builds/Pushes to Docker repository. See `--build` parameter                                                    |
+| build                  | ✓  | ✓  | ✓  |                                                             | Builds/Pushes to Docker repository. See [user guide on build and push image](https://kompose.io/user-guide/#build-and-push-image) |                                                    |
 | build: context         | ✓  | ✓  | ✓  |                                                             |                                                                                                                |
 | build: dockerfile      | ✓  | ✓  | ✓  |                                                             |                                                                                                                |
 | build: args            | n  | n  | n  |                                                             |                                                                                                                |

docs/user-guide.md

@@ -390,6 +390,28 @@ If the Docker Compose file has service name with `_` or `.` in it (eg.`web_servi
 
 Please note that changing service name might break some `docker-compose` files.
 
+## Build and push image
+
+If the Docker Compose file has `build` or `build:context, build:dockerfile` keys, build will run when `--build` specified.
+
+And Image will push to *docker.io* (default) when `--push-image=true` specified.
+
+It is possible to push to custom registry by specify `--push-image-registry`, which will override the registry from image name. 
+
+### Authentication on registry
+
+Kompose uses the docker authentication from file `$DOCKER_CONFIG/config.json`, `$HOME/.docker/config.json`, and `$HOME/.dockercfg` after `docker login`.
+
+**This only works fine on Linux but macOS would fail when using `"credsStore": "osxkeychain"`.**
+
+However, there is an approach to push successfully on macOS, by not using `osxkeychain` for `credsStore`. To disable `osxkeychain`:
+* remove `credsStore` from `config.json` file, and `docker login` again.
+* for some docker desktop versions, there is a setting `Securely store Docker logins in macOS keychain`, which should be unchecked. Then restart docker desktop if needed, and `docker login` again.
+
+Now `config.json` should contain base64 encoded passwords, then push image should succeed. Working, but not safe though! Use it at your risk.
+
+For Windows, there is also `credsStore` which is `wincred`. Technically it will fail on authentication as macOS does, but you can try the approach above like macOS too.   
+
 ## Docker Compose Versions
 
 Kompose supports Docker Compose versions: 1, 2 and 3. We have limited support on versions 2.1 and 3.2 due to their experimental nature.

pkg/kobject/kobject.go

@@ -52,6 +52,7 @@ type ConvertOptions struct {
 	BuildBranch                 string
 	Build                       string
 	PushImage                   bool
+	PushImageRegistry           string
 	CreateChart                 bool
 	GenerateYaml                bool
 	GenerateJSON                bool

pkg/transformer/kubernetes/kubernetes.go

@@ -1167,14 +1167,11 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject.
 				}
 
 				// Push the built image to the repo!
-				if opt.PushImage {
+				err = transformer.PushDockerImageWithOpt(service, name, opt)
-					log.Infof("Push image enabled. Attempting to push image '%s'", service.Image)
-					err = transformer.PushDockerImage(service, name)
 				if err != nil {
 					return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name)
 				}
 			}
-			}
 
 			podSpec := PodSpec{}
 
@@ -1316,14 +1313,11 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject.
 				}
 
 				// Push the built image to the repo!
-				if opt.PushImage {
+				err = transformer.PushDockerImageWithOpt(service, name, opt)
-					log.Infof("Push image enabled. Attempting to push image '%s'", service.Image)
-					err = transformer.PushDockerImage(service, name)
 				if err != nil {
 					return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name)
 				}
 			}
-			}
 
 			// Generate pod only and nothing more
 			if (service.Restart == "no" || service.Restart == "on-failure") && !opt.IsPodController() {

pkg/transformer/openshift/openshift.go

@@ -318,13 +318,11 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C
 			}
 
 			// Push the built container to the repo!
-			if opt.PushImage {
+			err = transformer.PushDockerImageWithOpt(service, name, opt)
-				err = transformer.PushDockerImage(service, name)
 			if err != nil {
 				log.Fatalf("Unable to push Docker image for service %v: %v", name, err)
 			}
 		}
-		}
 
 		// Generate pod only and nothing more
 		if service.Restart == "no" || service.Restart == "on-failure" {

pkg/transformer/utils.go

@@ -308,29 +308,50 @@ func BuildDockerImage(service kobject.ServiceConfig, name string) error {
 	return nil
 }
 
-// PushDockerImage pushes docker image
+// PushDockerImageWithOpt pushes docker image
-func PushDockerImage(service kobject.ServiceConfig, serviceName string) error {
+func PushDockerImageWithOpt(service kobject.ServiceConfig, serviceName string, opt kobject.ConvertOptions) error {
-	log.Debugf("Pushing Docker image '%s'", service.Image)
+	if !opt.PushImage {
+		// Don't do anything if registry is specified but push is disabled, just WARN about it
+		if opt.PushImageRegistry != "" {
+			log.Warnf("Push image registry '%s' is specified but push image is disabled, skipping pushing to repository", opt.PushImageRegistry)
+		}
+		return nil
+	}
+
+	log.Infof("Push image is enabled. Attempting to push image '%s'", service.Image)
 
 	// Don't do anything if service.Image is blank, but at least WARN about it
-	// lse, let's push the image
+	// else, let's push the image
 	if service.Image == "" {
 		log.Warnf("No image name has been passed for service %s, skipping pushing to repository", serviceName)
 		return nil
 	}
 
-	// Connect to the Docker client
+	image, err := docker.ParseImage(service.Image, opt.PushImageRegistry)
+	if err != nil {
+		return err
+	}
+
 	client, err := docker.Client()
 	if err != nil {
 		return err
 	}
 
-	push := docker.Push{Client: *client}
+	if opt.PushImageRegistry != "" {
-	err = push.PushImage(service.Image)
+		log.Info("Push image registry is specified. Tag the image into registry firstly.")
+		tag := docker.Tag{Client: *client}
+		err = tag.TagImage(image)
 
 		if err != nil {
 			return err
 		}
+	}
+
+	push := docker.Push{Client: *client}
+	err = push.PushImage(image)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }

pkg/utils/docker/image.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	"path"
+
+	dockerparser "github.com/novln/docker-parser"
+)
+
+// Image contains the basic information parsed from full image name
+// see github.com/novln/docker-parser Reference
+type Image struct {
+	Name       string // the image's name (ie: debian[:8.2])
+	ShortName  string // the image's name (ie: debian)
+	Tag        string // the image's tag (or digest)
+	Registry   string // the image's registry. (ie: host[:port])
+	Repository string // the image's repository. (ie: registry/name)
+	Remote     string // the image's remote identifier. (ie: registry/name[:tag])
+}
+
+func NewImageFromParsed(parsed *dockerparser.Reference) Image {
+	return Image{
+		Name:       parsed.Name(),
+		ShortName:  parsed.ShortName(),
+		Tag:        parsed.Tag(),
+		Registry:   parsed.Registry(),
+		Repository: parsed.Repository(),
+		Remote:     parsed.Remote(),
+	}
+}
+
+// ParseImage Using https://github.com/novln/docker-parser in order to parse the appropriate name and registry.
+// 1. Return default registry when the registry is not specified from image
+// 2. Return target registry when the registry is specified from command line
+func ParseImage(fullImageName string, targetRegistry string) (Image, error) {
+	var image Image
+
+	// First parse to fill default fields for image
+	// See github.com/novln/docker-parser/docker/reference.go
+	parsedImage, err := dockerparser.Parse(fullImageName)
+
+	if err != nil {
+		return image, err
+	}
+
+	// Registry from command argument is high priority than parsed from name of image.
+	if targetRegistry != "" {
+		// Parse again for validating registry
+		fullImageName = path.Join(targetRegistry, parsedImage.Name())
+		parsedImage, err = dockerparser.Parse(fullImageName)
+		if err != nil {
+			return image, err
+		}
+	}
+
+	image = NewImageFromParsed(parsedImage)
+
+	return image, nil
+}

pkg/utils/docker/image_test.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseImage(t *testing.T) {
+	type args struct {
+		fullImageName  string
+		targetRegistry string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    Image
+		wantErr bool
+	}{
+		{
+			"Given empty registry Then default registry expected",
+			args{
+				"foo/bar",
+				"",
+			},
+			Image{
+				"foo/bar:latest",
+				"foo/bar",
+				"latest",
+				"docker.io",
+				"docker.io/foo/bar",
+				"docker.io/foo/bar:latest",
+			},
+			false,
+		},
+		{
+			"Given registry from image Then parsed registry expected",
+			args{
+				"docker.io/foo/bar",
+				"",
+			},
+			Image{
+				"foo/bar:latest",
+				"foo/bar",
+				"latest",
+				"docker.io",
+				"docker.io/foo/bar",
+				"docker.io/foo/bar:latest",
+			},
+			false,
+		},
+		{
+			"Given target registry Then target registry expected",
+			args{
+				"foo/bar",
+				"localhost:5000",
+			},
+			Image{
+				"foo/bar:latest",
+				"foo/bar",
+				"latest",
+				"localhost:5000",
+				"localhost:5000/foo/bar",
+				"localhost:5000/foo/bar:latest",
+			},
+			false,
+		},
+		{
+			"Given registry from image and target registry Then target registry expected",
+			args{
+				"docker.io/foo/bar",
+				"localhost:5000",
+			},
+			Image{
+				"foo/bar:latest",
+				"foo/bar",
+				"latest",
+				"localhost:5000",
+				"localhost:5000/foo/bar",
+				"localhost:5000/foo/bar:latest",
+			},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseImage(tt.args.fullImageName, tt.args.targetRegistry)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseImage() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ParseImage() got = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}

pkg/utils/docker/push.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 
 	dockerlib "github.com/fsouza/go-dockerclient"
-	dockerparser "github.com/novln/docker-parser"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
@@ -35,23 +34,15 @@ PushImage pushes a Docker image via the Docker API. Takes the image name,
 parses the URL details and then push based on environment authentication
 credentials.
 */
-func (c *Push) PushImage(fullImageName string) error {
+func (c *Push) PushImage(image Image) error {
-	outputBuffer := bytes.NewBuffer(nil)
+	log.Infof("Pushing image '%s' to registry '%s'", image.Name, image.Registry)
-
-	// Using https://github.com/novln/docker-parser in order to parse the appropriate
-	// name and registry.
-	parsedImage, err := dockerparser.Parse(fullImageName)
-	if err != nil {
-		return err
-	}
-	image, registry := parsedImage.Name(), parsedImage.Registry()
-
-	log.Infof("Pushing image '%s' to registry '%s'", image, registry)
 
 	// Let's setup the push and authentication options
+	outputBuffer := bytes.NewBuffer(nil)
 	options := dockerlib.PushImageOptions{
-		Name:         fullImageName,
+		Tag:          image.Tag,
-		Registry:     parsedImage.Registry(),
+		Name:         image.Repository,
+		Registry:     image.Registry,
 		OutputStream: outputBuffer,
 	}
 
@@ -61,36 +52,44 @@ func (c *Push) PushImage(fullImageName string) error {
 	credentials, err := dockerlib.NewAuthConfigurationsFromDockerCfg()
 	if err != nil {
 		log.Warn(errors.Wrap(err, "Unable to retrieve .docker/config.json authentication details. Check that 'docker login' works successfully on the command line."))
-	}
-
-	// Fallback to unauthenticated access in case if no auth credentials are retrieved
-	if credentials == nil || len(credentials.Configs) == 0 {
-		log.Info("Authentication credentials are not detected. Will try push without authentication.")
-		credentials = &dockerlib.AuthConfigurations{
-			Configs: map[string]dockerlib.AuthConfiguration{
-				registry: {},
-			},
-		}
-	}
-
-	// Push the image to the repository (based on the URL)
-	// We will iterate through all available authentication configurations until we find one that pushes successfully
-	// and then return nil.
-	if len(credentials.Configs) > 1 {
-		log.Info("Multiple authentication credentials detected. Will try each configuration.")
-	}
-
-	for k, v := range credentials.Configs {
-		log.Infof("Attempting authentication credentials '%s", k)
-		err = c.Client.PushImage(options, v)
-		if err != nil {
-			log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image, registry, err)
 	} else {
-			log.Debugf("Image '%s' push output:\n%s", image, outputBuffer)
+		handleDockerRegistry(credentials)
-			log.Infof("Successfully pushed image '%s' to registry '%s'", image, registry)
-			return nil
-		}
 	}
 
+	// Find the authentication matched to registry
+	auth, ok := credentials.Configs[image.Registry]
+	if !ok {
+		// Fallback to unauthenticated access in case if no auth credentials are retrieved
+		log.Infof("Authentication credential of registry '%s' is not found. Will try push without authentication.", image.Registry)
+		// Header X-Registry-Auth is required
+		// Or API error (400): Bad parameters and missing X-Registry-Auth: EOF will throw
+		// Just to make not empty struct
+		auth = dockerlib.AuthConfiguration{Username: "docker"}
+	}
+
+	log.Debugf("Pushing image with options %+v", options)
+	err = c.Client.PushImage(options, auth)
+	if err != nil {
+		log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image.Name, image.Registry, err)
 		return errors.New("unable to push docker image(s). Check that `docker login` works successfully on the command line")
 	}
+
+	log.Debugf("Image '%+v' push output:\n%s", image, outputBuffer)
+	log.Infof("Successfully pushed image '%s' to registry '%s'", image.Name, image.Registry)
+	return nil
+}
+
+// handleDockerRegistry adapt legacy docker registry address
+// After docker login to docker.io, there must be https://index.docker.io/v1/ in config.json of authentication
+// Reference: https://docs.docker.com/engine/api/v1.23/
+//     > However (for legacy reasons) the “official” Docker, Inc. hosted registry
+//     > must be specified with both a “https://” prefix and a “/v1/” suffix
+//     > even though Docker will prefer to use the v2 registry API.
+func handleDockerRegistry(auth *dockerlib.AuthConfigurations) {
+	const address = "docker.io"
+	const legacyAddress = "https://index.docker.io/v1/"
+
+	if legacyAddressConfig, ok := auth.Configs[legacyAddress]; ok {
+		auth.Configs[address] = legacyAddressConfig
+	}
+}

pkg/utils/docker/tag.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	dockerlib "github.com/fsouza/go-dockerclient"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+// Tag will provide methods for interaction with API regarding tagging images
+type Tag struct {
+	Client dockerlib.Client
+}
+
+func (c *Tag) TagImage(image Image) error {
+	options := dockerlib.TagImageOptions{
+		Tag:  image.Tag,
+		Repo: image.Repository,
+	}
+
+	log.Infof("Tagging image '%s' into repository '%s'", image.Name, image.Repository)
+	err := c.Client.TagImage(image.ShortName, options)
+	if err != nil {
+		log.Errorf("Unable to tag image '%s' into repository '%s'. Error: %s", image.Name, image.Registry, err)
+		return errors.New("unable to tag docker image(s)")
+	}
+
+	log.Infof("Successfully tagged image '%s'", image.Remote)
+	return nil
+}

script/test/cmd/tests_new.sh

@@ -95,3 +95,10 @@ convert::check_artifacts_generated "kompose --build local -f $KOMPOSE_ROOT/scrip
 # Test build v3 relative compose file with context
 relative_path=$(realpath --relative-to="$PWD" "$KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-v3.yml")
 convert::check_artifacts_generated "kompose --build local -f $relative_path convert -o $TEMP_DIR/output_file" "$TEMP_DIR/output_file"
+
+#####
+# Test the build config with push image
+# see tests_push_image.sh for local push test
+# Should warn when push image disabled
+cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-build-no-image.yml -o $TEMP_DIR/output_file convert --build=local --push-image-registry=whatever"
+convert::expect_warning "$cmd" "Push image registry 'whatever' is specified but push image is disabled, skipping pushing to repository"

script/test/cmd/tests_push_image.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing pe#rmissions and
+# limitations under the License.
+
+# Here are tests for pushing image on authentication and custom registry.
+# These tests only work on local for authentication inconvenient.
+# Prerequisites:
+# * `docker.io` account and docker login successfully
+# * custom registry which login as well. Or a local hosted registry.
+# * `jq` installed
+
+# Variables
+TEMP_DIR="/tmp/kompose"
+mkdir -p $TEMP_DIR
+mkdir -p "$TEMP_DIR/build"
+
+DOCKER_LOGIN_USER="lexcao" # TODO change this to your account for pushing to docker.io
+COMPOSE_FILE="$TEMP_DIR/docker-compose-push.yml"
+BUILD_FILE="$TEMP_DIR/build/Dockerfile"
+CUSTOM_REGISTRY="localhost:5000" # TODO change this to your local registry
+
+# Custom compose file based on parameter
+build_file_content="FROM busybox
+                    RUN touch /test"
+echo "$build_file_content" >> "$BUILD_FILE"
+
+compose_file_content="version: \"2\"
+
+services:
+    foo:
+        build: \"./build\"
+        image: docker.io/$DOCKER_LOGIN_USER/foobar"
+
+echo "$compose_file_content" >> "$COMPOSE_FILE"
+
+# Some helper functions
+function get_docker_hub_tag() {
+  local image=$1
+  local tag=$2
+  curl "https://hub.docker.com/v2/repositories/$image/tags/$tag/" | jq
+}
+
+function get_custom_registry_tag() {
+  local image=$1
+  local tag=$2
+  curl "http://$CUSTOM_REGISTRY/v2/$image/manifests/$tag" -I
+}
+
+###################################################################################
+cmd="kompose convert -f $COMPOSE_FILE -o $TEMP_DIR/output_file --build=local --push-image=true"
+
+printf "Push image without custom registry default to docker.io\n"
+echo "executing cmd '$cmd'"
+$cmd
+printf "\nVerify push success...\n"
+get_docker_hub_tag "$DOCKER_LOGIN_USER/foobar" "latest"
+
+#######
+printf "\nPush image with custom registry\n"
+cmd="$cmd --push-image-registry=$CUSTOM_REGISTRY"
+echo "executing cmd '$cmd'"
+$cmd
+#kompose convert -f "$COMPOSE_FILE" -o "$TEMP_DIR/output_file" --build=local --push-image=true
+printf "\nVerify push success...\n"
+get_custom_registry_tag "$DOCKER_LOGIN_USER/foobar" "latest"
+
+# Clean resource
+rm -rf $TEMP_DIR