diff --git a/cmd/convert.go b/cmd/convert.go index a1257a3d..8e87627b 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -47,6 +47,7 @@ var ( ConvertReplicas int ConvertController string ConvertPushImage bool + ConvertPushImageRegistry string ConvertOpt kobject.ConvertOptions ConvertYAMLIndent int @@ -88,6 +89,7 @@ var convertCmd = &cobra.Command{ BuildRepo: ConvertBuildRepo, BuildBranch: ConvertBuildBranch, PushImage: ConvertPushImage, + PushImageRegistry: ConvertPushImageRegistry, CreateDeploymentConfig: ConvertDeploymentConfig, EmptyVols: ConvertEmptyVols, Volumes: ConvertVolumes, @@ -147,6 +149,7 @@ func init() { // Standard between the two convertCmd.Flags().StringVar(&ConvertBuild, "build", "none", `Set the type of build ("local"|"build-config"(OpenShift only)|"none")`) convertCmd.Flags().BoolVar(&ConvertPushImage, "push-image", false, "If we should push the docker image we built") + convertCmd.Flags().StringVar(&ConvertPushImageRegistry, "push-image-registry", "", "Specify registry for pushing image, which will override registry from image name.") convertCmd.Flags().BoolVarP(&ConvertYaml, "yaml", "y", false, "Generate resource files into YAML format") convertCmd.Flags().MarkDeprecated("yaml", "YAML is the default format now.") convertCmd.Flags().MarkShorthandDeprecated("y", "YAML is the default format now.") diff --git a/docs/conversion.md b/docs/conversion.md index b15bf9c3..27b73136 100644 --- a/docs/conversion.md +++ b/docs/conversion.md @@ -15,7 +15,7 @@ __Glossary:__ | Keys | V1 | V2 | V3 | Kubernetes / OpenShift | Notes | |------------------------|----|----|----|-------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| -| build | ✓ | ✓ | ✓ | | Builds/Pushes to Docker repository. See `--build` parameter | +| build | ✓ | ✓ | ✓ | | Builds/Pushes to Docker repository. See [user guide on build and push image](https://kompose.io/user-guide/#build-and-push-image) | | | build: context | ✓ | ✓ | ✓ | | | | build: dockerfile | ✓ | ✓ | ✓ | | | | build: args | n | n | n | | | diff --git a/docs/user-guide.md b/docs/user-guide.md index da203e37..e5b06ce3 100755 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -390,6 +390,28 @@ If the Docker Compose file has service name with `_` or `.` in it (eg.`web_servi Please note that changing service name might break some `docker-compose` files. +## Build and push image + +If the Docker Compose file has `build` or `build:context, build:dockerfile` keys, build will run when `--build` specified. + +And Image will push to *docker.io* (default) when `--push-image=true` specified. + +It is possible to push to custom registry by specify `--push-image-registry`, which will override the registry from image name. + +### Authentication on registry + +Kompose uses the docker authentication from file `$DOCKER_CONFIG/config.json`, `$HOME/.docker/config.json`, and `$HOME/.dockercfg` after `docker login`. + +**This only works fine on Linux but macOS would fail when using `"credsStore": "osxkeychain"`.** + +However, there is an approach to push successfully on macOS, by not using `osxkeychain` for `credsStore`. To disable `osxkeychain`: +* remove `credsStore` from `config.json` file, and `docker login` again. +* for some docker desktop versions, there is a setting `Securely store Docker logins in macOS keychain`, which should be unchecked. Then restart docker desktop if needed, and `docker login` again. + +Now `config.json` should contain base64 encoded passwords, then push image should succeed. Working, but not safe though! Use it at your risk. + +For Windows, there is also `credsStore` which is `wincred`. Technically it will fail on authentication as macOS does, but you can try the approach above like macOS too. + ## Docker Compose Versions Kompose supports Docker Compose versions: 1, 2 and 3. We have limited support on versions 2.1 and 3.2 due to their experimental nature. diff --git a/pkg/kobject/kobject.go b/pkg/kobject/kobject.go index 2878db2b..c20b20a7 100644 --- a/pkg/kobject/kobject.go +++ b/pkg/kobject/kobject.go @@ -52,6 +52,7 @@ type ConvertOptions struct { BuildBranch string Build string PushImage bool + PushImageRegistry string CreateChart bool GenerateYaml bool GenerateJSON bool diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index dbde49be..bb386fdf 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -1167,12 +1167,9 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. } // Push the built image to the repo! - if opt.PushImage { - log.Infof("Push image enabled. Attempting to push image '%s'", service.Image) - err = transformer.PushDockerImage(service, name) - if err != nil { - return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name) - } + err = transformer.PushDockerImageWithOpt(service, name, opt) + if err != nil { + return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name) } } @@ -1316,12 +1313,9 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. } // Push the built image to the repo! - if opt.PushImage { - log.Infof("Push image enabled. Attempting to push image '%s'", service.Image) - err = transformer.PushDockerImage(service, name) - if err != nil { - return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name) - } + err = transformer.PushDockerImageWithOpt(service, name, opt) + if err != nil { + return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name) } } diff --git a/pkg/transformer/openshift/openshift.go b/pkg/transformer/openshift/openshift.go index 81033f1c..172d2350 100644 --- a/pkg/transformer/openshift/openshift.go +++ b/pkg/transformer/openshift/openshift.go @@ -318,11 +318,9 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C } // Push the built container to the repo! - if opt.PushImage { - err = transformer.PushDockerImage(service, name) - if err != nil { - log.Fatalf("Unable to push Docker image for service %v: %v", name, err) - } + err = transformer.PushDockerImageWithOpt(service, name, opt) + if err != nil { + log.Fatalf("Unable to push Docker image for service %v: %v", name, err) } } diff --git a/pkg/transformer/utils.go b/pkg/transformer/utils.go index 6627c480..98ca4df6 100644 --- a/pkg/transformer/utils.go +++ b/pkg/transformer/utils.go @@ -308,26 +308,47 @@ func BuildDockerImage(service kobject.ServiceConfig, name string) error { return nil } -// PushDockerImage pushes docker image -func PushDockerImage(service kobject.ServiceConfig, serviceName string) error { - log.Debugf("Pushing Docker image '%s'", service.Image) +// PushDockerImageWithOpt pushes docker image +func PushDockerImageWithOpt(service kobject.ServiceConfig, serviceName string, opt kobject.ConvertOptions) error { + if !opt.PushImage { + // Don't do anything if registry is specified but push is disabled, just WARN about it + if opt.PushImageRegistry != "" { + log.Warnf("Push image registry '%s' is specified but push image is disabled, skipping pushing to repository", opt.PushImageRegistry) + } + return nil + } + + log.Infof("Push image is enabled. Attempting to push image '%s'", service.Image) // Don't do anything if service.Image is blank, but at least WARN about it - // lse, let's push the image + // else, let's push the image if service.Image == "" { log.Warnf("No image name has been passed for service %s, skipping pushing to repository", serviceName) return nil } - // Connect to the Docker client + image, err := docker.ParseImage(service.Image, opt.PushImageRegistry) + if err != nil { + return err + } + client, err := docker.Client() if err != nil { return err } - push := docker.Push{Client: *client} - err = push.PushImage(service.Image) + if opt.PushImageRegistry != "" { + log.Info("Push image registry is specified. Tag the image into registry firstly.") + tag := docker.Tag{Client: *client} + err = tag.TagImage(image) + if err != nil { + return err + } + } + + push := docker.Push{Client: *client} + err = push.PushImage(image) if err != nil { return err } diff --git a/pkg/utils/docker/image.go b/pkg/utils/docker/image.go new file mode 100644 index 00000000..edd1ec48 --- /dev/null +++ b/pkg/utils/docker/image.go @@ -0,0 +1,74 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "path" + + dockerparser "github.com/novln/docker-parser" +) + +// Image contains the basic information parsed from full image name +// see github.com/novln/docker-parser Reference +type Image struct { + Name string // the image's name (ie: debian[:8.2]) + ShortName string // the image's name (ie: debian) + Tag string // the image's tag (or digest) + Registry string // the image's registry. (ie: host[:port]) + Repository string // the image's repository. (ie: registry/name) + Remote string // the image's remote identifier. (ie: registry/name[:tag]) +} + +func NewImageFromParsed(parsed *dockerparser.Reference) Image { + return Image{ + Name: parsed.Name(), + ShortName: parsed.ShortName(), + Tag: parsed.Tag(), + Registry: parsed.Registry(), + Repository: parsed.Repository(), + Remote: parsed.Remote(), + } +} + +// ParseImage Using https://github.com/novln/docker-parser in order to parse the appropriate name and registry. +// 1. Return default registry when the registry is not specified from image +// 2. Return target registry when the registry is specified from command line +func ParseImage(fullImageName string, targetRegistry string) (Image, error) { + var image Image + + // First parse to fill default fields for image + // See github.com/novln/docker-parser/docker/reference.go + parsedImage, err := dockerparser.Parse(fullImageName) + + if err != nil { + return image, err + } + + // Registry from command argument is high priority than parsed from name of image. + if targetRegistry != "" { + // Parse again for validating registry + fullImageName = path.Join(targetRegistry, parsedImage.Name()) + parsedImage, err = dockerparser.Parse(fullImageName) + if err != nil { + return image, err + } + } + + image = NewImageFromParsed(parsedImage) + + return image, nil +} diff --git a/pkg/utils/docker/image_test.go b/pkg/utils/docker/image_test.go new file mode 100644 index 00000000..fb04188b --- /dev/null +++ b/pkg/utils/docker/image_test.go @@ -0,0 +1,112 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "reflect" + "testing" +) + +func TestParseImage(t *testing.T) { + type args struct { + fullImageName string + targetRegistry string + } + tests := []struct { + name string + args args + want Image + wantErr bool + }{ + { + "Given empty registry Then default registry expected", + args{ + "foo/bar", + "", + }, + Image{ + "foo/bar:latest", + "foo/bar", + "latest", + "docker.io", + "docker.io/foo/bar", + "docker.io/foo/bar:latest", + }, + false, + }, + { + "Given registry from image Then parsed registry expected", + args{ + "docker.io/foo/bar", + "", + }, + Image{ + "foo/bar:latest", + "foo/bar", + "latest", + "docker.io", + "docker.io/foo/bar", + "docker.io/foo/bar:latest", + }, + false, + }, + { + "Given target registry Then target registry expected", + args{ + "foo/bar", + "localhost:5000", + }, + Image{ + "foo/bar:latest", + "foo/bar", + "latest", + "localhost:5000", + "localhost:5000/foo/bar", + "localhost:5000/foo/bar:latest", + }, + false, + }, + { + "Given registry from image and target registry Then target registry expected", + args{ + "docker.io/foo/bar", + "localhost:5000", + }, + Image{ + "foo/bar:latest", + "foo/bar", + "latest", + "localhost:5000", + "localhost:5000/foo/bar", + "localhost:5000/foo/bar:latest", + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseImage(tt.args.fullImageName, tt.args.targetRegistry) + if (err != nil) != tt.wantErr { + t.Errorf("ParseImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseImage() got = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/utils/docker/push.go b/pkg/utils/docker/push.go index 11d72d51..bd07b9b3 100644 --- a/pkg/utils/docker/push.go +++ b/pkg/utils/docker/push.go @@ -20,7 +20,6 @@ import ( "bytes" dockerlib "github.com/fsouza/go-dockerclient" - dockerparser "github.com/novln/docker-parser" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -35,23 +34,15 @@ PushImage pushes a Docker image via the Docker API. Takes the image name, parses the URL details and then push based on environment authentication credentials. */ -func (c *Push) PushImage(fullImageName string) error { - outputBuffer := bytes.NewBuffer(nil) - - // Using https://github.com/novln/docker-parser in order to parse the appropriate - // name and registry. - parsedImage, err := dockerparser.Parse(fullImageName) - if err != nil { - return err - } - image, registry := parsedImage.Name(), parsedImage.Registry() - - log.Infof("Pushing image '%s' to registry '%s'", image, registry) +func (c *Push) PushImage(image Image) error { + log.Infof("Pushing image '%s' to registry '%s'", image.Name, image.Registry) // Let's setup the push and authentication options + outputBuffer := bytes.NewBuffer(nil) options := dockerlib.PushImageOptions{ - Name: fullImageName, - Registry: parsedImage.Registry(), + Tag: image.Tag, + Name: image.Repository, + Registry: image.Registry, OutputStream: outputBuffer, } @@ -61,36 +52,44 @@ func (c *Push) PushImage(fullImageName string) error { credentials, err := dockerlib.NewAuthConfigurationsFromDockerCfg() if err != nil { log.Warn(errors.Wrap(err, "Unable to retrieve .docker/config.json authentication details. Check that 'docker login' works successfully on the command line.")) + } else { + handleDockerRegistry(credentials) } - // Fallback to unauthenticated access in case if no auth credentials are retrieved - if credentials == nil || len(credentials.Configs) == 0 { - log.Info("Authentication credentials are not detected. Will try push without authentication.") - credentials = &dockerlib.AuthConfigurations{ - Configs: map[string]dockerlib.AuthConfiguration{ - registry: {}, - }, - } + // Find the authentication matched to registry + auth, ok := credentials.Configs[image.Registry] + if !ok { + // Fallback to unauthenticated access in case if no auth credentials are retrieved + log.Infof("Authentication credential of registry '%s' is not found. Will try push without authentication.", image.Registry) + // Header X-Registry-Auth is required + // Or API error (400): Bad parameters and missing X-Registry-Auth: EOF will throw + // Just to make not empty struct + auth = dockerlib.AuthConfiguration{Username: "docker"} } - // Push the image to the repository (based on the URL) - // We will iterate through all available authentication configurations until we find one that pushes successfully - // and then return nil. - if len(credentials.Configs) > 1 { - log.Info("Multiple authentication credentials detected. Will try each configuration.") + log.Debugf("Pushing image with options %+v", options) + err = c.Client.PushImage(options, auth) + if err != nil { + log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image.Name, image.Registry, err) + return errors.New("unable to push docker image(s). Check that `docker login` works successfully on the command line") } - for k, v := range credentials.Configs { - log.Infof("Attempting authentication credentials '%s", k) - err = c.Client.PushImage(options, v) - if err != nil { - log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image, registry, err) - } else { - log.Debugf("Image '%s' push output:\n%s", image, outputBuffer) - log.Infof("Successfully pushed image '%s' to registry '%s'", image, registry) - return nil - } - } - - return errors.New("unable to push docker image(s). Check that `docker login` works successfully on the command line") + log.Debugf("Image '%+v' push output:\n%s", image, outputBuffer) + log.Infof("Successfully pushed image '%s' to registry '%s'", image.Name, image.Registry) + return nil +} + +// handleDockerRegistry adapt legacy docker registry address +// After docker login to docker.io, there must be https://index.docker.io/v1/ in config.json of authentication +// Reference: https://docs.docker.com/engine/api/v1.23/ +// > However (for legacy reasons) the “official” Docker, Inc. hosted registry +// > must be specified with both a “https://” prefix and a “/v1/” suffix +// > even though Docker will prefer to use the v2 registry API. +func handleDockerRegistry(auth *dockerlib.AuthConfigurations) { + const address = "docker.io" + const legacyAddress = "https://index.docker.io/v1/" + + if legacyAddressConfig, ok := auth.Configs[legacyAddress]; ok { + auth.Configs[address] = legacyAddressConfig + } } diff --git a/pkg/utils/docker/tag.go b/pkg/utils/docker/tag.go new file mode 100644 index 00000000..6c0e6f07 --- /dev/null +++ b/pkg/utils/docker/tag.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + dockerlib "github.com/fsouza/go-dockerclient" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// Tag will provide methods for interaction with API regarding tagging images +type Tag struct { + Client dockerlib.Client +} + +func (c *Tag) TagImage(image Image) error { + options := dockerlib.TagImageOptions{ + Tag: image.Tag, + Repo: image.Repository, + } + + log.Infof("Tagging image '%s' into repository '%s'", image.Name, image.Repository) + err := c.Client.TagImage(image.ShortName, options) + if err != nil { + log.Errorf("Unable to tag image '%s' into repository '%s'. Error: %s", image.Name, image.Registry, err) + return errors.New("unable to tag docker image(s)") + } + + log.Infof("Successfully tagged image '%s'", image.Remote) + return nil +} diff --git a/script/test/cmd/tests_new.sh b/script/test/cmd/tests_new.sh index 53eb2fc3..912a9814 100755 --- a/script/test/cmd/tests_new.sh +++ b/script/test/cmd/tests_new.sh @@ -95,3 +95,10 @@ convert::check_artifacts_generated "kompose --build local -f $KOMPOSE_ROOT/scrip # Test build v3 relative compose file with context relative_path=$(realpath --relative-to="$PWD" "$KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-v3.yml") convert::check_artifacts_generated "kompose --build local -f $relative_path convert -o $TEMP_DIR/output_file" "$TEMP_DIR/output_file" + +##### +# Test the build config with push image +# see tests_push_image.sh for local push test +# Should warn when push image disabled +cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-build-no-image.yml -o $TEMP_DIR/output_file convert --build=local --push-image-registry=whatever" +convert::expect_warning "$cmd" "Push image registry 'whatever' is specified but push image is disabled, skipping pushing to repository" diff --git a/script/test/cmd/tests_push_image.sh b/script/test/cmd/tests_push_image.sh new file mode 100755 index 00000000..d9d5cc4f --- /dev/null +++ b/script/test/cmd/tests_push_image.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing pe#rmissions and +# limitations under the License. + +# Here are tests for pushing image on authentication and custom registry. +# These tests only work on local for authentication inconvenient. +# Prerequisites: +# * `docker.io` account and docker login successfully +# * custom registry which login as well. Or a local hosted registry. +# * `jq` installed + +# Variables +TEMP_DIR="/tmp/kompose" +mkdir -p $TEMP_DIR +mkdir -p "$TEMP_DIR/build" + +DOCKER_LOGIN_USER="lexcao" # TODO change this to your account for pushing to docker.io +COMPOSE_FILE="$TEMP_DIR/docker-compose-push.yml" +BUILD_FILE="$TEMP_DIR/build/Dockerfile" +CUSTOM_REGISTRY="localhost:5000" # TODO change this to your local registry + +# Custom compose file based on parameter +build_file_content="FROM busybox + RUN touch /test" +echo "$build_file_content" >> "$BUILD_FILE" + +compose_file_content="version: \"2\" + +services: + foo: + build: \"./build\" + image: docker.io/$DOCKER_LOGIN_USER/foobar" + +echo "$compose_file_content" >> "$COMPOSE_FILE" + +# Some helper functions +function get_docker_hub_tag() { + local image=$1 + local tag=$2 + curl "https://hub.docker.com/v2/repositories/$image/tags/$tag/" | jq +} + +function get_custom_registry_tag() { + local image=$1 + local tag=$2 + curl "http://$CUSTOM_REGISTRY/v2/$image/manifests/$tag" -I +} + +################################################################################### +cmd="kompose convert -f $COMPOSE_FILE -o $TEMP_DIR/output_file --build=local --push-image=true" + +printf "Push image without custom registry default to docker.io\n" +echo "executing cmd '$cmd'" +$cmd +printf "\nVerify push success...\n" +get_docker_hub_tag "$DOCKER_LOGIN_USER/foobar" "latest" + +####### +printf "\nPush image with custom registry\n" +cmd="$cmd --push-image-registry=$CUSTOM_REGISTRY" +echo "executing cmd '$cmd'" +$cmd +#kompose convert -f "$COMPOSE_FILE" -o "$TEMP_DIR/output_file" --build=local --push-image=true +printf "\nVerify push success...\n" +get_custom_registry_tag "$DOCKER_LOGIN_USER/foobar" "latest" + +# Clean resource +rm -rf $TEMP_DIR