deep-stack/kompose
7
0
Fork 0
forked from LaconicNetwork/kompose
Code Pull Requests Activity

Merge pull request #1409 from lexcao/feat/image/registry

Browse Source 
Support custom registry on pushing image
This commit is contained in:
Kubernetes Prow Robot 2021-08-11 08:26:47 -07:00 committed by GitHub
commit b02ad5de69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 422 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/convert.go
View File

@ -47,6 +47,7 @@ var (
ConvertReplicas int
ConvertController string
ConvertPushImage bool
ConvertPushImageRegistry string
ConvertOpt kobject.ConvertOptions
ConvertYAMLIndent int
@ -88,6 +89,7 @@ var convertCmd = &cobra.Command{
BuildRepo: ConvertBuildRepo,
BuildBranch: ConvertBuildBranch,
PushImage: ConvertPushImage,
PushImageRegistry: ConvertPushImageRegistry,
CreateDeploymentConfig: ConvertDeploymentConfig,
EmptyVols: ConvertEmptyVols,
Volumes: ConvertVolumes,
@ -147,6 +149,7 @@ func init() {
// Standard between the two
convertCmd.Flags().StringVar(&ConvertBuild, "build", "none", `Set the type of build ("local"|"build-config"(OpenShift only)|"none")`)
convertCmd.Flags().BoolVar(&ConvertPushImage, "push-image", false, "If we should push the docker image we built")
convertCmd.Flags().StringVar(&ConvertPushImageRegistry, "push-image-registry", "", "Specify registry for pushing image, which will override registry from image name.")
convertCmd.Flags().BoolVarP(&ConvertYaml, "yaml", "y", false, "Generate resource files into YAML format")
convertCmd.Flags().MarkDeprecated("yaml", "YAML is the default format now.")
convertCmd.Flags().MarkShorthandDeprecated("y", "YAML is the default format now.")

2
docs/conversion.md
View File

@ -15,7 +15,7 @@ __Glossary:__
| Keys | V1 | V2 | V3 | Kubernetes / OpenShift | Notes |
|------------------------|----|----|----|-------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|
| build | ✓ | ✓ | ✓ | | Builds/Pushes to Docker repository. See `--build` parameter |
| build | ✓ | ✓ | ✓ | | Builds/Pushes to Docker repository. See [user guide on build and push image](https://kompose.io/user-guide/#build-and-push-image) | |
| build: context | ✓ | ✓ | ✓ | | |
| build: dockerfile | ✓ | ✓ | ✓ | | |
| build: args | n | n | n | | |

22
docs/user-guide.md
View File

@ -390,6 +390,28 @@ If the Docker Compose file has service name with `_` or `.` in it (eg.`web_servi
Please note that changing service name might break some `docker-compose` files.
## Build and push image
If the Docker Compose file has `build` or `build:context, build:dockerfile` keys, build will run when `--build` specified.
And Image will push to *docker.io* (default) when `--push-image=true` specified.
It is possible to push to custom registry by specify `--push-image-registry`, which will override the registry from image name.
### Authentication on registry
Kompose uses the docker authentication from file `$DOCKER_CONFIG/config.json`, `$HOME/.docker/config.json`, and `$HOME/.dockercfg` after `docker login`.
**This only works fine on Linux but macOS would fail when using `"credsStore": "osxkeychain"`.**
However, there is an approach to push successfully on macOS, by not using `osxkeychain` for `credsStore`. To disable `osxkeychain`:
* remove `credsStore` from `config.json` file, and `docker login` again.
* for some docker desktop versions, there is a setting `Securely store Docker logins in macOS keychain`, which should be unchecked. Then restart docker desktop if needed, and `docker login` again.
Now `config.json` should contain base64 encoded passwords, then push image should succeed. Working, but not safe though! Use it at your risk.
For Windows, there is also `credsStore` which is `wincred`. Technically it will fail on authentication as macOS does, but you can try the approach above like macOS too.
## Docker Compose Versions
Kompose supports Docker Compose versions: 1, 2 and 3. We have limited support on versions 2.1 and 3.2 due to their experimental nature.

1
pkg/kobject/kobject.go
View File

@ -52,6 +52,7 @@ type ConvertOptions struct {
BuildBranch string
Build string
PushImage bool
PushImageRegistry string
CreateChart bool
GenerateYaml bool
GenerateJSON bool

10
pkg/transformer/kubernetes/kubernetes.go
View File

@ -1167,14 +1167,11 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject.
}
// Push the built image to the repo!
if opt.PushImage {
log.Infof("Push image enabled. Attempting to push image '%s'", service.Image)
err = transformer.PushDockerImage(service, name)
err = transformer.PushDockerImageWithOpt(service, name, opt)
if err != nil {
return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name)
}
}
}
podSpec := PodSpec{}
@ -1316,14 +1313,11 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject.
}
// Push the built image to the repo!
if opt.PushImage {
log.Infof("Push image enabled. Attempting to push image '%s'", service.Image)
err = transformer.PushDockerImage(service, name)
err = transformer.PushDockerImageWithOpt(service, name, opt)
if err != nil {
return nil, errors.Wrapf(err, "Unable to push Docker image for service %v", name)
}
}
}
// Generate pod only and nothing more
if (service.Restart == "no" || service.Restart == "on-failure") && !opt.IsPodController() {

4
pkg/transformer/openshift/openshift.go
View File

@ -318,13 +318,11 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C
}
// Push the built container to the repo!
if opt.PushImage {
err = transformer.PushDockerImage(service, name)
err = transformer.PushDockerImageWithOpt(service, name, opt)
if err != nil {
log.Fatalf("Unable to push Docker image for service %v: %v", name, err)
}
}
}
// Generate pod only and nothing more
if service.Restart == "no" || service.Restart == "on-failure" {

35
pkg/transformer/utils.go
View File

@ -308,29 +308,50 @@ func BuildDockerImage(service kobject.ServiceConfig, name string) error {
return nil
}
// PushDockerImage pushes docker image
func PushDockerImage(service kobject.ServiceConfig, serviceName string) error {
log.Debugf("Pushing Docker image '%s'", service.Image)
// PushDockerImageWithOpt pushes docker image
func PushDockerImageWithOpt(service kobject.ServiceConfig, serviceName string, opt kobject.ConvertOptions) error {
if !opt.PushImage {
// Don't do anything if registry is specified but push is disabled, just WARN about it
if opt.PushImageRegistry != "" {
log.Warnf("Push image registry '%s' is specified but push image is disabled, skipping pushing to repository", opt.PushImageRegistry)
}
return nil
}
log.Infof("Push image is enabled. Attempting to push image '%s'", service.Image)
// Don't do anything if service.Image is blank, but at least WARN about it
// lse, let's push the image
// else, let's push the image
if service.Image == "" {
log.Warnf("No image name has been passed for service %s, skipping pushing to repository", serviceName)
return nil
}
// Connect to the Docker client
image, err := docker.ParseImage(service.Image, opt.PushImageRegistry)
if err != nil {
return err
}
client, err := docker.Client()
if err != nil {
return err
}
push := docker.Push{Client: *client}
err = push.PushImage(service.Image)
if opt.PushImageRegistry != "" {
log.Info("Push image registry is specified. Tag the image into registry firstly.")
tag := docker.Tag{Client: *client}
err = tag.TagImage(image)
if err != nil {
return err
}
}
push := docker.Push{Client: *client}
err = push.PushImage(image)
if err != nil {
return err
}
return nil
}

74
pkg/utils/docker/image.go Normal file
View File

@ -0,0 +1,74 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"path"
dockerparser "github.com/novln/docker-parser"
)
// Image contains the basic information parsed from full image name
// see github.com/novln/docker-parser Reference
type Image struct {
Name string // the image's name (ie: debian[:8.2])
ShortName string // the image's name (ie: debian)
Tag string // the image's tag (or digest)
Registry string // the image's registry. (ie: host[:port])
Repository string // the image's repository. (ie: registry/name)
Remote string // the image's remote identifier. (ie: registry/name[:tag])
}
func NewImageFromParsed(parsed *dockerparser.Reference) Image {
return Image{
Name: parsed.Name(),
ShortName: parsed.ShortName(),
Tag: parsed.Tag(),
Registry: parsed.Registry(),
Repository: parsed.Repository(),
Remote: parsed.Remote(),
}
}
// ParseImage Using https://github.com/novln/docker-parser in order to parse the appropriate name and registry.
// 1. Return default registry when the registry is not specified from image
// 2. Return target registry when the registry is specified from command line
func ParseImage(fullImageName string, targetRegistry string) (Image, error) {
var image Image
// First parse to fill default fields for image
// See github.com/novln/docker-parser/docker/reference.go
parsedImage, err := dockerparser.Parse(fullImageName)
if err != nil {
return image, err
}
// Registry from command argument is high priority than parsed from name of image.
if targetRegistry != "" {
// Parse again for validating registry
fullImageName = path.Join(targetRegistry, parsedImage.Name())
parsedImage, err = dockerparser.Parse(fullImageName)
if err != nil {
return image, err
}
}
image = NewImageFromParsed(parsedImage)
return image, nil
}

112
pkg/utils/docker/image_test.go Normal file
View File

@ -0,0 +1,112 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"reflect"
"testing"
)
func TestParseImage(t *testing.T) {
type args struct {
fullImageName string
targetRegistry string
}
tests := []struct {
name string
args args
want Image
wantErr bool
}{
{
"Given empty registry Then default registry expected",
args{
"foo/bar",
"",
},
Image{
"foo/bar:latest",
"foo/bar",
"latest",
"docker.io",
"docker.io/foo/bar",
"docker.io/foo/bar:latest",
},
false,
},
{
"Given registry from image Then parsed registry expected",
args{
"docker.io/foo/bar",
"",
},
Image{
"foo/bar:latest",
"foo/bar",
"latest",
"docker.io",
"docker.io/foo/bar",
"docker.io/foo/bar:latest",
},
false,
},
{
"Given target registry Then target registry expected",
args{
"foo/bar",
"localhost:5000",
},
Image{
"foo/bar:latest",
"foo/bar",
"latest",
"localhost:5000",
"localhost:5000/foo/bar",
"localhost:5000/foo/bar:latest",
},
false,
},
{
"Given registry from image and target registry Then target registry expected",
args{
"docker.io/foo/bar",
"localhost:5000",
},
Image{
"foo/bar:latest",
"foo/bar",
"latest",
"localhost:5000",
"localhost:5000/foo/bar",
"localhost:5000/foo/bar:latest",
},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseImage(tt.args.fullImageName, tt.args.targetRegistry)
if (err != nil) != tt.wantErr {
t.Errorf("ParseImage() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ParseImage() got = %+v, want %+v", got, tt.want)
}
})
}
}

85
pkg/utils/docker/push.go
View File

@ -20,7 +20,6 @@ import (
"bytes"
dockerlib "github.com/fsouza/go-dockerclient"
dockerparser "github.com/novln/docker-parser"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
@ -35,23 +34,15 @@ PushImage pushes a Docker image via the Docker API. Takes the image name,
parses the URL details and then push based on environment authentication
credentials.
*/
func (c *Push) PushImage(fullImageName string) error {
outputBuffer := bytes.NewBuffer(nil)
// Using https://github.com/novln/docker-parser in order to parse the appropriate
// name and registry.
parsedImage, err := dockerparser.Parse(fullImageName)
if err != nil {
return err
}
image, registry := parsedImage.Name(), parsedImage.Registry()
log.Infof("Pushing image '%s' to registry '%s'", image, registry)
func (c *Push) PushImage(image Image) error {
log.Infof("Pushing image '%s' to registry '%s'", image.Name, image.Registry)
// Let's setup the push and authentication options
outputBuffer := bytes.NewBuffer(nil)
options := dockerlib.PushImageOptions{
Name: fullImageName,
Registry: parsedImage.Registry(),
Tag: image.Tag,
Name: image.Repository,
Registry: image.Registry,
OutputStream: outputBuffer,
}
@ -61,36 +52,44 @@ func (c *Push) PushImage(fullImageName string) error {
credentials, err := dockerlib.NewAuthConfigurationsFromDockerCfg()
if err != nil {
log.Warn(errors.Wrap(err, "Unable to retrieve .docker/config.json authentication details. Check that 'docker login' works successfully on the command line."))
}
// Fallback to unauthenticated access in case if no auth credentials are retrieved
if credentials == nil || len(credentials.Configs) == 0 {
log.Info("Authentication credentials are not detected. Will try push without authentication.")
credentials = &dockerlib.AuthConfigurations{
Configs: map[string]dockerlib.AuthConfiguration{
registry: {},
},
}
}
// Push the image to the repository (based on the URL)
// We will iterate through all available authentication configurations until we find one that pushes successfully
// and then return nil.
if len(credentials.Configs) > 1 {
log.Info("Multiple authentication credentials detected. Will try each configuration.")
}
for k, v := range credentials.Configs {
log.Infof("Attempting authentication credentials '%s", k)
err = c.Client.PushImage(options, v)
if err != nil {
log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image, registry, err)
} else {
log.Debugf("Image '%s' push output:\n%s", image, outputBuffer)
log.Infof("Successfully pushed image '%s' to registry '%s'", image, registry)
return nil
}
handleDockerRegistry(credentials)
}
// Find the authentication matched to registry
auth, ok := credentials.Configs[image.Registry]
if !ok {
// Fallback to unauthenticated access in case if no auth credentials are retrieved
log.Infof("Authentication credential of registry '%s' is not found. Will try push without authentication.", image.Registry)
// Header X-Registry-Auth is required
// Or API error (400): Bad parameters and missing X-Registry-Auth: EOF will throw
// Just to make not empty struct
auth = dockerlib.AuthConfiguration{Username: "docker"}
}
log.Debugf("Pushing image with options %+v", options)
err = c.Client.PushImage(options, auth)
if err != nil {
log.Errorf("Unable to push image '%s' to registry '%s'. Error: %s", image.Name, image.Registry, err)
return errors.New("unable to push docker image(s). Check that `docker login` works successfully on the command line")
}
log.Debugf("Image '%+v' push output:\n%s", image, outputBuffer)
log.Infof("Successfully pushed image '%s' to registry '%s'", image.Name, image.Registry)
return nil
}
// handleDockerRegistry adapt legacy docker registry address
// After docker login to docker.io, there must be https://index.docker.io/v1/ in config.json of authentication
// Reference: https://docs.docker.com/engine/api/v1.23/
// > However (for legacy reasons) the “official” Docker, Inc. hosted registry
// > must be specified with both a “https://” prefix and a “/v1/” suffix
// > even though Docker will prefer to use the v2 registry API.
func handleDockerRegistry(auth *dockerlib.AuthConfigurations) {
const address = "docker.io"
const legacyAddress = "https://index.docker.io/v1/"
if legacyAddressConfig, ok := auth.Configs[legacyAddress]; ok {
auth.Configs[address] = legacyAddressConfig
}
}

45
pkg/utils/docker/tag.go Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
dockerlib "github.com/fsouza/go-dockerclient"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// Tag will provide methods for interaction with API regarding tagging images
type Tag struct {
Client dockerlib.Client
}
func (c *Tag) TagImage(image Image) error {
options := dockerlib.TagImageOptions{
Tag: image.Tag,
Repo: image.Repository,
}
log.Infof("Tagging image '%s' into repository '%s'", image.Name, image.Repository)
err := c.Client.TagImage(image.ShortName, options)
if err != nil {
log.Errorf("Unable to tag image '%s' into repository '%s'. Error: %s", image.Name, image.Registry, err)
return errors.New("unable to tag docker image(s)")
}
log.Infof("Successfully tagged image '%s'", image.Remote)
return nil
}

7
script/test/cmd/tests_new.sh
View File

@ -95,3 +95,10 @@ convert::check_artifacts_generated "kompose --build local -f $KOMPOSE_ROOT/scrip
# Test build v3 relative compose file with context
relative_path=$(realpath --relative-to="$PWD" "$KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-v3.yml")
convert::check_artifacts_generated "kompose --build local -f $relative_path convert -o $TEMP_DIR/output_file" "$TEMP_DIR/output_file"
#####
# Test the build config with push image
# see tests_push_image.sh for local push test
# Should warn when push image disabled
cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/buildconfig/docker-compose-build-no-image.yml -o $TEMP_DIR/output_file convert --build=local --push-image-registry=whatever"
convert::expect_warning "$cmd" "Push image registry 'whatever' is specified but push image is disabled, skipping pushing to repository"

80
script/test/cmd/tests_push_image.sh Executable file
View File

@ -0,0 +1,80 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing pe#rmissions and
# limitations under the License.
# Here are tests for pushing image on authentication and custom registry.
# These tests only work on local for authentication inconvenient.
# Prerequisites:
# * `docker.io` account and docker login successfully
# * custom registry which login as well. Or a local hosted registry.
# * `jq` installed
# Variables
TEMP_DIR="/tmp/kompose"
mkdir -p $TEMP_DIR
mkdir -p "$TEMP_DIR/build"
DOCKER_LOGIN_USER="lexcao" # TODO change this to your account for pushing to docker.io
COMPOSE_FILE="$TEMP_DIR/docker-compose-push.yml"
BUILD_FILE="$TEMP_DIR/build/Dockerfile"
CUSTOM_REGISTRY="localhost:5000" # TODO change this to your local registry
# Custom compose file based on parameter
build_file_content="FROM busybox
RUN touch /test"
echo "$build_file_content" >> "$BUILD_FILE"
compose_file_content="version: \"2\"
services:
foo:
build: \"./build\"
image: docker.io/$DOCKER_LOGIN_USER/foobar"
echo "$compose_file_content" >> "$COMPOSE_FILE"
# Some helper functions
function get_docker_hub_tag() {
local image=$1
local tag=$2
curl "https://hub.docker.com/v2/repositories/$image/tags/$tag/" | jq
}
function get_custom_registry_tag() {
local image=$1
local tag=$2
curl "http://$CUSTOM_REGISTRY/v2/$image/manifests/$tag" -I
}
###################################################################################
cmd="kompose convert -f $COMPOSE_FILE -o $TEMP_DIR/output_file --build=local --push-image=true"
printf "Push image without custom registry default to docker.io\n"
echo "executing cmd '$cmd'"
$cmd
printf "\nVerify push success...\n"
get_docker_hub_tag "$DOCKER_LOGIN_USER/foobar" "latest"
#######
printf "\nPush image with custom registry\n"
cmd="$cmd --push-image-registry=$CUSTOM_REGISTRY"
echo "executing cmd '$cmd'"
$cmd
#kompose convert -f "$COMPOSE_FILE" -o "$TEMP_DIR/output_file" --build=local --push-image=true
printf "\nVerify push success...\n"
get_custom_registry_tag "$DOCKER_LOGIN_USER/foobar" "latest"
# Clean resource
rm -rf $TEMP_DIR