deep-stack/kompose
7
0
Fork 0
forked from LaconicNetwork/kompose
Code Pull Requests Activity

Add support for file based secret (#1159)

Browse Source 
* Support file based secret
* Fix issue about read-only volume path and add support to absolute path at long-syntax
This commit is contained in:
João Vitor Paes de Barros do Carmo 2019-12-01 00:05:31 -04:00 committed by Hang Yan
parent ac2b852955
commit 30736a3973
16 changed files with 687 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/conversion.md
View File

@ -33,7 +33,7 @@ __Glossary:__
| deploy: replicas | - | - | ✓ | Deployment.Spec.Replicas / DeploymentConfig.Spec.Replicas | |
| deploy: placement | - | - | ✓ | Pod.Spec.NodeSelector | |
| deploy: update_config | - | - | n | | |
| deploy: resources | - | - | ✓ | Containers.Resources.Limits.Memory / Containers.Resources.Limits.CPU | Support for memory as well as cpu |
| deploy: resources | - | - | ✓ | Containers.Resources.Limits.Memory / Containers.Resources.Limits.CPU | Support for memory as well as cpu |
| deploy: restart_policy | - | - | ✓ | Pod generation | This generated a Pod, see the [user guide on restart](http://kompose.io/user-guide/#restart) |
| deploy: labels | - | - | n | | |
| devices | x | x | x | | Not supported within Kubernetes, See issue https://github.com/kubernetes/kubernetes/issues/5607 |
@ -66,10 +66,10 @@ __Glossary:__
| ports | ✓ | ✓ | ✓ | Service.Spec.Ports | |
| ports: short-syntax | ✓ | ✓ | ✓ | Service.Spec.Ports | |
| ports: long-syntax | - | - | ✓ | Service.Spec.Ports | |
| secrets | - | - | n | | |
| secrets: short-syntax | - | - | n | | |
| secrets: long-syntax | - | - | n | | |
| security_opt | x | x | x | | Kubernetes uses its own container naming scheme |
| secrets | - | - | ✓ | Secret | External Secret is not Supported |
| secrets: short-syntax | - | - | ✓ | Secret | External Secret is not Supported |
| secrets: long-syntax | - | - | ✓ | Secret | External Secret is not Supported |
| security_opt | x | x | x | | Kubernetes uses its own container naming scheme |
| stop_grace_period | ✓ | ✓ | ✓ | Pod.Spec.TerminationGracePeriodSeconds | |
| stop_signal | x | x | x | | Not supported within Kubernetes. See issue https://github.com/kubernetes/kubernetes/issues/30051 |
| sysctls | n | n | n | | |

3
glide.yaml
View File

@ -7,6 +7,9 @@ import:
- package: github.com/sirupsen/logrus
version: 1.0.3
- package: github.com/spf13/cast
version: v1.2.0
- package: golang.org/x/sys
version: ebfc5b4631820b793c9010c87fd8fef0f39eb082

7
pkg/kobject/kobject.go
View File

@ -29,6 +29,8 @@ type KomposeObject struct {
// Transformer need to know origin format in order to tell user what tag is not supported in origin format
// as they can have different names. For example environment variables are called environment in compose but Env in bundle.
LoadedFrom string
Secrets map[string]dockerCliTypes.SecretConfig
}
// ConvertOptions holds all options that controls transformation process
@ -110,8 +112,9 @@ type ServiceConfig struct {
Replicas int `compose:"replicas"`
GroupAdd []int64 `compose:"group_add"`
Volumes []Volumes `compose:""`
HealthChecks HealthCheck `compose:""`
Placement map[string]string `compose:""`
Secrets []dockerCliTypes.ServiceSecretConfig
HealthChecks HealthCheck `compose:""`
Placement map[string]string `compose:""`
//This is for long LONG SYNTAX link(https://docs.docker.com/compose/compose-file/#long-syntax)
Configs []dockerCliTypes.ServiceConfigObjConfig `compose:""`
//This is for SHORT SYNTAX link(https://docs.docker.com/compose/compose-file/#configs)

2
pkg/loader/compose/v3.go Normal file → Executable file
View File

@ -252,6 +252,7 @@ func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.Kompose
komposeObject := kobject.KomposeObject{
ServiceConfigs: make(map[string]kobject.ServiceConfig),
LoadedFrom: "compose",
Secrets: composeObject.Secrets,
}
// Step 2. Parse through the object and convert it to kobject.KomposeObject!
@ -280,6 +281,7 @@ func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.Kompose
serviceConfig.Labels = composeServiceConfig.Labels
serviceConfig.HostName = composeServiceConfig.Hostname
serviceConfig.DomainName = composeServiceConfig.DomainName
serviceConfig.Secrets = composeServiceConfig.Secrets
//Adding network key related info
if len(composeServiceConfig.Networks) == 0 {

11
pkg/transformer/kubernetes/k8sutils.go
View File

@ -621,6 +621,17 @@ func GetEnvsFromFile(file string, opt kobject.ConvertOptions) (map[string]string
return envLoad, nil
}
// GetSecretDataFromFile load secret content data
func GetSecretDataFromFile(file string, opt kobject.ConvertOptions) ([]byte, error) {
composeDir, err := transformer.GetComposeFileDir(opt.InputFiles)
if err != nil {
return nil, errors.Wrap(err, "Unable to load file context")
}
fileLocation := path.Join(composeDir, file)
return ioutil.ReadFile(fileLocation)
}
// TODO(hang): merge these two functions
// GetContentFromFile gets the content from the file..
func GetContentFromFile(file string, opt kobject.ConvertOptions) (string, error) {
fileBytes, err := ioutil.ReadFile(file)

127
pkg/transformer/kubernetes/kubernetes.go Normal file → Executable file
View File

@ -53,6 +53,7 @@ import (
"github.com/kubernetes/kompose/pkg/loader/compose"
"github.com/pkg/errors"
"github.com/spf13/cast"
"k8s.io/kubernetes/pkg/api/meta"
"k8s.io/kubernetes/pkg/labels"
)
@ -391,6 +392,37 @@ func (k *Kubernetes) initIngress(name string, service kobject.ServiceConfig, por
return ingress
}
// CreateSecrets create secrets
func (k *Kubernetes) CreateSecrets(komposeObject kobject.KomposeObject) ([]*api.Secret, error) {
var objects []*api.Secret
for name, config := range komposeObject.Secrets {
if config.File != "" {
data, err := GetSecretDataFromFile(config.File, k.Opt)
if err != nil {
log.Fatal("unable to read secret from file: ", config.File)
return nil, err
}
secret := &api.Secret{
TypeMeta: unversioned.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: name,
Labels: transformer.ConfigLabels(name),
},
Type: api.SecretTypeOpaque,
Data: map[string][]byte{name: data},
}
objects = append(objects, secret)
} else {
log.Warnf("External secrets %s is not currently supported - ignoring", name)
}
}
return objects, nil
}
// CreatePVC initializes PersistentVolumeClaim
func (k *Kubernetes) CreatePVC(name string, mode string, size string, selectorValue string) (*api.PersistentVolumeClaim, error) {
volSize, err := resource.ParseQuantity(size)
@ -541,6 +573,86 @@ func (k *Kubernetes) ConfigTmpfs(name string, service kobject.ServiceConfig) ([]
return volumeMounts, volumes
}
// ConfigSecretVolumes config volumes from secret.
// Link: https://docs.docker.com/compose/compose-file/#secrets
// In kubernetes' Secret resource, it has a data structure like a map[string]bytes, every key will act like the file name
// when mount to a container. This is the part that missing in compose. So we will create a single key secret from compose
// config and the key's name will be the secret's name, it's value is the file content.
// compose'secret can only be mounted at `/run/secrets`, so we will hardcoded this.
func (k *Kubernetes) ConfigSecretVolumes(name string, service kobject.ServiceConfig) ([]api.VolumeMount, []api.Volume) {
var volumeMounts []api.VolumeMount
var volumes []api.Volume
if len(service.Secrets) > 0 {
for _, secretConfig := range service.Secrets {
if secretConfig.UID != "" {
log.Warnf("Ignore pid in secrets for service: %s", name)
}
if secretConfig.GID != "" {
log.Warnf("Ignore gid in secrets for service: %s", name)
}
var itemPath string // should be the filename
var mountPath = "" // should be the directory
// if is used the short-syntax
if secretConfig.Target == "" {
// the secret path (mountPath) should be inside the default directory /run/secrets
mountPath = "/run/secrets/" + secretConfig.Source
// the itemPath should be the source itself
itemPath = secretConfig.Source
} else {
// if is the long-syntax, i should get the last part of path and consider it the filename
pathSplitted := strings.Split(secretConfig.Target, "/")
lastPart := pathSplitted[len(pathSplitted)-1]
// if the filename (lastPart) and the target is the same
if lastPart == secretConfig.Target {
// the secret path should be the source (it need to be inside a directory and only the filename was given)
mountPath = secretConfig.Source
} else {
// should then get the target without the filename (lastPart)
mountPath = mountPath + strings.TrimSuffix(secretConfig.Target, "/"+lastPart) // menos ultima parte
}
// if the target isn't absolute path
if strings.HasPrefix(secretConfig.Target, "/") == false {
// concat the default secret directory
mountPath = "/run/secrets/" + mountPath
}
itemPath = lastPart
}
volSource := api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: secretConfig.Source,
Items: []api.KeyToPath{{
Key: secretConfig.Source,
Path: itemPath,
}},
},
}
if secretConfig.Mode != nil {
mode := cast.ToInt32(*secretConfig.Mode)
volSource.Secret.DefaultMode = &mode
}
vol := api.Volume{
Name: secretConfig.Source,
VolumeSource: volSource,
}
volumes = append(volumes, vol)
volMount := api.VolumeMount{
Name: vol.Name,
MountPath: mountPath,
}
volumeMounts = append(volumeMounts, volMount)
}
}
return volumeMounts, volumes
}
// ConfigVolumes configure the container volumes.
func (k *Kubernetes) ConfigVolumes(name string, service kobject.ServiceConfig) ([]api.VolumeMount, []api.Volume, []*api.PersistentVolumeClaim, error) {
volumeMounts := []api.VolumeMount{}
@ -561,6 +673,11 @@ func (k *Kubernetes) ConfigVolumes(name string, service kobject.ServiceConfig) (
useHostPath = true
}
// config volumes from secret if present
secretsVolumeMounts, secretsVolumes := k.ConfigSecretVolumes(name, service)
volumeMounts = append(volumeMounts, secretsVolumeMounts...)
volumes = append(volumes, secretsVolumes...)
var count int
//iterating over array of `Vols` struct as it contains all necessary information about volumes
for _, volume := range service.Volumes {
@ -865,6 +982,16 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject.
// this will hold all the converted data
var allobjects []runtime.Object
if komposeObject.Secrets != nil {
secrets, err := k.CreateSecrets(komposeObject)
if err != nil {
return nil, errors.Wrapf(err, "Unable to create Secret resource")
}
for _, item := range secrets {
allobjects = append(allobjects, item)
}
}
sortedKeys := SortedKeys(komposeObject)
for _, name := range sortedKeys {
service := komposeObject.ServiceConfigs[name]

10
pkg/transformer/openshift/openshift.go
View File

@ -275,6 +275,16 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C
buildRepo := opt.BuildRepo
buildBranch := opt.BuildBranch
if komposeObject.Secrets != nil {
secrets, err := o.CreateSecrets(komposeObject)
if err != nil {
return nil, errors.Wrapf(err, "create secrets error")
}
for _, item := range secrets {
allobjects = append(allobjects, item)
}
}
sortedKeys := kubernetes.SortedKeys(komposeObject)
for _, name := range sortedKeys {
service := komposeObject.ServiceConfigs[name]

24
script/test/cmd/tests.sh
View File

@ -458,7 +458,31 @@ cmd="kompose --provider openshift -f $KOMPOSE_ROOT/script/test/fixtures/service-
sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/service-name-change/output-os-template.json > /tmp/output-os.json
convert::expect_success_and_warning "$cmd" "/tmp/output-os.json" "Unsupported root level volumes key - ignoring"
#####
# Test secrets
# Kubernetes Test
cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/secrets/docker-compose-secrets-long.yml convert --stdout -j"
sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/secrets/output-long-k8s.json > /tmp/output-k8s.json
convert::expect_success_and_warning "$cmd" "/tmp/output-k8s.json" "External secrets my_other_secret is not currently supported - ignoring"
cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/secrets/docker-compose-secrets-short.yml convert --stdout -j"
sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/secrets/output-short-k8s.json > /tmp/output-k8s.json
convert::expect_success_and_warning "$cmd" "/tmp/output-k8s.json" "External secrets my_other_secret is not currently supported - ignoring"
# Openshift Test
cmd="kompose --provider openshift -f $KOMPOSE_ROOT/script/test/fixtures/secrets/docker-compose-secrets-long.yml convert --stdout -j"
sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/secrets/output-long-os.json > /tmp/output-os.json
convert::expect_success_and_warning "$cmd" "/tmp/output-os.json" "External secrets my_other_secret is not currently supported - ignoring"
cmd="kompose --provider openshift -f $KOMPOSE_ROOT/script/test/fixtures/secrets/docker-compose-secrets-short.yml convert --stdout -j"
sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/secrets/output-short-os.json > /tmp/output-os.json
convert::expect_success_and_warning "$cmd" "/tmp/output-os.json" "External secrets my_other_secret is not currently supported - ignoring"
#####
# Test regarding validating dockerfilepath
convert::expect_failure "kompose -f $KOMPOSE_ROOT/script/test/fixtures/dockerfilepath/docker-compose.yml convert --stdout"

17
script/test/fixtures/secrets/docker-compose-secrets-long.yml vendored Normal file
View File

@ -0,0 +1,17 @@
version: "3.1"
services:
redis:
image: redis:latest
deploy:
replicas: 1
secrets:
- source: my_secret
target: redis_secret
uid: '103'
gid: '103'
mode: 0440
secrets:
my_secret:
file: ./my_secret.txt
my_other_secret:
external: true

14
script/test/fixtures/secrets/docker-compose-secrets-short.yml vendored Normal file
View File

@ -0,0 +1,14 @@
version: "3.1"
services:
redis:
image: redis:latest
deploy:
replicas: 1
secrets:
- my_secret
- my_other_secret
secrets:
my_secret:
file: ./my_secret.txt
my_other_secret:
external: true

1
script/test/fixtures/secrets/my_secret.txt vendored Normal file
View File

@ -0,0 +1 @@
For the Watch!

81
script/test/fixtures/secrets/output-long-k8s.json vendored Normal file
View File

@ -0,0 +1,81 @@
{
"kind": "List",
"apiVersion": "v1",
"metadata": {},
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "my_secret",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "my_secret"
}
},
"data": {
"my_secret": "Rm9yIHRoZSBXYXRjaCE="
},
"type": "Opaque"
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
},
"annotations": {
"kompose.cmd": "%CMD%",
"kompose.version": "%VERSION%"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"volumes": [
{
"name": "my_secret",
"secret": {
"secretName": "my_secret",
"items": [
{
"key": "my_secret",
"path": "redis_secret"
}
],
"defaultMode": 288
}
}
],
"containers": [
{
"name": "redis",
"image": "redis:latest",
"resources": {},
"volumeMounts": [
{
"name": "my_secret",
"mountPath": "/run/secrets"
}
]
}
],
"restartPolicy": "Always"
}
},
"strategy": {}
},
"status": {}
}
]
}

133
script/test/fixtures/secrets/output-long-os.json vendored Normal file
View File

@ -0,0 +1,133 @@
{
"kind": "List",
"apiVersion": "v1",
"metadata": {},
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "my_secret",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "my_secret"
}
},
"data": {
"my_secret": "Rm9yIHRoZSBXYXRjaCE="
},
"type": "Opaque"
},
{
"kind": "DeploymentConfig",
"apiVersion": "v1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
},
"annotations": {
"kompose.cmd": "%CMD%",
"kompose.version": "%VERSION%"
}
},
"spec": {
"strategy": {
"resources": {}
},
"triggers": [
{
"type": "ConfigChange"
},
{
"type": "ImageChange",
"imageChangeParams": {
"automatic": true,
"containerNames": [
"redis"
],
"from": {
"kind": "ImageStreamTag",
"name": "redis:latest"
}
}
}
],
"replicas": 1,
"test": false,
"selector": {
"io.kompose.service": "redis"
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"volumes": [
{
"name": "my_secret",
"secret": {
"secretName": "my_secret",
"items": [
{
"key": "my_secret",
"path": "redis_secret"
}
],
"defaultMode": 288
}
}
],
"containers": [
{
"name": "redis",
"image": " ",
"resources": {},
"volumeMounts": [
{
"name": "my_secret",
"mountPath": "/run/secrets"
}
]
}
],
"restartPolicy": "Always"
}
}
},
"status": {}
},
{
"kind": "ImageStream",
"apiVersion": "v1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"tags": [
{
"name": "latest",
"annotations": null,
"from": {
"kind": "DockerImage",
"name": "redis:latest"
},
"generation": null,
"importPolicy": {}
}
]
},
"status": {
"dockerImageRepository": ""
}
}
]
}

96
script/test/fixtures/secrets/output-short-k8s.json vendored Normal file
View File

@ -0,0 +1,96 @@
{
"kind": "List",
"apiVersion": "v1",
"metadata": {},
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "my_secret",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "my_secret"
}
},
"data": {
"my_secret": "Rm9yIHRoZSBXYXRjaCE="
},
"type": "Opaque"
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
},
"annotations": {
"kompose.cmd": "%CMD%",
"kompose.version": "%VERSION%"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"volumes": [
{
"name": "my_secret",
"secret": {
"secretName": "my_secret",
"items": [
{
"key": "my_secret",
"path": "my_secret"
}
]
}
},
{
"name": "my_other_secret",
"secret": {
"secretName": "my_other_secret",
"items": [
{
"key": "my_other_secret",
"path": "my_other_secret"
}
]
}
}
],
"containers": [
{
"name": "redis",
"image": "redis:latest",
"resources": {},
"volumeMounts": [
{
"name": "my_secret",
"mountPath": "/run/secrets"
},
{
"name": "my_other_secret",
"mountPath": "/run/secrets"
}
]
}
],
"restartPolicy": "Always"
}
},
"strategy": {}
},
"status": {}
}
]
}

148
script/test/fixtures/secrets/output-short-os.json vendored Normal file
View File

@ -0,0 +1,148 @@
{
"kind": "List",
"apiVersion": "v1",
"metadata": {},
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "my_secret",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "my_secret"
}
},
"data": {
"my_secret": "Rm9yIHRoZSBXYXRjaCE="
},
"type": "Opaque"
},
{
"kind": "DeploymentConfig",
"apiVersion": "v1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
},
"annotations": {
"kompose.cmd": "%CMD%",
"kompose.version": "%VERSION%"
}
},
"spec": {
"strategy": {
"resources": {}
},
"triggers": [
{
"type": "ConfigChange"
},
{
"type": "ImageChange",
"imageChangeParams": {
"automatic": true,
"containerNames": [
"redis"
],
"from": {
"kind": "ImageStreamTag",
"name": "redis:latest"
}
}
}
],
"replicas": 1,
"test": false,
"selector": {
"io.kompose.service": "redis"
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"volumes": [
{
"name": "my_secret",
"secret": {
"secretName": "my_secret",
"items": [
{
"key": "my_secret",
"path": "my_secret"
}
]
}
},
{
"name": "my_other_secret",
"secret": {
"secretName": "my_other_secret",
"items": [
{
"key": "my_other_secret",
"path": "my_other_secret"
}
]
}
}
],
"containers": [
{
"name": "redis",
"image": " ",
"resources": {},
"volumeMounts": [
{
"name": "my_secret",
"mountPath": "/run/secrets"
},
{
"name": "my_other_secret",
"mountPath": "/run/secrets"
}
]
}
],
"restartPolicy": "Always"
}
}
},
"status": {}
},
{
"kind": "ImageStream",
"apiVersion": "v1",
"metadata": {
"name": "redis",
"creationTimestamp": null,
"labels": {
"io.kompose.service": "redis"
}
},
"spec": {
"tags": [
{
"name": "latest",
"annotations": null,
"from": {
"kind": "DockerImage",
"name": "redis:latest"
},
"generation": null,
"importPolicy": {}
}
]
},
"status": {
"dockerImageRepository": ""
}
}
]
}

10
vendor/github.com/joho/godotenv/godotenv.go generated vendored
View File

@ -308,6 +308,16 @@ func parseValue(value string, envMap map[string]string) string {
}
}
// expand variables
value = os.Expand(value, func(key string) string {
if val, ok := envMap[key]; ok {
return val
}
if val, ok := os.LookupEnv(key); ok {
return val
}
return ""
})
return value
}