From f4bfe1fcb5de6259d8364941274106e05361549f Mon Sep 17 00:00:00 2001 From: Charlie Drage Date: Thu, 7 Sep 2017 08:50:31 -0400 Subject: [PATCH] Add env_file + ConfigMaps feature to Kompose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using env_file with Docker Compose, a ConfigMap will be generated For example: ```sh ▶ ./kompose convert -f script/test/fixtures/configmaps/docker-compose.yml INFO Kubernetes file "redis-service.yaml" created INFO Kubernetes file "redis-deployment.yaml" created INFO Kubernetes file "foo-env-configmap.yaml" created INFO Kubernetes file "bar-env-configmap.yaml" created ``` File: ```yaml version: '3' services: redis: image: 'bitnami/redis:latest' environment: - ALLOW_EMPTY_PASSWORD=no # Env file will override environment / warn! env_file: - "foo.env" - bar.env labels: kompose.service.type: nodeport ports: - '6379:6379' ``` To: ```yaml apiVersion: v1 data: ALLOW_EMPTY_PASSWORD: "yes" kind: ConfigMap metadata: creationTimestamp: null name: foo-env ``` ```yaml ... - env: - name: ALLOW_EMPTY_PASSWORD valueFrom: configMapKeyRef: key: ALLOW_EMPTY_PASSWORD name: foo-env ``` --- pkg/kobject/kobject.go | 2 +- pkg/loader/compose/v3.go | 3 + pkg/transformer/kubernetes/k8sutils.go | 34 ++++- pkg/transformer/kubernetes/kubernetes.go | 99 +++++++++++-- pkg/transformer/openshift/openshift.go | 7 +- pkg/transformer/openshift/openshift_test.go | 3 +- script/test/cmd/tests.sh | 6 + script/test/fixtures/configmap/bar.env | 3 + .../fixtures/configmap/docker-compose.yaml | 15 ++ script/test/fixtures/configmap/foo.env | 2 + .../configmap/output-k8s-template.json | 135 ++++++++++++++++++ 11 files changed, 295 insertions(+), 14 deletions(-) create mode 100644 script/test/fixtures/configmap/bar.env create mode 100644 script/test/fixtures/configmap/docker-compose.yaml create mode 100644 script/test/fixtures/configmap/foo.env create mode 100644 script/test/fixtures/configmap/output-k8s-template.json diff --git a/pkg/kobject/kobject.go b/pkg/kobject/kobject.go index ec206837..8c025ae3 100644 --- a/pkg/kobject/kobject.go +++ b/pkg/kobject/kobject.go @@ -62,10 +62,10 @@ type ConvertOptions struct { // ServiceConfig holds the basic struct of a container type ServiceConfig struct { - // use tags to mark from what element this value comes ContainerName string Image string `compose:"image"` Environment []EnvVar `compose:"environment"` + EnvFile []string `compose:"env_file"` Port []Ports `compose:"ports"` Command []string `compose:"command"` WorkingDir string `compose:""` diff --git a/pkg/loader/compose/v3.go b/pkg/loader/compose/v3.go index b1fe685c..55288156 100644 --- a/pkg/loader/compose/v3.go +++ b/pkg/loader/compose/v3.go @@ -322,6 +322,9 @@ func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.Kompose serviceConfig.Environment = append(serviceConfig.Environment, env) } + // Get env_file + serviceConfig.EnvFile = composeServiceConfig.EnvFile + // Parse the ports // v3 uses a new format called "long syntax" starting in 3.2 // https://docs.docker.com/compose/compose-file/#ports diff --git a/pkg/transformer/kubernetes/k8sutils.go b/pkg/transformer/kubernetes/k8sutils.go index 3df9c42b..07aa81ca 100644 --- a/pkg/transformer/kubernetes/k8sutils.go +++ b/pkg/transformer/kubernetes/k8sutils.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "reflect" "strconv" @@ -31,6 +32,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ghodss/yaml" + "github.com/joho/godotenv" "github.com/kubernetes/kompose/pkg/kobject" "github.com/kubernetes/kompose/pkg/transformer" @@ -330,9 +332,13 @@ func (k *Kubernetes) CreateHeadlessService(name string, service kobject.ServiceC } // UpdateKubernetesObjects loads configurations to k8s objects -func (k *Kubernetes) UpdateKubernetesObjects(name string, service kobject.ServiceConfig, objects *[]runtime.Object) error { +func (k *Kubernetes) UpdateKubernetesObjects(name string, service kobject.ServiceConfig, opt kobject.ConvertOptions, objects *[]runtime.Object) error { + // Configure the environment variables. - envs := k.ConfigEnvs(name, service) + envs, err := k.ConfigEnvs(name, service, opt) + if err != nil { + return errors.Wrap(err, "Unable to load env variables") + } // Configure the container volumes. volumesMount, volumes, pvc, err := k.ConfigVolumes(name, service) @@ -570,3 +576,27 @@ func DurationStrToSecondsInt(s string) (*int64, error) { r := (int64)(duration.Seconds()) return &r, nil } + +func GetEnvsFromFile(file string, opt kobject.ConvertOptions) (map[string]string, error) { + // Get the correct file context / directory + composeDir, err := transformer.GetComposeFileDir(opt.InputFiles) + if err != nil { + return nil, errors.Wrap(err, "Unable to load file context") + } + fileLocation := path.Join(composeDir, file) + + // Load environment variables from file + envLoad, err := godotenv.Read(fileLocation) + if err != nil { + return nil, errors.Wrap(err, "Unable to read env_file") + } + + return envLoad, nil +} + +func FormatEnvName(name string) string { + envName := strings.Trim(name, "./") + envName = strings.Replace(envName, ".", "-", -1) + envName = strings.Replace(envName, "/", "-", -1) + return envName +} diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index edfbe0de..5531d3a9 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -158,6 +158,33 @@ func (k *Kubernetes) InitSvc(name string, service kobject.ServiceConfig) *api.Se return svc } +// InitConfigMap initialized a ConfigMap object +func (k *Kubernetes) InitConfigMap(name string, service kobject.ServiceConfig, opt kobject.ConvertOptions, envFile string) *api.ConfigMap { + + envs, err := GetEnvsFromFile(envFile, opt) + if err != nil { + log.Fatalf("Unable to retrieve env file: %s", err) + } + + // Remove root pathing + // replace all other slashes / preiods + envName := FormatEnvName(envFile) + + // In order to differentiate files, we append to the name and remove '.env' if applicate from the file name + configMap := &api.ConfigMap{ + TypeMeta: unversioned.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: api.ObjectMeta{ + Name: envName, + }, + Data: envs, + } + + return configMap +} + // InitD initializes Kubernetes Deployment object func (k *Kubernetes) InitD(name string, service kobject.ServiceConfig, replicas int) *extensions.Deployment { dc := &extensions.Deployment{ @@ -476,19 +503,59 @@ func (k *Kubernetes) ConfigPVCVolumeSource(name string, readonly bool) *api.Volu } // ConfigEnvs configures the environment variables. -func (k *Kubernetes) ConfigEnvs(name string, service kobject.ServiceConfig) []api.EnvVar { +func (k *Kubernetes) ConfigEnvs(name string, service kobject.ServiceConfig, opt kobject.ConvertOptions) ([]api.EnvVar, error) { + envs := transformer.EnvSort{} - for _, v := range service.Environment { - envs = append(envs, api.EnvVar{ - Name: v.Name, - Value: v.Value, - }) + + // If there is an env_file, use ConfigMaps and ignore the environment variables + // already specified + if len(service.EnvFile) > 0 { + + // Load each env_file + + for _, file := range service.EnvFile { + + envName := FormatEnvName(file) + + // Load environment variables from file + envLoad, err := GetEnvsFromFile(file, opt) + if err != nil { + return envs, errors.Wrap(err, "Unable to read env_file") + } + + // Add configMapKeyRef to each environment variable + for k, _ := range envLoad { + envs = append(envs, api.EnvVar{ + Name: k, + ValueFrom: &api.EnvVarSource{ + ConfigMapKeyRef: &api.ConfigMapKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: envName, + }, + Key: k, + }}, + }) + } + + } + + } else { + + // Load up the environment variables + for _, v := range service.Environment { + envs = append(envs, api.EnvVar{ + Name: v.Name, + Value: v.Value, + }) + } + } + // Stable sorts data while keeping the original order of equal elements // we need this because envs are not populated in any random order // this sorting ensures they are populated in a particular order sort.Stable(envs) - return envs + return envs, nil } // CreateKubernetesObjects generates a Kubernetes artifact for each input type service @@ -517,6 +584,13 @@ func (k *Kubernetes) CreateKubernetesObjects(name string, service kobject.Servic objects = append(objects, k.InitRC(name, service, replica)) } + if len(service.EnvFile) > 0 { + for _, envFile := range service.EnvFile { + configMap := k.InitConfigMap(name, service, opt, envFile) + objects = append(objects, configMap) + } + } + return objects } @@ -610,7 +684,10 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. } } - k.UpdateKubernetesObjects(name, service, &objects) + err := k.UpdateKubernetesObjects(name, service, opt, &objects) + if err != nil { + return nil, errors.Wrap(err, "Error transforming Kubernetes objects") + } allobjects = append(allobjects, objects...) } @@ -744,6 +821,12 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con return err } log.Infof("Successfully created Pod: %s", t.Name) + case *api.ConfigMap: + _, err := client.ConfigMaps(namespace).Create(t) + if err != nil { + return err + } + log.Infof("Successfully created Config Map: %s", t.Name) } } diff --git a/pkg/transformer/openshift/openshift.go b/pkg/transformer/openshift/openshift.go index dd46df43..1ccb4b98 100644 --- a/pkg/transformer/openshift/openshift.go +++ b/pkg/transformer/openshift/openshift.go @@ -398,8 +398,11 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C } } - // Update and then append the objects (we're done generating) - o.UpdateKubernetesObjects(name, service, &objects) + err := o.UpdateKubernetesObjects(name, service, opt, &objects) + if err != nil { + return nil, errors.Wrap(err, "Error transforming Kubernetes objects") + } + allobjects = append(allobjects, objects...) } diff --git a/pkg/transformer/openshift/openshift_test.go b/pkg/transformer/openshift/openshift_test.go index 302e42fe..ebf346b3 100644 --- a/pkg/transformer/openshift/openshift_test.go +++ b/pkg/transformer/openshift/openshift_test.go @@ -62,9 +62,10 @@ func TestOpenShiftUpdateKubernetesObjects(t *testing.T) { var object []runtime.Object o := OpenShift{} serviceConfig := newServiceConfig() + opt := kobject.ConvertOptions{} object = append(object, o.initDeploymentConfig("foobar", serviceConfig, 3)) - o.UpdateKubernetesObjects("foobar", serviceConfig, &object) + o.UpdateKubernetesObjects("foobar", serviceConfig, opt, &object) for _, obj := range object { switch tobj := obj.(type) { diff --git a/script/test/cmd/tests.sh b/script/test/cmd/tests.sh index 3f2be307..b90d33c7 100755 --- a/script/test/cmd/tests.sh +++ b/script/test/cmd/tests.sh @@ -418,6 +418,12 @@ cmd="kompose convert --stdout -j --provider=openshift -f $KOMPOSE_ROOT/script/te sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" "$KOMPOSE_ROOT/script/test/fixtures/healthcheck/output-os-template.json" > /tmp/output-os.json convert::expect_success "kompose convert --stdout -j --provider=openshift -f $KOMPOSE_ROOT/script/test/fixtures/healthcheck/docker-compose.yaml" "/tmp/output-os.json" +# Test ConfigMap generation +cmd="kompose convert --stdout -j -f $KOMPOSE_ROOT/script/test/fixtures/configmap/docker-compose.yaml" +sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" "$KOMPOSE_ROOT/script/test/fixtures/configmap/output-k8s-template.json" > /tmp/output-k8s.json +convert::expect_success "kompose convert --stdout -j -f $KOMPOSE_ROOT/script/test/fixtures/configmap/docker-compose.yaml" "/tmp/output-k8s.json" + + # Test V3 Support of Docker Compose # Test deploy mode: global diff --git a/script/test/fixtures/configmap/bar.env b/script/test/fixtures/configmap/bar.env new file mode 100644 index 00000000..8c113bf9 --- /dev/null +++ b/script/test/fixtures/configmap/bar.env @@ -0,0 +1,3 @@ +# Multi-line test +FOO=BAR +BAR=FOO diff --git a/script/test/fixtures/configmap/docker-compose.yaml b/script/test/fixtures/configmap/docker-compose.yaml new file mode 100644 index 00000000..caa11ac6 --- /dev/null +++ b/script/test/fixtures/configmap/docker-compose.yaml @@ -0,0 +1,15 @@ +version: '3' + +services: + redis: + image: 'bitnami/redis:latest' + environment: + - ALLOW_EMPTY_PASSWORD=no + # Env file will override environment / warn! + env_file: + - "foo.env" + - bar.env + labels: + kompose.service.type: nodeport + ports: + - '6379:6379' diff --git a/script/test/fixtures/configmap/foo.env b/script/test/fixtures/configmap/foo.env new file mode 100644 index 00000000..63b8c81a --- /dev/null +++ b/script/test/fixtures/configmap/foo.env @@ -0,0 +1,2 @@ +# Test comment! +ALLOW_EMPTY_PASSWORD=yes diff --git a/script/test/fixtures/configmap/output-k8s-template.json b/script/test/fixtures/configmap/output-k8s-template.json new file mode 100644 index 00000000..d2479673 --- /dev/null +++ b/script/test/fixtures/configmap/output-k8s-template.json @@ -0,0 +1,135 @@ +{ + "kind": "List", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "redis", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "redis" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.service.type": "nodeport", + "kompose.version": "%VERSION%" + } + }, + "spec": { + "ports": [ + { + "name": "6379", + "port": 6379, + "targetPort": 6379 + } + ], + "selector": { + "io.kompose.service": "redis" + }, + "type": "NodePort" + }, + "status": { + "loadBalancer": {} + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "redis", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "redis" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.service.type": "nodeport", + "kompose.version": "%VERSION%" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "io.kompose.service": "redis" + } + }, + "spec": { + "containers": [ + { + "name": "redis", + "image": "bitnami/redis:latest", + "ports": [ + { + "containerPort": 6379 + } + ], + "env": [ + { + "name": "ALLOW_EMPTY_PASSWORD", + "valueFrom": { + "configMapKeyRef": { + "name": "foo-env", + "key": "ALLOW_EMPTY_PASSWORD" + } + } + }, + { + "name": "BAR", + "valueFrom": { + "configMapKeyRef": { + "name": "bar-env", + "key": "BAR" + } + } + }, + { + "name": "FOO", + "valueFrom": { + "configMapKeyRef": { + "name": "bar-env", + "key": "FOO" + } + } + } + ], + "resources": {} + } + ], + "restartPolicy": "Always" + } + }, + "strategy": {} + }, + "status": {} + }, + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "foo-env", + "creationTimestamp": null + }, + "data": { + "ALLOW_EMPTY_PASSWORD": "yes" + } + }, + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "bar-env", + "creationTimestamp": null + }, + "data": { + "BAR": "FOO", + "FOO": "BAR" + } + } + ] +}