From d48ae643256aa8f2a5f9003cfecc9c0482275f57 Mon Sep 17 00:00:00 2001 From: huikaihoo Date: Fri, 30 Nov 2018 01:07:17 +0800 Subject: [PATCH] Add label support to named volumes in docker compose v3 to Kubernetes (#1083) * Support read specific label (kompose.volume.size) from named volume and apply to kubernetes supported volume size * Fix the PVC size in log message when deploy Kubernetes * Skip creation of PersistentVolumeClaim if it is already created in the same kubernetes deploy * Add selector to PersistentVolumeClaim only when specific label (kompose.volume.selector) is used in named volume * Add test case to named-volume for the new labels --- pkg/kobject/kobject.go | 19 ++- pkg/loader/compose/v3.go | 42 ++++- pkg/transformer/kubernetes/kubernetes.go | 37 ++++- script/test/cmd/tests.sh | 4 + .../named-volume/docker-compose-v3.yml | 16 ++ .../named-volume/output-k8s-v3.json | 157 ++++++++++++++++++ 6 files changed, 256 insertions(+), 19 deletions(-) create mode 100644 script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml create mode 100644 script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json diff --git a/pkg/kobject/kobject.go b/pkg/kobject/kobject.go index a6166e99..540c8b2b 100644 --- a/pkg/kobject/kobject.go +++ b/pkg/kobject/kobject.go @@ -143,13 +143,14 @@ type Ports struct { // Volumes holds the volume struct of container type Volumes struct { - SvcName string // Service name to which volume is linked - MountPath string // Mountpath extracted from docker-compose file - VFrom string // denotes service name from which volume is coming - VolumeName string // name of volume if provided explicitly - Host string // host machine address - Container string // Mountpath - Mode string // access mode for volume - PVCName string // name of PVC - PVCSize string // PVC size + SvcName string // Service name to which volume is linked + MountPath string // Mountpath extracted from docker-compose file + VFrom string // denotes service name from which volume is coming + VolumeName string // name of volume if provided explicitly + Host string // host machine address + Container string // Mountpath + Mode string // access mode for volume + PVCName string // name of PVC + PVCSize string // PVC size + SelectorValue string // Value of the label selector } diff --git a/pkg/loader/compose/v3.go b/pkg/loader/compose/v3.go index c479b257..e0b3d45e 100644 --- a/pkg/loader/compose/v3.go +++ b/pkg/loader/compose/v3.go @@ -422,11 +422,51 @@ func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.Kompose komposeObject.ServiceConfigs[normalizeServiceNames(name)] = serviceConfig } - handleVolume(&komposeObject) + handleV3Volume(&komposeObject, &composeObject.Volumes) return komposeObject, nil } +func handleV3Volume(komposeObject *kobject.KomposeObject, volumes *map[string]types.VolumeConfig) { + for name := range komposeObject.ServiceConfigs { + // retrieve volumes of service + vols, err := retrieveVolume(name, *komposeObject) + if err != nil { + errors.Wrap(err, "could not retrieve vvolume") + } + for volName, vol := range vols { + size, selector := getV3VolumeLabels(vol.VolumeName, volumes) + if len(size) > 0 || len(selector) > 0 { + // We can't assign value to struct field in map while iterating over it, so temporary variable `temp` is used here + var temp = vols[volName] + temp.PVCSize = size + temp.SelectorValue = selector + vols[volName] = temp + } + } + // We can't assign value to struct field in map while iterating over it, so temporary variable `temp` is used here + var temp = komposeObject.ServiceConfigs[name] + temp.Volumes = vols + komposeObject.ServiceConfigs[name] = temp + } +} + +func getV3VolumeLabels(name string, volumes *map[string]types.VolumeConfig) (string, string) { + size, selector := "", "" + + if volume, ok := (*volumes)[name]; ok { + for key, value := range volume.Labels { + if key == "kompose.volume.size" { + size = value + } else if key == "kompose.volume.selector" { + selector = value + } + } + } + + return size, selector +} + func mergeComposeObject(oldCompose *types.Config, newCompose *types.Config) (*types.Config, error) { if oldCompose == nil || newCompose == nil { return nil, fmt.Errorf("Merge multiple compose error, compose config is nil") diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index c8618f78..ef0f29bb 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -373,7 +373,7 @@ func (k *Kubernetes) initIngress(name string, service kobject.ServiceConfig, por } // CreatePVC initializes PersistentVolumeClaim -func (k *Kubernetes) CreatePVC(name string, mode string, size string) (*api.PersistentVolumeClaim, error) { +func (k *Kubernetes) CreatePVC(name string, mode string, size string, selectorValue string) (*api.PersistentVolumeClaim, error) { volSize, err := resource.ParseQuantity(size) if err != nil { return nil, errors.Wrap(err, "resource.ParseQuantity failed, Error parsing size") @@ -397,6 +397,12 @@ func (k *Kubernetes) CreatePVC(name string, mode string, size string) (*api.Pers }, } + if len(selectorValue) > 0 { + pvc.Spec.Selector = &unversioned.LabelSelector{ + MatchLabels: transformer.ConfigLabels(selectorValue), + } + } + if mode == "ro" { pvc.Spec.AccessModes = []api.PersistentVolumeAccessMode{api.ReadOnlyMany} } else { @@ -578,13 +584,17 @@ func (k *Kubernetes) ConfigVolumes(name string, service kobject.ServiceConfig) ( if volume.VFrom == "" { defaultSize := PVCRequestSize - for key, value := range service.Labels { - if key == "kompose.volume.size" { - defaultSize = value + if len(volume.PVCSize) > 0 { + defaultSize = volume.PVCSize + } else { + for key, value := range service.Labels { + if key == "kompose.volume.size" { + defaultSize = value + } } } - createdPVC, err := k.CreatePVC(volumeName, volume.Mode, defaultSize) + createdPVC, err := k.CreatePVC(volumeName, volume.Mode, defaultSize, volume.SelectorValue) if err != nil { return nil, nil, nil, errors.Wrap(err, "k.CreatePVC failed") @@ -978,6 +988,8 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con return err } + pvcCreatedSet := make(map[string]bool) + log.Infof("Deploying application in %q namespace", namespace) for _, v := range objects { @@ -1010,11 +1022,18 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con } log.Infof("Successfully created Service: %s", t.Name) case *api.PersistentVolumeClaim: - _, err := client.PersistentVolumeClaims(namespace).Create(t) - if err != nil { - return err + if pvcCreatedSet[t.Name] { + log.Infof("Skip creation of PersistentVolumeClaim as it is already created: %s", t.Name) + } else { + _, err := client.PersistentVolumeClaims(namespace).Create(t) + if err != nil { + return err + } + pvcCreatedSet[t.Name] = true + storage := t.Spec.Resources.Requests[api.ResourceStorage] + capacity := storage.String() + log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, capacity) } - log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, PVCRequestSize) case *extensions.Ingress: _, err := client.Ingress(namespace).Create(t) if err != nil { diff --git a/script/test/cmd/tests.sh b/script/test/cmd/tests.sh index 193cda37..65312fbf 100755 --- a/script/test/cmd/tests.sh +++ b/script/test/cmd/tests.sh @@ -185,6 +185,10 @@ cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/do sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-k8s-template.json > /tmp/output-k8s.json convert::expect_success "$cmd" "/tmp/output-k8s.json" +cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml convert --stdout -j" +sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json > /tmp/output-k8s.json +convert::expect_success "$cmd" "/tmp/output-k8s.json" + # openshift test cmd="kompose --provider=openshift -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/docker-compose.yml convert --stdout -j" sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-os-template.json > /tmp/output-os.json diff --git a/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml b/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml new file mode 100644 index 00000000..7bce3895 --- /dev/null +++ b/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml @@ -0,0 +1,16 @@ +version: '3' +services: + db: + image: postgres:10.1 + ports: + - "5432" + labels: + kompose.volume.size: 200Mi + volumes: + - db-data:/var/lib/postgresql/data + - db-config:/var/lib/postgresql/config +volumes: + db-data: + labels: + kompose.volume.selector: db-data-dev + kompose.volume.size: 500Mi \ No newline at end of file diff --git a/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json b/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json new file mode 100644 index 00000000..2bf78a45 --- /dev/null +++ b/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json @@ -0,0 +1,157 @@ +{ + "kind": "List", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "db", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.version": "%VERSION%", + "kompose.volume.size": "200Mi" + } + }, + "spec": { + "ports": [ + { + "name": "5432", + "port": 5432, + "targetPort": 5432 + } + ], + "selector": { + "io.kompose.service": "db" + } + }, + "status": { + "loadBalancer": {} + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "db", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.version": "%VERSION%", + "kompose.volume.size": "200Mi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + } + }, + "spec": { + "volumes": [ + { + "name": "db-data", + "persistentVolumeClaim": { + "claimName": "db-data" + } + }, + { + "name": "db-config", + "persistentVolumeClaim": { + "claimName": "db-config" + } + } + ], + "containers": [ + { + "name": "db", + "image": "postgres:10.1", + "ports": [ + { + "containerPort": 5432 + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "db-data", + "mountPath": "/var/lib/postgresql/data" + }, + { + "name": "db-config", + "mountPath": "/var/lib/postgresql/config" + } + ] + } + ], + "restartPolicy": "Always" + } + }, + "strategy": { + "type": "Recreate" + } + }, + "status": {} + }, + { + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "db-data", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db-data" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "selector": { + "matchLabels": { + "io.kompose.service": "db-data-dev" + } + }, + "resources": { + "requests": { + "storage": "500Mi" + } + } + }, + "status": {} + }, + { + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "db-config", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db-config" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "200Mi" + } + } + }, + "status": {} + } + ] +}