From 3172bc05b57f27ddaee0653695f850f277b8beff Mon Sep 17 00:00:00 2001 From: AhmedGrati Date: Tue, 24 Oct 2023 16:19:22 +0100 Subject: [PATCH 1/4] fix: remove e2e flakiness by making the loop deterministic Signed-off-by: AhmedGrati --- pkg/transformer/kubernetes/k8sutils.go | 4 +-- pkg/transformer/kubernetes/k8sutils_test.go | 2 +- pkg/transformer/kubernetes/kubernetes.go | 35 +++++++++++---------- pkg/transformer/openshift/openshift.go | 2 +- script/test/fixtures/env/output-k8s.yaml | 12 +++---- script/test/fixtures/env/output-os.yaml | 12 +++---- 6 files changed, 35 insertions(+), 32 deletions(-) diff --git a/pkg/transformer/kubernetes/k8sutils.go b/pkg/transformer/kubernetes/k8sutils.go index bb6ac558..1e9d49b3 100644 --- a/pkg/transformer/kubernetes/k8sutils.go +++ b/pkg/transformer/kubernetes/k8sutils.go @@ -806,9 +806,9 @@ func (k *Kubernetes) RemoveDupObjects(objs *[]runtime.Object) { } // SortedKeys Ensure the kubernetes objects are in a consistent order -func SortedKeys(komposeObject kobject.KomposeObject) []string { +func SortedKeys[V kobject.ServiceConfig | kobject.ServiceConfigGroup](serviceConfig map[string]V) []string { var sortedKeys []string - for name := range komposeObject.ServiceConfigs { + for name := range serviceConfig { sortedKeys = append(sortedKeys, name) } sort.Strings(sortedKeys) diff --git a/pkg/transformer/kubernetes/k8sutils_test.go b/pkg/transformer/kubernetes/k8sutils_test.go index 15cfd87f..d050a610 100644 --- a/pkg/transformer/kubernetes/k8sutils_test.go +++ b/pkg/transformer/kubernetes/k8sutils_test.go @@ -511,7 +511,7 @@ func TestSortedKeys(t *testing.T) { komposeObject := kobject.KomposeObject{ ServiceConfigs: map[string]kobject.ServiceConfig{"b": service, "a": service1}, } - a := SortedKeys(komposeObject) + a := SortedKeys(komposeObject.ServiceConfigs) if !reflect.DeepEqual(a, c) { t.Logf("Test Fail output should be %s", c) } diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index 57e8ee18..e0d5a471 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -1455,20 +1455,23 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. if opt.ServiceGroupMode != "" { log.Debugf("Service group mode is: %s", opt.ServiceGroupMode) komposeObjectToServiceConfigGroupMapping := KomposeObjectToServiceConfigGroupMapping(&komposeObject, opt) - for name, group := range komposeObjectToServiceConfigGroupMapping { + sortedGroupMappingKeys := SortedKeys(komposeObjectToServiceConfigGroupMapping) + for _, group := range sortedGroupMappingKeys { + groupMapping := komposeObjectToServiceConfigGroupMapping[group] var objects []runtime.Object podSpec := PodSpec{} + var groupName string // if using volume group, the name here will be a volume config string. reset to the first service name if opt.ServiceGroupMode == "volume" { if opt.ServiceGroupName != "" { - name = opt.ServiceGroupName + groupName = opt.ServiceGroupName } else { var names []string - for _, svc := range group { + for _, svc := range groupMapping { names = append(names, svc.Name) } - name = strings.Join(names, "-") + groupName = strings.Join(names, "-") } } @@ -1476,7 +1479,7 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. // ports conflict check between services portsUses := map[string]bool{} - for _, service := range group { + for _, service := range groupMapping { // first do ports check ports := ConfigPorts(service) for _, port := range ports { @@ -1487,7 +1490,7 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. portsUses[key] = true } - log.Infof("Group Service %s to [%s]", service.Name, name) + log.Infof("Group Service %s to [%s]", service.Name, groupName) service.WithKomposeAnnotation = opt.WithKomposeAnnotation podSpec.Append(AddContainer(service, opt)) @@ -1495,17 +1498,17 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. return nil, err } // override.. - objects = append(objects, k.CreateWorkloadAndConfigMapObjects(name, service, opt)...) - k.configKubeServiceAndIngressForService(service, name, &objects) + objects = append(objects, k.CreateWorkloadAndConfigMapObjects(groupName, service, opt)...) + k.configKubeServiceAndIngressForService(service, groupName, &objects) // Configure the container volumes. - volumesMount, volumes, pvc, cms, err := k.ConfigVolumes(name, service) + volumesMount, volumes, pvc, cms, err := k.ConfigVolumes(groupName, service) if err != nil { return nil, errors.Wrap(err, "k.ConfigVolumes failed") } // Configure Tmpfs if len(service.TmpFs) > 0 { - TmpVolumesMount, TmpVolumes := k.ConfigTmpfs(name, service) + TmpVolumesMount, TmpVolumes := k.ConfigTmpfs(groupName, service) volumes = append(volumes, TmpVolumes...) volumesMount = append(volumesMount, TmpVolumesMount...) } @@ -1527,14 +1530,14 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. podSpec.Append( SetPorts(service), - ImagePullPolicy(name, service), - RestartPolicy(name, service), - SecurityContext(name, service), + ImagePullPolicy(groupName, service), + RestartPolicy(groupName, service), + SecurityContext(groupName, service), HostName(service), DomainName(service), ResourcesLimits(service), ResourcesRequests(service), - TerminationGracePeriodSeconds(name, service), + TerminationGracePeriodSeconds(groupName, service), TopologySpreadConstraints(service), ) @@ -1542,7 +1545,7 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. podSpec.Append(ServiceAccountName(serviceAccountName)) } - err = k.UpdateKubernetesObjectsMultipleContainers(name, service, &objects, podSpec) + err = k.UpdateKubernetesObjectsMultipleContainers(groupName, service, &objects, podSpec) if err != nil { return nil, errors.Wrap(err, "Error transforming Kubernetes objects") } @@ -1557,7 +1560,7 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. allobjects = append(allobjects, objects...) } } - sortedKeys := SortedKeys(komposeObject) + sortedKeys := SortedKeys(komposeObject.ServiceConfigs) for _, name := range sortedKeys { service := komposeObject.ServiceConfigs[name] diff --git a/pkg/transformer/openshift/openshift.go b/pkg/transformer/openshift/openshift.go index ba53409b..04f5a155 100644 --- a/pkg/transformer/openshift/openshift.go +++ b/pkg/transformer/openshift/openshift.go @@ -280,7 +280,7 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C } } - sortedKeys := kubernetes.SortedKeys(komposeObject) + sortedKeys := kubernetes.SortedKeys(komposeObject.ServiceConfigs) for _, name := range sortedKeys { service := komposeObject.ServiceConfigs[name] var objects []runtime.Object diff --git a/script/test/fixtures/env/output-k8s.yaml b/script/test/fixtures/env/output-k8s.yaml index 61bf3efe..05a6f4b8 100644 --- a/script/test/fixtures/env/output-k8s.yaml +++ b/script/test/fixtures/env/output-k8s.yaml @@ -67,12 +67,12 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: FOO valueFrom: configMapKeyRef: key: FOO - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 name: another-namenode ports: @@ -95,8 +95,8 @@ kind: ConfigMap metadata: creationTimestamp: null labels: - io.kompose.service: another-namenode-home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + io.kompose.service: another-namenode-Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env --- apiVersion: apps/v1 @@ -125,14 +125,14 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: CLUSTER_NAME value: test - name: FOO valueFrom: configMapKeyRef: key: FOO - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 name: namenode ports: diff --git a/script/test/fixtures/env/output-os.yaml b/script/test/fixtures/env/output-os.yaml index f4283959..fa1778b9 100644 --- a/script/test/fixtures/env/output-os.yaml +++ b/script/test/fixtures/env/output-os.yaml @@ -49,8 +49,8 @@ kind: ConfigMap metadata: creationTimestamp: null labels: - io.kompose.service: another-namenode-home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + io.kompose.service: another-namenode-Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env --- apiVersion: apps.openshift.io/v1 @@ -79,12 +79,12 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: FOO valueFrom: configMapKeyRef: key: FOO - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: ' ' name: another-namenode ports: @@ -166,14 +166,14 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: CLUSTER_NAME value: test - name: FOO valueFrom: configMapKeyRef: key: FOO - name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: ' ' name: namenode ports: From 0a3b1897c0a3f40e6f0838657dced4c533d220cc Mon Sep 17 00:00:00 2001 From: AhmedGrati Date: Tue, 24 Oct 2023 16:31:22 +0100 Subject: [PATCH 2/4] fix: remove e2e flakiness by making the loop deterministic Signed-off-by: AhmedGrati --- pkg/transformer/kubernetes/k8sutils.go | 15 ++++++++------- .../fixtures/service-group/output-k8s.yaml | 18 +++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pkg/transformer/kubernetes/k8sutils.go b/pkg/transformer/kubernetes/k8sutils.go index 1e9d49b3..5ea78e13 100644 --- a/pkg/transformer/kubernetes/k8sutils.go +++ b/pkg/transformer/kubernetes/k8sutils.go @@ -686,14 +686,15 @@ func getServiceGroupID(service kobject.ServiceConfig, mode string) string { // A warn/info message should be printed to let the user know. func KomposeObjectToServiceConfigGroupMapping(komposeObject *kobject.KomposeObject, opt kobject.ConvertOptions) map[string]kobject.ServiceConfigGroup { serviceConfigGroup := make(map[string]kobject.ServiceConfigGroup) - - for name, service := range komposeObject.ServiceConfigs { - groupID := getServiceGroupID(service, opt.ServiceGroupMode) + sortedServiceConfigs := SortedKeys(komposeObject.ServiceConfigs) + for _, service := range sortedServiceConfigs { + serviceConfig := komposeObject.ServiceConfigs[service] + groupID := getServiceGroupID(serviceConfig, opt.ServiceGroupMode) if groupID != "" { - service.Name = name - service.InGroup = true - serviceConfigGroup[groupID] = append(serviceConfigGroup[groupID], service) - komposeObject.ServiceConfigs[name] = service + serviceConfig.Name = service + serviceConfig.InGroup = true + serviceConfigGroup[groupID] = append(serviceConfigGroup[groupID], serviceConfig) + komposeObject.ServiceConfigs[service] = serviceConfig } } diff --git a/script/test/fixtures/service-group/output-k8s.yaml b/script/test/fixtures/service-group/output-k8s.yaml index e22fdd17..57929661 100644 --- a/script/test/fixtures/service-group/output-k8s.yaml +++ b/script/test/fixtures/service-group/output-k8s.yaml @@ -39,6 +39,14 @@ spec: io.kompose.service: librenms-dispatcher spec: containers: + - env: + - name: TZ + image: librenms/dispatcher:latest + name: dispatcher + resources: {} + volumeMounts: + - mountPath: /data + name: librenms-dispatcher-claim0 - env: - name: TZ image: librenms/librenms:latest @@ -51,15 +59,7 @@ spec: volumeMounts: - mountPath: /data name: librenms-dispatcher-claim0 - - env: - - name: TZ - image: librenms/dispatcher:latest - name: dispatcher - resources: {} - volumeMounts: - - mountPath: /data - name: librenms-dispatcher-claim0 - hostname: dispatcher + hostname: librenms restartPolicy: Always volumes: - name: librenms-dispatcher-claim0 From fa96d2b80a4d08d931a19e463a3b5e91f9aba22f Mon Sep 17 00:00:00 2001 From: AhmedGrati Date: Tue, 24 Oct 2023 16:34:28 +0100 Subject: [PATCH 3/4] fix: remove e2e flakiness by making the loop deterministic Signed-off-by: AhmedGrati --- script/test/fixtures/env/output-k8s.yaml | 12 ++++++------ script/test/fixtures/env/output-os.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/script/test/fixtures/env/output-k8s.yaml b/script/test/fixtures/env/output-k8s.yaml index 05a6f4b8..61bf3efe 100644 --- a/script/test/fixtures/env/output-k8s.yaml +++ b/script/test/fixtures/env/output-k8s.yaml @@ -67,12 +67,12 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: FOO valueFrom: configMapKeyRef: key: FOO - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 name: another-namenode ports: @@ -95,8 +95,8 @@ kind: ConfigMap metadata: creationTimestamp: null labels: - io.kompose.service: another-namenode-Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + io.kompose.service: another-namenode-home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env --- apiVersion: apps/v1 @@ -125,14 +125,14 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: CLUSTER_NAME value: test - name: FOO valueFrom: configMapKeyRef: key: FOO - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 name: namenode ports: diff --git a/script/test/fixtures/env/output-os.yaml b/script/test/fixtures/env/output-os.yaml index fa1778b9..f4283959 100644 --- a/script/test/fixtures/env/output-os.yaml +++ b/script/test/fixtures/env/output-os.yaml @@ -49,8 +49,8 @@ kind: ConfigMap metadata: creationTimestamp: null labels: - io.kompose.service: another-namenode-Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + io.kompose.service: another-namenode-home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env --- apiVersion: apps.openshift.io/v1 @@ -79,12 +79,12 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: FOO valueFrom: configMapKeyRef: key: FOO - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: ' ' name: another-namenode ports: @@ -166,14 +166,14 @@ spec: valueFrom: configMapKeyRef: key: BAR - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env - name: CLUSTER_NAME value: test - name: FOO valueFrom: configMapKeyRef: key: FOO - name: Users-ahmedgrati-projects-kompose-script-test-fixtures-env-hadoop-hive-namenode-env + name: home-runner-work-kompose-kompose-script-test-fixtures-env-hadoop-hive-namenode-env image: ' ' name: namenode ports: From 53de2c44fdd8c5e207480e1262a4e86a820a230d Mon Sep 17 00:00:00 2001 From: AhmedGrati Date: Tue, 24 Oct 2023 23:08:34 +0100 Subject: [PATCH 4/4] fix failing unit tests Signed-off-by: AhmedGrati --- pkg/transformer/kubernetes/k8sutils.go | 1 + pkg/transformer/kubernetes/kubernetes.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pkg/transformer/kubernetes/k8sutils.go b/pkg/transformer/kubernetes/k8sutils.go index 5ea78e13..03967e49 100644 --- a/pkg/transformer/kubernetes/k8sutils.go +++ b/pkg/transformer/kubernetes/k8sutils.go @@ -687,6 +687,7 @@ func getServiceGroupID(service kobject.ServiceConfig, mode string) string { func KomposeObjectToServiceConfigGroupMapping(komposeObject *kobject.KomposeObject, opt kobject.ConvertOptions) map[string]kobject.ServiceConfigGroup { serviceConfigGroup := make(map[string]kobject.ServiceConfigGroup) sortedServiceConfigs := SortedKeys(komposeObject.ServiceConfigs) + for _, service := range sortedServiceConfigs { serviceConfig := komposeObject.ServiceConfigs[service] groupID := getServiceGroupID(serviceConfig, opt.ServiceGroupMode) diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index e0d5a471..3ced87bf 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -1473,6 +1473,8 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. } groupName = strings.Join(names, "-") } + } else { + groupName = group } // added a container