From 22d9233f8bb719ed598684eb58860b73c0ebe863 Mon Sep 17 00:00:00 2001 From: yuefanxiao <591649725@qq.com> Date: Thu, 8 May 2025 01:23:08 +0800 Subject: [PATCH] fix: resolve env_file variable interpolation issues by adding support for and default syntax --- pkg/transformer/kubernetes/k8sutils.go | 5 ++ pkg/transformer/kubernetes/kubernetes.go | 66 ++++++++++++---- pkg/transformer/kubernetes/kubernetes_test.go | 79 +++++++++++++++++++ pkg/transformer/openshift/openshift.go | 8 +- script/test/cmd/tests.sh | 8 ++ .../test/fixtures/envfile-interpolation/.env | 4 + .../envfile-interpolation/compose.yaml | 14 ++++ .../envfile-interpolation/output-k8s.yaml | 56 +++++++++++++ .../envfile-interpolation/output-os.yaml | 56 +++++++++++++ 9 files changed, 276 insertions(+), 20 deletions(-) create mode 100644 script/test/fixtures/envfile-interpolation/.env create mode 100644 script/test/fixtures/envfile-interpolation/compose.yaml create mode 100644 script/test/fixtures/envfile-interpolation/output-k8s.yaml create mode 100644 script/test/fixtures/envfile-interpolation/output-os.yaml diff --git a/pkg/transformer/kubernetes/k8sutils.go b/pkg/transformer/kubernetes/k8sutils.go index 5e7da57e..82421630 100644 --- a/pkg/transformer/kubernetes/k8sutils.go +++ b/pkg/transformer/kubernetes/k8sutils.go @@ -31,6 +31,7 @@ import ( "text/template" "time" + "github.com/compose-spec/compose-go/v2/dotenv" "github.com/compose-spec/compose-go/v2/types" "github.com/joho/godotenv" "github.com/kubernetes/kompose/pkg/kobject" @@ -962,6 +963,10 @@ func GetEnvsFromFile(file string) (map[string]string, error) { return envLoad, nil } +func LoadEnvFiles(file string, lookup func(key string) (string, bool)) (map[string]string, error) { + return dotenv.ReadWithLookup(lookup, file) +} + // GetContentFromFile gets the content from the file.. func GetContentFromFile(file string) (string, error) { fileBytes, err := os.ReadFile(file) diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index 360cfcc7..66b44b4b 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -219,6 +219,38 @@ func (k *Kubernetes) InitSvc(name string, service kobject.ServiceConfig) *api.Se return svc } +// InitConfigMapForEnvWithLookup initializes a ConfigMap object from an env_file with variable interpolation support +// using the provided lookup function to resolve variable references like ${VAR} or ${VAR:-default} +func (k *Kubernetes) InitConfigMapForEnvWithLookup(name string, opt kobject.ConvertOptions, envFile string, lookup func(key string) (string, bool)) *api.ConfigMap { + workDir, err := transformer.GetComposeFileDir(opt.InputFiles) + if err != nil { + log.Fatalf("Unable to get compose file directory: %s", err) + } + envs, err := LoadEnvFiles(filepath.Join(workDir, envFile), lookup) + if err != nil { + log.Fatalf("Unable to retrieve env file: %s", err) + } + + // Remove root pathing + // replace all other slashes / periods + envName := FormatEnvName(envFile, name) + + // In order to differentiate files, we append to the name and remove '.env' if applicable from the file name + configMap := &api.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: envName, + Labels: transformer.ConfigLabels(name + "-" + envName), + }, + Data: envs, + } + + return configMap +} + // InitConfigMapForEnv initializes a ConfigMap object func (k *Kubernetes) InitConfigMapForEnv(name string, opt kobject.ConvertOptions, envFile string) *api.ConfigMap { workDir, err := transformer.GetComposeFileDir(opt.InputFiles) @@ -1326,13 +1358,8 @@ func (k *Kubernetes) CreateWorkloadAndConfigMapObjects(name string, service kobj objects = append(objects, k.InitSS(name, service, replica)) } - if len(service.EnvFile) > 0 { - for _, envFile := range service.EnvFile { - configMap := k.InitConfigMapForEnv(name, opt, envFile) - objects = append(objects, configMap) - } - } - + envConfigMaps := k.PargeEnvFiletoConfigMaps(name, service, opt) + objects = append(objects, envConfigMaps...) return objects } @@ -1671,13 +1698,8 @@ func (k *Kubernetes) Transform(komposeObject kobject.KomposeObject, opt kobject. pod := k.InitPod(name, service) objects = append(objects, pod) } - - if len(service.EnvFile) > 0 { - for _, envFile := range service.EnvFile { - configMap := k.InitConfigMapForEnv(name, opt, envFile) - objects = append(objects, configMap) - } - } + envConfigMaps := k.PargeEnvFiletoConfigMaps(name, service, opt) + objects = append(objects, envConfigMaps...) } else { objects = k.CreateWorkloadAndConfigMapObjects(name, service, opt) } @@ -1776,3 +1798,19 @@ func (k *Kubernetes) configHorizontalPodScaler(name string, service kobject.Serv *objects = append(*objects, &hpa) return nil } + +func (k *Kubernetes) PargeEnvFiletoConfigMaps(name string, service kobject.ServiceConfig, opt kobject.ConvertOptions) []runtime.Object { + envs := make(map[string]string) + for _, env := range service.Environment { + envs[env.Name] = env.Value + } + configMaps := make([]runtime.Object, 0) + for _, envFile := range service.EnvFile { + configMap := k.InitConfigMapForEnvWithLookup(name, opt, envFile, func(key string) (string, bool) { + v, ok := envs[key] + return v, ok + }) + configMaps = append(configMaps, configMap) + } + return configMaps +} diff --git a/pkg/transformer/kubernetes/kubernetes_test.go b/pkg/transformer/kubernetes/kubernetes_test.go index e252073c..f4a6422e 100644 --- a/pkg/transformer/kubernetes/kubernetes_test.go +++ b/pkg/transformer/kubernetes/kubernetes_test.go @@ -19,6 +19,8 @@ package kubernetes import ( "encoding/json" "fmt" + "os" + "path/filepath" "reflect" "strings" "testing" @@ -36,6 +38,7 @@ import ( networkingv1beta1 "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -1254,3 +1257,79 @@ func newSecrets(stringsSecretConfig SecretsConfig) types.Secrets { }, } } + +// TestPargeEnvFiletoConfigMaps tests the conversion of environment variable files to ConfigMap objects +func TestPargeEnvFiletoConfigMaps(t *testing.T) { + // Prepare a temp .env file for the expression test + tempFile, err := os.CreateTemp("", ".env") + if err != nil { + t.Fatalf("Failed to create temp env file: %v", err) + } + defer os.Remove(tempFile.Name()) + content := []byte(`FOO=bar +BAR=${FOO}_baz +DOC_ENGINE=${DOC_ENGINE:-elasticsearch} +COMPOSE_PROFILES=${DOC_ENGINE} +UNDEFINED_VAR=${MISSING_VAR:-default_value} +`) + if _, err := tempFile.Write(content); err != nil { + t.Fatalf("Failed to write to temp env file: %v", err) + } + tempFile.Close() + + tempFileName := filepath.Base(tempFile.Name()) + testCases := map[string]struct { + service kobject.ServiceConfig + opt kobject.ConvertOptions + want int + check func(t *testing.T, cms []runtime.Object) + }{ + "Env file with variable expressions": { + service: kobject.ServiceConfig{ + Name: "test-app", + Environment: []kobject.EnvVar{ + { + Name: "DOC_ENGINE", + Value: "test-env", + }, + }, + EnvFile: []string{tempFileName}, + }, + opt: kobject.ConvertOptions{InputFiles: []string{tempFile.Name()}}, + want: 1, + check: func(t *testing.T, cms []runtime.Object) { + cm, ok := cms[0].(*api.ConfigMap) + if !ok { + t.Errorf("Returned object is not a ConfigMap") + return + } + if cm.Data["FOO"] != "bar" { + t.Errorf("Expected FOO=bar, got %s", cm.Data["FOO"]) + } + if cm.Data["BAR"] != "bar_baz" { + t.Errorf("Expected BAR=bar_baz, got %s", cm.Data["BAR"]) + } + if cm.Data["DOC_ENGINE"] != "test-env" { + t.Errorf("Expected DOC_ENGINE=test-env, got %s", cm.Data["DOC_ENGINE"]) + } + if cm.Data["COMPOSE_PROFILES"] != "test-env" { + t.Errorf("Expected COMPOSE_PROFILES=test-env, got %s", cm.Data["COMPOSE_PROFILES"]) + } + if cm.Data["UNDEFINED_VAR"] != "default_value" { + t.Errorf("Expected UNDEFINED_VAR=default_value, got %s", cm.Data["UNDEFINED_VAR"]) + } + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + k := Kubernetes{} + cms := k.PargeEnvFiletoConfigMaps(tc.service.Name, tc.service, tc.opt) + if len(cms) != tc.want { + t.Errorf("Expected %d ConfigMaps, got %d", tc.want, len(cms)) + } + tc.check(t, cms) + }) + } +} diff --git a/pkg/transformer/openshift/openshift.go b/pkg/transformer/openshift/openshift.go index ae382b7d..2947de4e 100644 --- a/pkg/transformer/openshift/openshift.go +++ b/pkg/transformer/openshift/openshift.go @@ -340,12 +340,8 @@ func (o *OpenShift) Transform(komposeObject kobject.KomposeObject, opt kobject.C objects = append(objects, pod) } - if len(service.EnvFile) > 0 { - for _, envFile := range service.EnvFile { - configMap := o.InitConfigMapForEnv(name, opt, envFile) - objects = append(objects, configMap) - } - } + envConfigMaps := o.PargeEnvFiletoConfigMaps(name, service, opt) + objects = append(objects, envConfigMaps...) } else { objects = o.CreateWorkloadAndConfigMapObjects(name, service, opt) diff --git a/script/test/cmd/tests.sh b/script/test/cmd/tests.sh index ccb185ff..7991f4cd 100755 --- a/script/test/cmd/tests.sh +++ b/script/test/cmd/tests.sh @@ -282,6 +282,14 @@ k8s_output="$KOMPOSE_ROOT/script/test/fixtures/compose-env-no-interpolation/outp convert::expect_success "$k8s_cmd" "$k8s_output" || exit 1 convert::expect_success "$os_cmd" "$os_output" || exit 1 +# Test configmap generated by env_file with variable interpolation +k8s_cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/env-lookup/compose.yaml convert --stdout --with-kompose-annotation=false" +os_cmd="kompose --provider=openshift -f $KOMPOSE_ROOT/script/test/fixtures/env-lookup/compose.yaml convert --stdout --with-kompose-annotation=false" +k8s_output="$KOMPOSE_ROOT/script/test/fixtures/env-lookup/output-k8s.yaml" +os_output="$KOMPOSE_ROOT/script/test/fixtures/env-lookup/output-os.yaml" +convert::expect_success_and_warning "$k8s_cmd" "$k8s_output" || exit 1 +convert::expect_success_and_warning "$os_cmd" "$os_output" || exit 1 + # Test support for subpath volume k8s_cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/vols-subpath/compose.yaml convert --stdout --with-kompose-annotation=false" k8s_output="$KOMPOSE_ROOT/script/test/fixtures/vols-subpath/output-k8s.yaml" diff --git a/script/test/fixtures/envfile-interpolation/.env b/script/test/fixtures/envfile-interpolation/.env new file mode 100644 index 00000000..6e6df478 --- /dev/null +++ b/script/test/fixtures/envfile-interpolation/.env @@ -0,0 +1,4 @@ +DOC_ENGINE=${DOC_ENGINE:-elasticsearch} +COMPOSE_PROFILES=${DOC_ENGINE} +MINIO_CONSOLE_PORT=9001 +MINIO_PORT=9000 \ No newline at end of file diff --git a/script/test/fixtures/envfile-interpolation/compose.yaml b/script/test/fixtures/envfile-interpolation/compose.yaml new file mode 100644 index 00000000..28233ec1 --- /dev/null +++ b/script/test/fixtures/envfile-interpolation/compose.yaml @@ -0,0 +1,14 @@ +version: '3' + +services: + minio: + image: quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z + container_name: ragflow-minio + command: server --console-address ":9001" /data + ports: + - ${MINIO_PORT}:9000 + - ${MINIO_CONSOLE_PORT}:9001 + env_file: .env + environment: + - DOC_ENGINE=test-env + restart: on-failure diff --git a/script/test/fixtures/envfile-interpolation/output-k8s.yaml b/script/test/fixtures/envfile-interpolation/output-k8s.yaml new file mode 100644 index 00000000..36190dd5 --- /dev/null +++ b/script/test/fixtures/envfile-interpolation/output-k8s.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: minio + name: minio +spec: + ports: + - name: "9000" + port: 9000 + targetPort: 9000 + - name: "9001" + port: 9001 + targetPort: 9001 + selector: + io.kompose.service: minio + +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + io.kompose.service: minio + name: minio +spec: + containers: + - args: + - server + - --console-address + - :9001 + - /data + envFrom: + - configMapRef: + name: env + image: quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z + name: ragflow-minio + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9001 + protocol: TCP + restartPolicy: OnFailure + +--- +apiVersion: v1 +data: + COMPOSE_PROFILES: test-env + DOC_ENGINE: test-env + MINIO_CONSOLE_PORT: "9001" + MINIO_PORT: "9000" +kind: ConfigMap +metadata: + labels: + io.kompose.service: minio-env + name: env \ No newline at end of file diff --git a/script/test/fixtures/envfile-interpolation/output-os.yaml b/script/test/fixtures/envfile-interpolation/output-os.yaml new file mode 100644 index 00000000..36190dd5 --- /dev/null +++ b/script/test/fixtures/envfile-interpolation/output-os.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: minio + name: minio +spec: + ports: + - name: "9000" + port: 9000 + targetPort: 9000 + - name: "9001" + port: 9001 + targetPort: 9001 + selector: + io.kompose.service: minio + +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + io.kompose.service: minio + name: minio +spec: + containers: + - args: + - server + - --console-address + - :9001 + - /data + envFrom: + - configMapRef: + name: env + image: quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z + name: ragflow-minio + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9001 + protocol: TCP + restartPolicy: OnFailure + +--- +apiVersion: v1 +data: + COMPOSE_PROFILES: test-env + DOC_ENGINE: test-env + MINIO_CONSOLE_PORT: "9001" + MINIO_PORT: "9000" +kind: ConfigMap +metadata: + labels: + io.kompose.service: minio-env + name: env \ No newline at end of file