enable -race on go test (#7305)

Introduce 'norace' build tag that is used to
filter out test files containing test cases
or test suites that, if run with the -race
flag on would cause race conditions.

Supersede #7300.

.github/workflows/test.yml

@@ -31,6 +31,7 @@ jobs:
       - name: Run cosmovisor tests
         run: cd cosmovisor; make
         if: "env.GIT_DIFF != ''"
+
   split-test-files:
     runs-on: ubuntu-latest
     steps:
@@ -76,7 +77,7 @@ jobs:
         if: "env.GIT_DIFF != ''"
       - name: test & coverage report creation
         run: |
-          cat xaa.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock'
+          cat xaa.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='norace ledger test_ledger_mock'
         if: "env.GIT_DIFF != ''"
       - name: filter out DONTCOVER
         run: |
@@ -113,7 +114,7 @@ jobs:
         if: "env.GIT_DIFF != ''"
       - name: test & coverage report creation
         run: |
-          cat xab.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock'
+          cat xab.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='norace ledger test_ledger_mock'
         if: "env.GIT_DIFF != ''"
       - name: filter out DONTCOVER
         run: |
@@ -150,7 +151,7 @@ jobs:
         if: "env.GIT_DIFF != ''"
       - name: test & coverage report creation
         run: |
-          cat xac.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock'
+          cat xac.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='norace ledger test_ledger_mock'
         if: "env.GIT_DIFF != ''"
       - name: filter out DONTCOVER
         run: |
@@ -187,7 +188,7 @@ jobs:
         if: "env.GIT_DIFF != ''"
       - name: test & coverage report creation
         run: |
-          cat xad.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock'
+          cat xad.txt | xargs go test -mod=readonly -timeout 15m -coverprofile=coverage.txt -covermode=atomic -tags='norace ledger test_ledger_mock'
         if: "env.GIT_DIFF != ''"
       - name: filter out DONTCOVER
         run: |
@@ -205,6 +206,86 @@ jobs:
           file: ./coverage.txt
         if: "env.GIT_DIFF != ''"
 
+  test-race-1:
+    runs-on: ubuntu-latest
+    needs: split-test-files
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v3
+        id: git_diff
+        with:
+          SUFFIX_FILTER: |
+            .go
+            .mod
+            .sum
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ github.sha }}-aa"
+        if: "env.GIT_DIFF != ''"
+      - name: Run tests with race detector
+        run: cat xaa.txt | xargs go test -mod=readonly -timeout 15m -race -tags='cgo ledger test_ledger_mock'
+
+  test-race-2:
+    runs-on: ubuntu-latest
+    needs: split-test-files
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v3
+        id: git_diff
+        with:
+          SUFFIX_FILTER: |
+            .go
+            .mod
+            .sum
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ github.sha }}-ab"
+        if: "env.GIT_DIFF != ''"
+      - name: Run tests with race detector
+        run: cat xab.txt | xargs go test -mod=readonly -timeout 15m -race -tags='cgo ledger test_ledger_mock'
+
+  test-race-3:
+    runs-on: ubuntu-latest
+    needs: split-test-files
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v3
+        id: git_diff
+        with:
+          SUFFIX_FILTER: |
+            .go
+            .mod
+            .sum
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ github.sha }}-ac"
+        if: "env.GIT_DIFF != ''"
+      - name: Run tests with race detector
+        run: cat xac.txt | xargs go test -mod=readonly -timeout 15m -race -tags='cgo ledger test_ledger_mock'
+
+  test-race-4:
+    runs-on: ubuntu-latest
+    needs: split-test-files
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v3
+        id: git_diff
+        with:
+          SUFFIX_FILTER: |
+            .go
+            .mod
+            .sum
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ github.sha }}-ad"
+        if: "env.GIT_DIFF != ''"
+      - name: Run tests with race detector
+        run: cat xad.txt | xargs go test -mod=readonly -timeout 15m -race -tags='cgo ledger test_ledger_mock'
+
   liveness-test:
     runs-on: ubuntu-latest
     timeout-minutes: 10