dependency updates for geth 1.10.25 (#65)

* cerc refactor waiting on unpublished dependencies

* more cerc_refactor with published ipld-eth-db unstable

* TearDownDB refactor

* missed second publisher_test TearDownDB

README.md

@@ -191,3 +191,19 @@ Config format:
     ```
 
 * NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported snapshot data with the data generated by statediffing. Reference: https://www.postgresql.org/docs/14/sql-copy.html
+
+### Troubleshooting
+
+* Run the following command to find any rows (in data dumps in `file` mode) having unexpected number of columns:
+
+    ```bash
+    ./scripts/find-bad-rows.sh -i <input-file> -c <expected-columns> -o [output-file] -d true
+    ```
+
+* Run the following command to select rows (from data dumps in `file` mode) other than the ones having unexpected number of columns:
+
+    ```bash
+    ./scripts/filter-bad-rows.sh -i <input-file> -c <expected-columns> -o <output-file>
+    ```
+
+* See [scripts](./scripts) for more details.

cmd/inplaceStateSnapshot.go

@@ -21,7 +21,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
 )
 
 // inPlaceStateSnapshotCmd represents the inPlaceStateSnapshot command

cmd/root.go

@@ -25,8 +25,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
 )
 
 var (

cmd/stateSnapshot.go

@@ -22,7 +22,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot"
 )
 
 // stateSnapshotCmd represents the stateSnapshot command

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: vulcanize/ipld-eth-db:v4.2.1-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.1-alpha-unstable
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "vulcanize_testing"

fixture/in_place_snapshot.go

@@ -18,9 +18,9 @@ package fixture
 import (
 	"math/big"
 
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )
 
 type Block struct {

fixture/service.go

@@ -6,7 +6,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 )
 
 var Block1_Header = types.Header{

go.mod

@@ -1,9 +1,10 @@
-module github.com/vulcanize/ipld-eth-state-snapshot
+module github.com/cerc-io/ipld-eth-state-snapshot
 
 go 1.18
 
 require (
-	github.com/ethereum/go-ethereum v1.10.23
+	github.com/cerc-io/go-eth-state-node-iterator v1.1.8
+	github.com/ethereum/go-ethereum v1.10.25
 	github.com/golang/mock v1.6.0
 	github.com/ipfs/go-cid v0.2.0
 	github.com/ipfs/go-ipfs-blockstore v1.2.0
@@ -14,7 +15,6 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.12.0
-	github.com/vulcanize/go-eth-state-node-iterator v1.1.4
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 )
 
@@ -110,4 +110,4 @@ require (
 	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.23 => github.com/vulcanize/go-ethereum v1.10.23-statediff-4.2.0-alpha
+replace github.com/ethereum/go-ethereum v1.10.25 => github.com/cerc-io/go-ethereum v1.10.25-statediff-4.2.1-alpha

go.sum

@@ -75,6 +75,10 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cerc-io/go-eth-state-node-iterator v1.1.8 h1:DKXrvxRbWuAJWAAEeEL4IQOf7mKLcwfgwN9IiohzAv0=
+github.com/cerc-io/go-eth-state-node-iterator v1.1.8/go.mod h1:rj+3CZ0Vf9uiS1M0gYzfapMUNZbxpA5HDVyRkiOU6Kc=
+github.com/cerc-io/go-ethereum v1.10.25-statediff-4.2.1-alpha h1:SDzJBohLSNO9sFYXqtwJWuYjg+fxxJ/2IpnHZw4x03o=
+github.com/cerc-io/go-ethereum v1.10.25-statediff-4.2.1-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -679,10 +683,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y=
-github.com/vulcanize/go-eth-state-node-iterator v1.1.4 h1:2jIbgZ/yxrNZt5zmsXtvUPSxT0sP5k1PR5/O1GN/Ma0=
-github.com/vulcanize/go-eth-state-node-iterator v1.1.4/go.mod h1:kFzlSi7rUB9mJWZf08xIEiiGuXnnUEn48It+WMHzxPc=
-github.com/vulcanize/go-ethereum v1.10.23-statediff-4.2.0-alpha h1:iy1uhdazPIlUJcCu6Kal+hFQiZblTDwSe2i644o167Y=
-github.com/vulcanize/go-ethereum v1.10.23-statediff-4.2.0-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=

main.go

@@ -18,7 +18,7 @@ package main
 import (
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-state-snapshot/cmd"
+	"github.com/cerc-io/ipld-eth-state-snapshot/cmd"
 )
 
 func main() {

pkg/snapshot/file/publisher.go

@@ -32,11 +32,11 @@ import (
 	"github.com/multiformats/go-multihash"
 	"github.com/sirupsen/logrus"
 
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )
 
 var _ snapt.Publisher = (*publisher)(nil)

pkg/snapshot/file/publisher_test.go

@@ -9,13 +9,13 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 
-	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
-	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
+	"github.com/cerc-io/ipld-eth-state-snapshot/test"
 )
 
 var (
@@ -93,7 +93,7 @@ func TestPgCopy(t *testing.T) {
 	test.NoError(t, err)
 	db := postgres.NewPostgresDB(driver)
 
-	sql.TearDownDB(t, db)
+	test_helpers.TearDownDB(t, db)
 
 	// copy from files
 	pgCopyStatement := `COPY %s FROM '%s' CSV`

pkg/snapshot/in_place_snapshot.go

@@ -16,9 +16,9 @@
 package snapshot
 
 import (
+	. "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
-	. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )
 
 const (

pkg/snapshot/in_place_snapshot_test.go

@@ -27,10 +27,10 @@ import (
 	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 	"github.com/multiformats/go-multihash"
 
-	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
-	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/pg"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
+	"github.com/cerc-io/ipld-eth-state-snapshot/test"
 )
 
 var (

pkg/snapshot/pg/publisher.go

@@ -31,12 +31,12 @@ import (
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )
 
 var _ snapt.Publisher = (*publisher)(nil)

pkg/snapshot/pg/publisher_test.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 
-	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
-	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
+	"github.com/cerc-io/ipld-eth-state-snapshot/test"
 )
 
 var (
@@ -48,7 +48,7 @@ func TestBasic(t *testing.T) {
 	test.NoError(t, err)
 	db := postgres.NewPostgresDB(driver)
 
-	sql.TearDownDB(t, db)
+	test_helpers.TearDownDB(t, db)
 
 	_ = writeData(t, db)
 

pkg/snapshot/service.go

@@ -33,9 +33,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
-	iter "github.com/vulcanize/go-eth-state-node-iterator"
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
-	. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	iter "github.com/cerc-io/go-eth-state-node-iterator"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
+	. "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 )
 
 var (

pkg/snapshot/service_test.go

@@ -15,10 +15,10 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/golang/mock/gomock"
 
-	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
-	mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
-	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+	fixt "github.com/cerc-io/ipld-eth-state-snapshot/fixture"
+	mock "github.com/cerc-io/ipld-eth-state-snapshot/mocks/snapshot"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
+	"github.com/cerc-io/ipld-eth-state-snapshot/test"
 )
 
 var (

pkg/snapshot/tracker.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 	log "github.com/sirupsen/logrus"
 
-	iter "github.com/vulcanize/go-eth-state-node-iterator"
+	iter "github.com/cerc-io/go-eth-state-node-iterator"
 )
 
 type trackedIter struct {

pkg/snapshot/util.go

@@ -7,10 +7,10 @@ import (
 
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 
-	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
-	file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file"
-	pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
-	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	"github.com/cerc-io/ipld-eth-state-snapshot/pkg/prom"
+	file "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/file"
+	pg "github.com/cerc-io/ipld-eth-state-snapshot/pkg/snapshot/pg"
+	snapt "github.com/cerc-io/ipld-eth-state-snapshot/pkg/types"
 )
 
 func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {

scripts/README.md

@@ -0,0 +1,73 @@
+## Data Validation
+
+* For a given table in the `ipld-eth-db` schema, we know the number of columns to be expected in each row in the data dump:
+
+  | Table             | Expected columns |
+  | ----------------- |:----------------:|
+  | public.nodes      | 5                |
+  | public.blocks     | 3                |
+  | eth.header_cids   | 16               |
+  | eth.state_cids    | 8                |
+  | eth.storage_cids  | 9                |
+
+### Find Bad Data
+
+* Run the following command to find any rows having unexpected number of columns:
+
+  ```bash
+  ./scripts/find-bad-rows.sh -i <input-file> -c <expected-columns> -o [output-file] -d [include-data]
+  ```
+
+  * `input-file` `-i`: Input data file path
+  * `expected-columns` `-c`: Expected number of columns in each row of the input file
+  * `output-file` `-o`: Output destination file path (default: `STDOUT`)
+  * `include-data` `-d`: Whether to include the data row in the output (`true | false`) (default: `false`)
+  * The output is of format: row number, number of columns, the data row
+
+    Eg:
+
+    ```bash
+    ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
+    ```
+    
+    Output:
+
+    ```
+    1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
+    ```
+
+    Eg:
+
+    ```bash
+    ./scripts/find-bad-rows.sh -i public.nodes.csv -c 5 -o res.txt -d true
+    ./scripts/find-bad-rows.sh -i public.blocks.csv -c 3 -o res.txt -d true
+    ./scripts/find-bad-rows.sh -i eth.header_cids.csv -c 16 -o res.txt -d true
+    ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
+    ./scripts/find-bad-rows.sh -i eth.storage_cids.csv -c 9 -o res.txt -d true
+    ```
+
+## Data Cleanup
+
+* In case of column count mismatch, data from `file` mode dumps can't be imported readily into `ipld-eth-db`.
+
+### Filter Bad Data
+
+* Run the following command to filter out rows having unexpected number of columns:
+
+  ```bash
+  ./scripts/filter-bad-rows.sh -i <input-file> -c <expected-columns> -o <output-file>
+  ```
+
+  * `input-file` `-i`: Input data file path
+  * `expected-columns` `-c`: Expected number of columns in each row of the input file
+  * `output-file` `-o`: Output destination file path
+
+    Eg:
+
+    ```bash
+    ./scripts/filter-bad-rows.sh -i public.nodes.csv -c 5 -o cleaned-public.nodes.csv
+    ./scripts/filter-bad-rows.sh -i public.blocks.csv -c 3 -o cleaned-public.blocks.csv
+    ./scripts/filter-bad-rows.sh -i eth.header_cids.csv -c 16 -o cleaned-eth.header_cids.csv
+    ./scripts/filter-bad-rows.sh -i eth.state_cids.csv -c 8 -o cleaned-eth.state_cids.csv
+    ./scripts/filter-bad-rows.sh -i eth.storage_cids.csv -c 9 -o cleaned-eth.storage_cids.csv
+    ```

scripts/filter-bad-rows.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# flags
+# -i <input-file>:        Input data file path
+# -c <expected-columns>:  Expected number of columns in each row of the input file
+# -o [output-file]:       Output destination file path
+
+# eg: ./scripts/filter-bad-rows.sh -i eth.state_cids.csv -c 8 -o cleaned-eth.state_cids.csv
+
+while getopts i:c:o: OPTION
+do
+  case "${OPTION}" in
+    i) inputFile=${OPTARG};;
+    c) expectedColumns=${OPTARG};;
+    o) outputFile=${OPTARG};;
+  esac
+done
+
+timestamp=$(date +%s)
+
+# select only rows having expected number of columns
+if [ -z "${outputFile}" ]; then
+  echo "Invalid destination file arg (-o) ${outputFile}"
+else
+  awk -F"," "NF==${expectedColumns}" ${inputFile} > ${outputFile}
+fi
+
+difference=$(($(date +%s)-timestamp))
+echo Time taken: $(date -d@${difference} -u +%H:%M:%S)

scripts/find-bad-rows.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# flags
+# -i <input-file>:        Input data file path
+# -c <expected-columns>:  Expected number of columns in each row of the input file
+# -o [output-file]:       Output destination file path (default: STDOUT)
+# -d [include-data]:      Whether to include the data row in output (true | false) (default: false)
+
+# eg: ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
+# output: 1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,
+#         baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,
+#         DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
+
+while getopts i:c:o:d: OPTION
+do
+  case "${OPTION}" in
+    i) inputFile=${OPTARG};;
+    c) expectedColumns=${OPTARG};;
+    o) outputFile=${OPTARG};;
+    d) data=${OPTARG};;
+  esac
+done
+
+timestamp=$(date +%s)
+
+# if data requested, dump row number, number of columns and the row
+if [ "${data}" = true ] ; then
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile} > ${outputFile}
+  fi
+# else, dump only row number, number of columns
+else
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile} > ${outputFile}
+  fi
+fi
+
+difference=$(($(date +%s)-timestamp))
+echo Time taken: $(date -d@${difference} -u +%H:%M:%S)