lotus/conformance/corpus_test.go
Raúl Kripalani 45cd510da1 conformance: support multiple protocol versions.
This PR introduces support for running multiple variants of a vector,
each of which targets a unique protocol version.

tvx tooling has been adapted to produce and parse the new version
of the schema.
2020-10-14 14:39:43 +01:00

142 lines
3.5 KiB
Go

package conformance
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/filecoin-project/test-vectors/schema"
)
var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){
schema.ClassMessage: ExecuteMessageVector,
schema.ClassTipset: ExecuteTipsetVector,
}
const (
// EnvSkipConformance, if 1, skips the conformance test suite.
EnvSkipConformance = "SKIP_CONFORMANCE"
// EnvCorpusRootDir is the name of the environment variable where the path
// to an alternative corpus location can be provided.
//
// The default is defaultCorpusRoot.
EnvCorpusRootDir = "CORPUS_DIR"
// defaultCorpusRoot is the directory where the test vector corpus is hosted.
// It is mounted on the Lotus repo as a git submodule.
//
// When running this test, the corpus root can be overridden through the
// -conformance.corpus CLI flag to run an alternate corpus.
defaultCorpusRoot = "../extern/test-vectors/corpus"
)
// ignore is a set of paths relative to root to skip.
var ignore = map[string]struct{}{
".git": {},
"schema.json": {},
}
// TestConformance is the entrypoint test that runs all test vectors found
// in the corpus root directory.
//
// It locates all json files via a recursive walk, skipping over the ignore set,
// as well as files beginning with _. It parses each file as a test vector, and
// runs it via the Driver.
func TestConformance(t *testing.T) {
if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
t.SkipNow()
}
// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
// falling back to defaultCorpusRoot if not provided.
corpusRoot := defaultCorpusRoot
if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
corpusRoot = dir
}
var vectors []string
err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
filename := filepath.Base(path)
rel, err := filepath.Rel(corpusRoot, path)
if err != nil {
t.Fatal(err)
}
if _, ok := ignore[rel]; ok {
// skip over using the right error.
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
// dive into directories.
return nil
}
if filepath.Ext(path) != ".json" {
// skip if not .json.
return nil
}
if ignored := strings.HasPrefix(filename, "_"); ignored {
// ignore files starting with _.
t.Logf("ignoring: %s", rel)
return nil
}
vectors = append(vectors, rel)
return nil
})
if err != nil {
t.Fatal(err)
}
if len(vectors) == 0 {
t.Fatalf("no test vectors found")
}
// Run a test for each vector.
for _, v := range vectors {
path := filepath.Join(corpusRoot, v)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("failed to read test raw file: %s", path)
}
var vector schema.TestVector
err = json.Unmarshal(raw, &vector)
if err != nil {
t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
continue
}
t.Run(v, func(t *testing.T) {
for _, h := range vector.Hints {
if h == schema.HintIncorrect {
t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
t.SkipNow()
}
}
// dispatch the execution depending on the vector class.
invokee, ok := invokees[vector.Class]
if !ok {
t.Fatalf("unsupported test vector class: %s", vector.Class)
}
for _, variant := range vector.Pre.Variants {
variant := variant
t.Run(variant.ID, func(t *testing.T) {
invokee(t, &vector, &variant)
})
}
})
}
}