diff --git a/cmd/ethtest/main.go b/cmd/ethtest/main.go index 93bf3ce65..c6073ce98 100644 --- a/cmd/ethtest/main.go +++ b/cmd/ethtest/main.go @@ -28,6 +28,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "github.com/codegangsta/cli" "github.com/ethereum/go-ethereum/logger/glog" @@ -40,6 +41,7 @@ var ( defaultTest = "all" defaultDir = "." allTests = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests"} + skipTests = []string{} TestFlag = cli.StringFlag{ Name: "test", @@ -60,6 +62,10 @@ var ( Name: "stdin", Usage: "Accept input from stdin instead of reading from file", } + SkipTestsFlag = cli.StringFlag{ + Name: "skip", + Usage: "Tests names to skip", + } ) func runTestWithReader(test string, r io.Reader) error { @@ -67,13 +73,13 @@ func runTestWithReader(test string, r io.Reader) error { var err error switch test { case "bt", "BlockTest", "BlockTests", "BlockChainTest": - err = tests.RunBlockTestWithReader(r) + err = tests.RunBlockTestWithReader(r, skipTests) case "st", "state", "StateTest", "StateTests": - err = tests.RunStateTestWithReader(r) + err = tests.RunStateTestWithReader(r, skipTests) case "tx", "TransactionTest", "TransactionTests": - err = tests.RunTransactionTestsWithReader(r) + err = tests.RunTransactionTestsWithReader(r, skipTests) case "vm", "VMTest", "VMTests": - err = tests.RunVmTestWithReader(r) + err = tests.RunVmTestWithReader(r, skipTests) default: err = fmt.Errorf("Invalid test type specified: %v", test) } @@ -174,6 +180,7 @@ func setupApp(c *cli.Context) { flagFile := c.GlobalString(FileFlag.Name) continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name) useStdIn := c.GlobalBool(ReadStdInFlag.Name) + skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ") if !useStdIn { runSuite(flagTest, flagFile) @@ -200,6 +207,7 @@ func main() { FileFlag, ContinueOnErrorFlag, ReadStdInFlag, + SkipTestsFlag, } if err := app.Run(os.Args); err != nil { diff --git a/tests/block_test.go b/tests/block_test.go index 9d21ba28d..bdf983786 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -6,67 +6,67 @@ import ( ) func TestBcValidBlockTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcUncleTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } - err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json")) + err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcUncleHeaderValidityTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcInvalidHeaderTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcInvalidRLPTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcRPCAPITests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcForkBlockTests(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcTotalDifficulty(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } } func TestBcWallet(t *testing.T) { - err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json")) + err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"), BlockSkipTests) if err != nil { t.Fatal(err) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 5222b214b..5fdc6402e 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -86,7 +86,7 @@ type btTransaction struct { Value string } -func RunBlockTestWithReader(r io.Reader) error { +func RunBlockTestWithReader(r io.Reader, skipTests []string) error { btjs := make(map[string]*btJSON) if err := readJson(r, &btjs); err != nil { return err @@ -97,13 +97,13 @@ func RunBlockTestWithReader(r io.Reader) error { return err } - if err := runBlockTests(bt); err != nil { + if err := runBlockTests(bt, skipTests); err != nil { return err } return nil } -func RunBlockTest(file string) error { +func RunBlockTest(file string, skipTests []string) error { btjs := make(map[string]*btJSON) if err := readJsonFile(file, &btjs); err != nil { return err @@ -113,15 +113,15 @@ func RunBlockTest(file string) error { if err != nil { return err } - if err := runBlockTests(bt); err != nil { + if err := runBlockTests(bt, skipTests); err != nil { return err } return nil } -func runBlockTests(bt map[string]*BlockTest) error { - skipTest := make(map[string]bool, len(BlockSkipTests)) - for _, name := range BlockSkipTests { +func runBlockTests(bt map[string]*BlockTest, skipTests []string) error { + skipTest := make(map[string]bool, len(skipTests)) + for _, name := range skipTests { skipTest[name] = true } diff --git a/tests/state_test.go b/tests/state_test.go index 9c3d6f209..e58f588f4 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -8,84 +8,84 @@ import ( func TestStateSystemOperations(t *testing.T) { fn := filepath.Join(stateTestDir, "stSystemOperationsTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateExample(t *testing.T) { fn := filepath.Join(stateTestDir, "stExample.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStatePreCompiledContracts(t *testing.T) { fn := filepath.Join(stateTestDir, "stPreCompiledContracts.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateRecursiveCreate(t *testing.T) { fn := filepath.Join(stateTestDir, "stRecursiveCreate.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateSpecial(t *testing.T) { fn := filepath.Join(stateTestDir, "stSpecialTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateRefund(t *testing.T) { fn := filepath.Join(stateTestDir, "stRefundTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateBlockHash(t *testing.T) { fn := filepath.Join(stateTestDir, "stBlockHashTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateInitCode(t *testing.T) { fn := filepath.Join(stateTestDir, "stInitCodeTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateLog(t *testing.T) { fn := filepath.Join(stateTestDir, "stLogTests.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestStateTransaction(t *testing.T) { fn := filepath.Join(stateTestDir, "stTransactionTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestCallCreateCallCode(t *testing.T) { fn := filepath.Join(stateTestDir, "stCallCreateCallCodeTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestMemory(t *testing.T) { fn := filepath.Join(stateTestDir, "stMemoryTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } @@ -95,7 +95,7 @@ func TestMemoryStress(t *testing.T) { t.Skip() } fn := filepath.Join(stateTestDir, "stMemoryStressTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } @@ -105,21 +105,21 @@ func TestQuadraticComplexity(t *testing.T) { t.Skip() } fn := filepath.Join(stateTestDir, "stQuadraticComplexityTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestSolidity(t *testing.T) { fn := filepath.Join(stateTestDir, "stSolidityTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } func TestWallet(t *testing.T) { fn := filepath.Join(stateTestDir, "stWalletTest.json") - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } @@ -127,7 +127,7 @@ func TestWallet(t *testing.T) { func TestStateTestsRandom(t *testing.T) { fns, _ := filepath.Glob("./files/StateTests/RandomTests/*") for _, fn := range fns { - if err := RunStateTest(fn); err != nil { + if err := RunStateTest(fn, StateSkipTests); err != nil { t.Error(err) } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 577935dfa..e9abad788 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -16,26 +16,26 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -func RunStateTestWithReader(r io.Reader) error { +func RunStateTestWithReader(r io.Reader, skipTests []string) error { tests := make(map[string]VmTest) if err := readJson(r, &tests); err != nil { return err } - if err := runStateTests(tests); err != nil { + if err := runStateTests(tests, skipTests); err != nil { return err } return nil } -func RunStateTest(p string) error { +func RunStateTest(p string, skipTests []string) error { tests := make(map[string]VmTest) if err := readJsonFile(p, &tests); err != nil { return err } - if err := runStateTests(tests); err != nil { + if err := runStateTests(tests, skipTests); err != nil { return err } @@ -43,9 +43,9 @@ func RunStateTest(p string) error { } -func runStateTests(tests map[string]VmTest) error { - skipTest := make(map[string]bool, len(StateSkipTests)) - for _, name := range StateSkipTests { +func runStateTests(tests map[string]VmTest, skipTests []string) error { + skipTest := make(map[string]bool, len(skipTests)) + for _, name := range skipTests { skipTest[name] = true } diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 41a20a1bb..70aa65cdd 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -6,21 +6,21 @@ import ( ) func TestTransactions(t *testing.T) { - err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json")) + err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"), TransSkipTests) if err != nil { t.Fatal(err) } } func TestWrongRLPTransactions(t *testing.T) { - err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json")) + err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"), TransSkipTests) if err != nil { t.Fatal(err) } } func Test10MBtx(t *testing.T) { - err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json")) + err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"), TransSkipTests) if err != nil { t.Fatal(err) } diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index af33f2c58..45caf26fd 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -32,9 +32,9 @@ type TransactionTest struct { Transaction TtTransaction } -func RunTransactionTestsWithReader(r io.Reader) error { - skipTest := make(map[string]bool, len(TransSkipTests)) - for _, name := range TransSkipTests { +func RunTransactionTestsWithReader(r io.Reader, skipTests []string) error { + skipTest := make(map[string]bool, len(skipTests)) + for _, name := range skipTests { skipTest[name] = true } @@ -59,18 +59,25 @@ func RunTransactionTestsWithReader(r io.Reader) error { return nil } -func RunTransactionTests(file string) error { - skipTest := make(map[string]bool, len(TransSkipTests)) - for _, name := range TransSkipTests { - skipTest[name] = true - } - - bt := make(map[string]TransactionTest) - if err := readJsonFile(file, &bt); err != nil { +func RunTransactionTests(file string, skipTests []string) error { + tests := make(map[string]TransactionTest) + if err := readJsonFile(file, &tests); err != nil { return err } - for name, test := range bt { + if err := runTransactionTests(tests, skipTests); err != nil { + return err + } + return nil +} + +func runTransactionTests(tests map[string]TransactionTest, skipTests []string) error { + skipTest := make(map[string]bool, len(skipTests)) + for _, name := range skipTests { + skipTest[name] = true + } + + for name, test := range tests { // if the test should be skipped, return if skipTest[name] { glog.Infoln("Skipping transaction test", name) diff --git a/tests/vm_test.go b/tests/vm_test.go index d16d65aac..4e417da5a 100644 --- a/tests/vm_test.go +++ b/tests/vm_test.go @@ -8,91 +8,91 @@ import ( // I've created a new function for each tests so it's easier to identify where the problem lies if any of them fail. func TestVMArithmetic(t *testing.T) { fn := filepath.Join(vmTestDir, "vmArithmeticTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestBitwiseLogicOperation(t *testing.T) { fn := filepath.Join(vmTestDir, "vmBitwiseLogicOperationTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestBlockInfo(t *testing.T) { fn := filepath.Join(vmTestDir, "vmBlockInfoTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestEnvironmentalInfo(t *testing.T) { fn := filepath.Join(vmTestDir, "vmEnvironmentalInfoTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestFlowOperation(t *testing.T) { fn := filepath.Join(vmTestDir, "vmIOandFlowOperationsTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestLogTest(t *testing.T) { fn := filepath.Join(vmTestDir, "vmLogTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestPerformance(t *testing.T) { fn := filepath.Join(vmTestDir, "vmPerformanceTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestPushDupSwap(t *testing.T) { fn := filepath.Join(vmTestDir, "vmPushDupSwapTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestVMSha3(t *testing.T) { fn := filepath.Join(vmTestDir, "vmSha3Test.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestVm(t *testing.T) { fn := filepath.Join(vmTestDir, "vmtests.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestVmLog(t *testing.T) { fn := filepath.Join(vmTestDir, "vmLogTest.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestInputLimits(t *testing.T) { fn := filepath.Join(vmTestDir, "vmInputLimits.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } func TestInputLimitsLight(t *testing.T) { fn := filepath.Join(vmTestDir, "vmInputLimitsLight.json") - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } @@ -100,7 +100,7 @@ func TestInputLimitsLight(t *testing.T) { func TestVMRandom(t *testing.T) { fns, _ := filepath.Glob(filepath.Join(baseDir, "RandomTests", "*")) for _, fn := range fns { - if err := RunVmTest(fn); err != nil { + if err := RunVmTest(fn, VmSkipTests); err != nil { t.Error(err) } } diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index afeedda2a..286991764 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -14,7 +14,7 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -func RunVmTestWithReader(r io.Reader) error { +func RunVmTestWithReader(r io.Reader, skipTests []string) error { tests := make(map[string]VmTest) err := readJson(r, &tests) if err != nil { @@ -25,14 +25,14 @@ func RunVmTestWithReader(r io.Reader) error { return err } - if err := runVmTests(tests); err != nil { + if err := runVmTests(tests, skipTests); err != nil { return err } return nil } -func RunVmTest(p string) error { +func RunVmTest(p string, skipTests []string) error { tests := make(map[string]VmTest) err := readJsonFile(p, &tests) @@ -40,16 +40,16 @@ func RunVmTest(p string) error { return err } - if err := runVmTests(tests); err != nil { + if err := runVmTests(tests, skipTests); err != nil { return err } return nil } -func runVmTests(tests map[string]VmTest) error { - skipTest := make(map[string]bool, len(VmSkipTests)) - for _, name := range VmSkipTests { +func runVmTests(tests map[string]VmTest, skipTests []string) error { + skipTest := make(map[string]bool, len(skipTests)) + for _, name := range skipTests { skipTest[name] = true }