Retry only the failing tests in the integration tests

fs/test_all.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"regexp"
 	"runtime"
 	"strings"
 	"time"
@@ -34,8 +35,8 @@ var (
 	}
 	binary = "fs.test"
 	// Flags
-	maxTries = flag.Int("maxtries", 3, "Number of times to try each test")
+	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
-	runTests = flag.String("run", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
+	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
 	verbose  = flag.Bool("verbose", false, "Run the tests with -v")
 	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
 	runOnly  = flag.String("run-only", "", "Run only those tests matching the regexp supplied")
@@ -50,6 +51,8 @@ type test struct {
 	try         int
 	err         error
 	output      []byte
+	failedTests []string
+	runFlag     string
 }
 
 // newTest creates a new test
@@ -73,17 +76,43 @@ func newTest(remote string, subdir bool) *test {
 	return t
 }
 
+var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)
+
+// findFailures looks for all the tests which failed
+func (t *test) findFailures() {
+	t.failedTests = nil
+	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
+		t.failedTests = append(t.failedTests, string(matches[1]))
+	}
+	if len(t.failedTests) != 0 {
+		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
+	} else {
+		t.runFlag = ""
+	}
+	if t.passed() && len(t.failedTests) != 0 {
+		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
+	} else if !t.passed() && len(t.failedTests) == 0 {
+		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
+	}
+}
+
 // trial runs a single test
 func (t *test) trial() {
-	log.Printf("%q - Starting (try %d/%d)", t.cmdString, t.try, *maxTries)
+	cmdLine := t.cmdLine[:]
-	cmd := exec.Command(t.cmdLine[0], t.cmdLine[1:]...)
+	if t.runFlag != "" {
+		cmdLine = append(cmdLine, "-test.run", t.runFlag)
+	}
+	cmdString := strings.Join(cmdLine, " ")
+	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
 	start := time.Now()
 	t.output, t.err = cmd.CombinedOutput()
 	duration := time.Since(start)
+	t.findFailures()
 	if t.passed() {
-		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
+		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
 	} else {
-		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
+		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
 	}
 }
 
@@ -93,7 +122,8 @@ func (t *test) cleanFs() error {
 	if err != nil {
 		return err
 	}
-	for dir := range f.ListDir() {
+	dirs, err := fs.NewLister().SetLevel(1).Start(f, "").GetDirs()
+	for _, dir := range dirs {
 		if fstest.MatchTestRemote.MatchString(dir.Name) {
 			log.Printf("Purging %s%s", t.remote, dir.Name)
 			dir, err := fs.NewFs(t.remote + dir.Name)