info: improve allowed character testing

cmd/info/info.go

@@ -21,11 +21,22 @@ import (
 	"github.com/spf13/cobra"
 )
 
+type position int
+
+const (
+	positionMiddle position = 1 << iota
+	positionLeft
+	positionRight
+	positionNone position = 0
+	positionAll  position = positionRight<<1 - 1
+)
+
 var (
 	checkNormalization bool
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
+	positionList       = []position{positionMiddle, positionLeft, positionRight}
 )
 
 func init() {
@@ -59,7 +70,7 @@ a bit of go code for each one.
 type results struct {
 	f                    fs.Fs
 	mu                   sync.Mutex
-	charNeedsEscaping    map[rune]bool
+	stringNeedsEscaping  map[string]position
 	maxFileLength        int
 	canWriteUnnormalized bool
 	canReadUnnormalized  bool
@@ -70,7 +81,7 @@ type results struct {
 func newResults(f fs.Fs) *results {
 	return &results{
 		f:                   f,
-		charNeedsEscaping: make(map[rune]bool),
+		stringNeedsEscaping: make(map[string]position),
 	}
 }
 
@@ -79,13 +90,13 @@ func (r *results) Print() {
 	fmt.Printf("// %s\n", r.f.Name())
 	if checkControl {
 		escape := []string{}
-		for c, needsEscape := range r.charNeedsEscaping {
+		for c, needsEscape := range r.stringNeedsEscaping {
-			if needsEscape {
+			if needsEscape != positionNone {
 				escape = append(escape, fmt.Sprintf("0x%02X", c))
 			}
 		}
 		sort.Strings(escape)
-		fmt.Printf("charNeedsEscaping = []byte{\n")
+		fmt.Printf("stringNeedsEscaping = []byte{\n")
 		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
 		fmt.Printf("}\n")
 	}
@@ -130,20 +141,45 @@ func (r *results) checkUTF8Normalization() {
 	}
 }
 
-// check we can write file with the rune passed in
+func (r *results) checkStringPositions(s string) {
-func (r *results) checkChar(c rune) {
+	fs.Infof(r.f, "Writing position file 0x%0X", s)
-	fs.Infof(r.f, "Writing file 0x%02X", c)
+	positionError := positionNone
-	path := fmt.Sprintf("0x%02X-%c-", c, c)
+
-	_, err := r.writeFile(path)
+	for _, pos := range positionList {
-	escape := false
+		path := ""
-	if err != nil {
+		switch pos {
-		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
+		case positionMiddle:
-		escape = true
+			path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
-	} else {
+		case positionLeft:
-		fs.Infof(r.f, "OK writing file 0x%02X", c)
+			path = fmt.Sprintf("%s-position-left-%0X", s, s)
+		case positionRight:
+			path = fmt.Sprintf("position-right-%0X-%s", s, s)
+		default:
+			panic("invalid position: " + pos.String())
 		}
+		_, writeErr := r.writeFile(path)
+		if writeErr != nil {
+			fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
+		} else {
+			fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
+		}
+		obj, getErr := r.f.NewObject(path)
+		if getErr != nil {
+			fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
+		} else {
+			if obj.Size() != 50 {
+				fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
+			} else {
+				fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
+			}
+		}
+		if writeErr != nil || getErr != nil {
+			positionError += pos
+		}
+	}
+
 	r.mu.Lock()
-	r.charNeedsEscaping[c] = escape
+	r.stringNeedsEscaping[s] = positionError
 	r.mu.Unlock()
 }
 
@@ -157,19 +193,28 @@ func (r *results) checkControls() {
 	}
 	var wg sync.WaitGroup
 	for i := rune(0); i < 128; i++ {
+		s := string(i)
 		if i == 0 || i == '/' {
 			// We're not even going to check NULL or /
-			r.charNeedsEscaping[i] = true
+			r.stringNeedsEscaping[s] = positionAll
 			continue
 		}
 		wg.Add(1)
-		c := i
+		go func(s string) {
-		go func() {
 			defer wg.Done()
 			token := <-tokens
-			r.checkChar(c)
+			r.checkStringPositions(s)
 			tokens <- token
-		}()
+		}(s)
+	}
+	for _, s := range []string{"＼", "\xBF", "\xFE"} {
+		wg.Add(1)
+		go func(s string) {
+			defer wg.Done()
+			token := <-tokens
+			r.checkStringPositions(s)
+			tokens <- token
+		}(s)
 	}
 	wg.Wait()
 	fs.Infof(r.f, "Done trying to create control character file names")
@@ -268,3 +313,35 @@ func readInfo(f fs.Fs) error {
 	r.Print()
 	return nil
 }
+
+func (e position) String() string {
+	switch e {
+	case positionNone:
+		return "none"
+	case positionAll:
+		return "all"
+	}
+	var buf bytes.Buffer
+	if e&positionMiddle != 0 {
+		buf.WriteString("middle")
+		e &= ^positionMiddle
+	}
+	if e&positionLeft != 0 {
+		if buf.Len() != 0 {
+			buf.WriteRune(',')
+		}
+		buf.WriteString("left")
+		e &= ^positionLeft
+	}
+	if e&positionRight != 0 {
+		if buf.Len() != 0 {
+			buf.WriteRune(',')
+		}
+		buf.WriteString("right")
+		e &= ^positionRight
+	}
+	if e != positionNone {
+		panic("invalid position")
+	}
+	return buf.String()
+}

cmd/info/process.sh

@@ -0,0 +1,40 @@
+set -euo pipefail
+
+for f in info-*.log; do 
+  for pos in middle left right; do
+    egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos
+    egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos
+  done
+  {
+    echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}"
+    echo "Write\tWrite\tWrite\tGet\tGet\tGet"
+    echo "Mid\tLeft\tRight\tMid\tLeft\tRight"
+    paste $f.write_{middle,left,right} $f.get_{middle,left,right}
+  } > $f.csv
+done
+
+for f in info-*.list; do
+  for pos in middle left right; do
+    cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos
+  done
+  {
+    echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}"
+    echo "List\tList\tList"
+    echo "Mid\tLeft\tRight"
+    for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do
+      echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss)
+      echo -n "\t"
+      echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/'    $f.left   | tr -d "\t\r" | grep -a . || echo Miss)
+      echo -n "\t"
+      echo    $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/'   $f.right  | tr -d "\t\r" | grep -a . || echo Miss)
+      # echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t"
+      # echo -n $(grep -a "position-left-$e"    $f.left   | tr -d "\t\r" || echo Miss)"\t"
+      # echo    $(grep -a "position-right-$e-"  $f.right  | tr -d "\t\r" || echo Miss)
+    done
+    } > $f.csv
+done
+
+for f in info-*.list; do 
+    paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv
+done
+paste *.full.csv > info-complete.csv

cmd/info/test.cmd

@@ -0,0 +1,3 @@
+rclone.exe purge    info
+rclone.exe info -vv info > info-LocalWindows.log  2>&1
+rclone.exe ls   -vv info > info-LocalWindows.list 2>&1

cmd/info/test.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env zsh
+#
+# example usage: 
+# $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh --list | \
+#   parallel -P20 $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh
+
+export PATH=$GOPATH/src/github.com/ncw/rclone:$PATH
+
+typeset -A allRemotes
+ allRemotes=(
+  TestAmazonCloudDrive '--low-level-retries=2 --checkers=5'
+  TestB2 ''
+  TestBox ''
+  TestDrive '--tpslimit=5'
+  TestCrypt ''
+  TestDropbox '--checkers=1'
+  TestJottacloud ''
+  TestMega ''
+  TestOneDrive ''
+  TestOpenDrive '--low-level-retries=2 --checkers=5'
+  TestPcloud '--low-level-retries=2 --timeout=15s'
+  TestS3 ''
+  Local ''
+)
+
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+ set -- ${(k)allRemotes[@]}
+elif [[ $1 = --list ]]; then
+  printf '%s\n' ${(k)allRemotes[@]}
+  exit 0
+fi
+
+for remote; do
+  dir=$remote:infotest
+  if [[ $remote = Local ]]; then
+    dir=infotest
+  fi
+  rclone purge    $dir || :
+  rclone info -vv $dir ${=allRemotes[$remote]} &> info-$remote.log
+  rclone ls   -vv $dir &> info-$remote.list
+done