MirrorSave
/
rclone
mirror of https://github.com/rclone/rclone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

http: remove duplicates from listings

This commit is contained in:
Nick Craig-Wood 2019-02-08 13:58:22 +00:00
parent 28230d93b4
commit 54a2e99d97
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/http/http.go
View File

@ -270,14 +270,20 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
var walk func(*html.Node) var (
walk func(*html.Node)
seen = make(map[string]struct{})
)
walk = func(n *html.Node) { walk = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "a" { if n.Type == html.ElementNode && n.Data == "a" {
for _, a := range n.Attr { for _, a := range n.Attr {
if a.Key == "href" { if a.Key == "href" {
name, err := parseName(base, a.Val) name, err := parseName(base, a.Val)
if err == nil { if err == nil {
if _, found := seen[name]; !found {
names = append(names, name) names = append(names, name)
seen[name] = struct{}{}
}
} }
break break
} }