From 6e33fe6ac8d0f97839fbbb3851064b7a7ee773eb Mon Sep 17 00:00:00 2001 From: andreimarcu Date: Wed, 7 Oct 2015 22:45:34 -0400 Subject: [PATCH] Metadata holds mimetype, sha256sum, archiveFiles --- delete.go | 5 +- display.go | 119 +++++++--------------------- expiry.go | 10 ++- fileserve.go | 21 +++-- meta.go | 218 +++++++++++++++++++++++++++++++++++++-------------- torrent.go | 6 +- upload.go | 43 +++++----- 7 files changed, 239 insertions(+), 183 deletions(-) diff --git a/delete.go b/delete.go index e70bf22..f727f60 100644 --- a/delete.go +++ b/delete.go @@ -23,14 +23,13 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { } // Ensure delete key is correct - deleteKey, err := metadataGetDeleteKey(filename) - + metadata, err := metadataRead(filename) if err != nil { unauthorizedHandler(c, w, r) // 401 - no metadata available return } - if deleteKey == requestKey { + if metadata.DeleteKey == requestKey { fileDelErr := os.Remove(filePath) metaDelErr := os.Remove(metaPath) diff --git a/display.go b/display.go index 7a44795..b60f9bc 100644 --- a/display.go +++ b/display.go @@ -1,23 +1,16 @@ package main import ( - "archive/tar" - "archive/zip" - "compress/bzip2" - "compress/gzip" "encoding/json" - "io" "io/ioutil" "net/http" "os" "path" "path/filepath" - "sort" "strconv" "strings" "time" - "bitbucket.org/taruti/mimemagic" "github.com/dustin/go-humanize" "github.com/flosch/pongo2" "github.com/microcosm-cc/bluemonday" @@ -30,21 +23,24 @@ const maxDisplayFileSizeBytes = 1024 * 512 func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] filePath := path.Join(Config.filesDir, fileName) - fileInfo, err := os.Stat(filePath) - if !fileExistsAndNotExpired(fileName) { + err := checkFile(fileName) + if err == NotFoundErr { notFoundHandler(c, w, r) return } - expiry, _ := metadataGetExpiry(fileName) - var expiryHuman string - if expiry != neverExpire { - expiryHuman = humanize.RelTime(time.Now(), expiry, "", "") + metadata, err := metadataRead(fileName) + if err != nil { + oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") + return } - sizeHuman := humanize.Bytes(uint64(fileInfo.Size())) + var expiryHuman string + if metadata.Expiry != neverExpire { + expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "") + } + sizeHuman := humanize.Bytes(uint64(metadata.Size)) extra := make(map[string]string) - files := []string{} file, _ := os.Open(filePath) defer file.Close() @@ -52,15 +48,15 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { header := make([]byte, 512) file.Read(header) - mimetype := mimemagic.Match("", header) extension := strings.TrimPrefix(filepath.Ext(fileName), ".") if strings.EqualFold("application/json", r.Header.Get("Accept")) { js, _ := json.Marshal(map[string]string{ - "filename": fileName, - "mimetype": mimetype, - "expiry": strconv.FormatInt(expiry.Unix(), 10), - "size": strconv.FormatInt(fileInfo.Size(), 10), + "filename": fileName, + "expiry": strconv.FormatInt(metadata.Expiry.Unix(), 10), + "size": strconv.FormatInt(metadata.Size, 10), + "mimetype": metadata.Mimetype, + "sha256sum": metadata.Sha256sum, }) w.Write(js) return @@ -68,81 +64,20 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { var tpl *pongo2.Template - if strings.HasPrefix(mimetype, "image/") { + if strings.HasPrefix(metadata.Mimetype, "image/") { tpl = Templates["display/image.html"] - } else if strings.HasPrefix(mimetype, "video/") { + + } else if strings.HasPrefix(metadata.Mimetype, "video/") { tpl = Templates["display/video.html"] - } else if strings.HasPrefix(mimetype, "audio/") { + + } else if strings.HasPrefix(metadata.Mimetype, "audio/") { tpl = Templates["display/audio.html"] - } else if mimetype == "application/pdf" { + + } else if metadata.Mimetype == "application/pdf" { tpl = Templates["display/pdf.html"] - } else if mimetype == "application/x-tar" { - f, _ := os.Open(filePath) - defer f.Close() - - tReadr := tar.NewReader(f) - for { - header, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - - if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg { - files = append(files, header.Name) - } - } - sort.Strings(files) - - } else if mimetype == "application/x-gzip" { - f, _ := os.Open(filePath) - defer f.Close() - - gzf, err := gzip.NewReader(f) - if err == nil { - tReadr := tar.NewReader(gzf) - for { - header, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - - if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg { - files = append(files, header.Name) - } - } - sort.Strings(files) - } - } else if mimetype == "application/x-bzip" { - f, _ := os.Open(filePath) - defer f.Close() - - bzf := bzip2.NewReader(f) - tReadr := tar.NewReader(bzf) - for { - header, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - - if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg { - files = append(files, header.Name) - } - } - sort.Strings(files) - - } else if mimetype == "application/zip" { - f, _ := os.Open(filePath) - defer f.Close() - - zf, err := zip.NewReader(f, fileInfo.Size()) - if err == nil { - for _, f := range zf.File { - files = append(files, f.Name) - } - } } else if supportedBinExtension(extension) { - if fileInfo.Size() < maxDisplayFileSizeBytes { + if metadata.Size < maxDisplayFileSizeBytes { bytes, err := ioutil.ReadFile(filePath) if err == nil { extra["extension"] = extension @@ -152,7 +87,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } } else if extension == "md" { - if fileInfo.Size() < maxDisplayFileSizeBytes { + if metadata.Size < maxDisplayFileSizeBytes { bytes, err := ioutil.ReadFile(filePath) if err == nil { unsafe := blackfriday.MarkdownCommon(bytes) @@ -170,12 +105,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } err = tpl.ExecuteWriter(pongo2.Context{ - "mime": mimetype, + "mime": metadata.Mimetype, "filename": fileName, "size": sizeHuman, "expiry": expiryHuman, "extra": extra, - "files": files, + "files": metadata.ArchiveFiles, }, w) if err != nil { diff --git a/expiry.go b/expiry.go index cea3c3b..9deaf72 100644 --- a/expiry.go +++ b/expiry.go @@ -13,7 +13,11 @@ func isTsExpired(ts time.Time) bool { } // Determine if the given filename is expired -func isFileExpired(filename string) bool { - exp, _ := metadataGetExpiry(filename) - return isTsExpired(exp) +func isFileExpired(filename string) (bool, error) { + metadata, err := metadataRead(filename) + if err != nil { + return false, err + } + + return isTsExpired(metadata.Expiry), nil } diff --git a/fileserve.go b/fileserve.go index e3fd5f0..3983232 100644 --- a/fileserve.go +++ b/fileserve.go @@ -13,9 +13,13 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] filePath := path.Join(Config.filesDir, fileName) - if !fileExistsAndNotExpired(fileName) { + err := checkFile(fileName) + if err == NotFoundErr { notFoundHandler(c, w, r) return + } else if err == BadMetadata { + oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") + return } if !Config.allowHotlink { @@ -55,19 +59,24 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } -func fileExistsAndNotExpired(filename string) bool { +func checkFile(filename string) error { filePath := path.Join(Config.filesDir, filename) _, err := os.Stat(filePath) if err != nil { - return false + return NotFoundErr } - if isFileExpired(filename) { + expired, err := isFileExpired(filename) + if err != nil { + return err + } + + if expired { os.Remove(path.Join(Config.filesDir, filename)) os.Remove(path.Join(Config.metaDir, filename)) - return false + return NotFoundErr } - return true + return nil } diff --git a/meta.go b/meta.go index b92171b..9df230f 100644 --- a/meta.go +++ b/meta.go @@ -1,84 +1,188 @@ package main import ( - "bufio" + "archive/tar" + "archive/zip" + "compress/bzip2" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" - "fmt" + "io" + "io/ioutil" "os" "path" - "strconv" + "sort" "time" + + "bitbucket.org/taruti/mimemagic" + "github.com/dchest/uniuri" ) -// Write metadata from Upload struct to file -func metadataWrite(filename string, upload *Upload) error { - // Write metadata, overwriting if necessary +type MetadataJSON struct { + DeleteKey string `json:"delete_key"` + Sha256sum string `json:"sha256sum"` + Mimetype string `json:"mimetype"` + Size int64 `json:"size"` + Expiry int64 `json:"expiry"` + ArchiveFiles []string `json:"archive_files,omitempty"` +} - file, err := os.Create(path.Join(Config.metaDir, upload.Filename)) +type Metadata struct { + DeleteKey string + Sha256sum string + Mimetype string + Size int64 + Expiry time.Time + ArchiveFiles []string +} + +var NotFoundErr = errors.New("File not found.") +var BadMetadata = errors.New("Corrupted metadata.") + +func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) { + file, err := os.Open(path.Join(Config.filesDir, fName)) + fileInfo, err := os.Stat(path.Join(Config.filesDir, fName)) + if err != nil { + return + } + defer file.Close() + + m.Size = fileInfo.Size() + m.Expiry = exp + + if delKey == "" { + m.DeleteKey = uniuri.NewLen(30) + } else { + m.DeleteKey = delKey + } + + // Get first 512 bytes for mimetype detection + header := make([]byte, 512) + file.Read(header) + + m.Mimetype = mimemagic.Match("", header) + + // Compute the sha256sum + hasher := sha256.New() + file.Seek(0, 0) + _, err = io.Copy(hasher, file) + if err == nil { + m.Sha256sum = hex.EncodeToString(hasher.Sum(nil)) + } + file.Seek(0, 0) + + // If archive, grab list of filenames + if m.Mimetype == "application/x-tar" { + tReadr := tar.NewReader(file) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) + } + } + sort.Strings(m.ArchiveFiles) + } else if m.Mimetype == "application/x-gzip" { + gzf, err := gzip.NewReader(file) + if err == nil { + tReadr := tar.NewReader(gzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) + } + } + sort.Strings(m.ArchiveFiles) + } + } else if m.Mimetype == "application/x-bzip" { + bzf := bzip2.NewReader(file) + tReadr := tar.NewReader(bzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) + } + } + sort.Strings(m.ArchiveFiles) + } else if m.Mimetype == "application/zip" { + zf, err := zip.NewReader(file, m.Size) + if err == nil { + for _, f := range zf.File { + m.ArchiveFiles = append(m.ArchiveFiles, f.Name) + } + } + sort.Strings(m.ArchiveFiles) + } + + return +} + +func metadataWrite(filename string, metadata *Metadata) error { + file, err := os.Create(path.Join(Config.metaDir, filename)) + if err != nil { + return err + } + defer file.Close() + + mjson := MetadataJSON{} + mjson.DeleteKey = metadata.DeleteKey + mjson.Mimetype = metadata.Mimetype + mjson.ArchiveFiles = metadata.ArchiveFiles + mjson.Sha256sum = metadata.Sha256sum + mjson.Expiry = metadata.Expiry.Unix() + mjson.Size = metadata.Size + + byt, err := json.Marshal(mjson) if err != nil { return err } - defer file.Close() + _, err = file.Write(byt) + if err != nil { + return err + } - w := bufio.NewWriter(file) - - fmt.Fprintln(w, upload.Expiry.Unix()) - fmt.Fprintln(w, upload.DeleteKey) - - return w.Flush() + return nil } -// Return list of strings from a filename's metadata source -func metadataRead(filename string) ([]string, error) { - file, err := os.Open(path.Join(Config.metaDir, filename)) - +func metadataRead(filename string) (metadata Metadata, err error) { + b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename)) if err != nil { - return nil, err + // Metadata does not exist, generate one + newMData, err := generateMetadata(filename, neverExpire, "") + if err != nil { + return metadata, err + } + metadataWrite(filename, &newMData) + b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename)) + if err != nil { + return metadata, BadMetadata + } } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - return lines, scanner.Err() -} - -func metadataGetExpiry(filename string) (expiry time.Time, err error) { - metadata, err := metadataRead(filename) - - // XXX in this case it's up to the caller to determine proper behavior - // for a nonexistant metadata file or broken file + mjson := MetadataJSON{} + err = json.Unmarshal(b, &mjson) if err != nil { - return + return metadata, BadMetadata } - if len(metadata) < 1 { - err = errors.New("ERR: Metadata file does not contain expiry") - return - } + metadata.DeleteKey = mjson.DeleteKey + metadata.Mimetype = mjson.Mimetype + metadata.ArchiveFiles = mjson.ArchiveFiles + metadata.Sha256sum = mjson.Sha256sum + metadata.Expiry = time.Unix(mjson.Expiry, 0) + metadata.Size = mjson.Size - expirySecs, err := strconv.ParseInt(metadata[0], 10, 64) - expiry = time.Unix(expirySecs, 0) return } - -func metadataGetDeleteKey(filename string) (string, error) { - metadata, err := metadataRead(filename) - - if len(metadata) < 2 { - err := errors.New("ERR: Metadata file does not contain deletion key") - return "", err - } - - if err != nil { - return "", err - } else { - return metadata[1], err - } -} diff --git a/torrent.go b/torrent.go index 3f7405c..e3c4952 100644 --- a/torrent.go +++ b/torrent.go @@ -80,9 +80,13 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] filePath := path.Join(Config.filesDir, fileName) - if !fileExistsAndNotExpired(fileName) { + err := checkFile(fileName) + if err == NotFoundErr { notFoundHandler(c, w, r) return + } else if err == BadMetadata { + oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") + return } encoded, err := createTorrent(fileName, filePath) diff --git a/upload.go b/upload.go index 6ea9247..ef3c9a1 100644 --- a/upload.go +++ b/upload.go @@ -40,10 +40,8 @@ type UploadRequest struct { // Metadata associated with a file as it would actually be stored type Upload struct { - Filename string // Final filename on disk - Size int64 - Expiry time.Time // Unix timestamp of expiry, 0=never - DeleteKey string // Deletion key, one generated if not provided + Filename string // Final filename on disk + Metadata Metadata } func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -246,34 +244,35 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { defer dst.Close() // Get the rest of the metadata needed for storage + var expiry time.Time if upReq.expiry == 0 { - upload.Expiry = neverExpire + expiry = neverExpire } else { - upload.Expiry = time.Now().Add(upReq.expiry) + expiry = time.Now().Add(upReq.expiry) } - // If no delete key specified, pick a random one. - if upReq.deletionKey == "" { - upload.DeleteKey = uniuri.NewLen(30) - } else { - upload.DeleteKey = upReq.deletionKey - } - - metadataWrite(upload.Filename, &upload) - bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src)) if bytes == 0 { os.Remove(path.Join(Config.filesDir, upload.Filename)) - os.Remove(path.Join(Config.metaDir, upload.Filename)) return upload, errors.New("Empty file") } else if err != nil { os.Remove(path.Join(Config.filesDir, upload.Filename)) - os.Remove(path.Join(Config.metaDir, upload.Filename)) return } - upload.Size = bytes + upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey) + if err != nil { + os.Remove(path.Join(Config.filesDir, upload.Filename)) + os.Remove(path.Join(Config.metaDir, upload.Filename)) + return + } + err = metadataWrite(upload.Filename, &upload.Metadata) + if err != nil { + os.Remove(path.Join(Config.filesDir, upload.Filename)) + os.Remove(path.Join(Config.metaDir, upload.Filename)) + return + } return } @@ -285,9 +284,11 @@ func generateJSONresponse(upload Upload) []byte { js, _ := json.Marshal(map[string]string{ "url": Config.siteURL + upload.Filename, "filename": upload.Filename, - "delete_key": upload.DeleteKey, - "expiry": strconv.FormatInt(upload.Expiry.Unix(), 10), - "size": strconv.FormatInt(upload.Size, 10), + "delete_key": upload.Metadata.DeleteKey, + "expiry": strconv.FormatInt(upload.Metadata.Expiry.Unix(), 10), + "size": strconv.FormatInt(upload.Metadata.Size, 10), + "mimetype": upload.Metadata.Mimetype, + "sha256sum": upload.Metadata.Sha256sum, }) return js