mirror of https://github.com/rclone/rclone.git
NameCrane Mail backend
This commit is contained in:
namecrane
GitHub
parent
c837664653
commit
abfeb88ba2
|
|
@ -37,6 +37,7 @@ import (
|
|||
_ "github.com/rclone/rclone/backend/mailru"
|
||||
_ "github.com/rclone/rclone/backend/mega"
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
_ "github.com/rclone/rclone/backend/namecrane"
|
||||
_ "github.com/rclone/rclone/backend/netstorage"
|
||||
_ "github.com/rclone/rclone/backend/onedrive"
|
||||
_ "github.com/rclone/rclone/backend/opendrive"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
package namecrane
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AuthManager manages the authentication token.
|
||||
type AuthManager struct {
|
||||
mu sync.Mutex
|
||||
token string
|
||||
expiresAt time.Time
|
||||
client *http.Client
|
||||
apiURL string
|
||||
username string
|
||||
password string
|
||||
}
|
||||
|
||||
// NewAuthManager initializes the AuthManager.
|
||||
func NewAuthManager(client *http.Client, apiURL, username, password string) *AuthManager {
|
||||
return &AuthManager{
|
||||
client: client,
|
||||
apiURL: apiURL,
|
||||
username: username,
|
||||
password: password,
|
||||
}
|
||||
}
|
||||
|
||||
// Authenticate obtains a new token.
|
||||
func (am *AuthManager) Authenticate(ctx context.Context) error {
|
||||
am.mu.Lock()
|
||||
defer am.mu.Unlock()
|
||||
|
||||
// If the token is still valid, skip re-authentication
|
||||
if time.Now().Before(am.expiresAt) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Construct the API URL for authentication
|
||||
url := fmt.Sprintf("%s/api/v1/auth/authenticate-user", am.apiURL)
|
||||
|
||||
// Prepare the request body
|
||||
requestBody := map[string]string{
|
||||
"username": am.username,
|
||||
"password": am.password,
|
||||
}
|
||||
jsonBody, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal authentication body: %w", err)
|
||||
}
|
||||
|
||||
// Create the HTTP request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create authentication request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
// Execute the request
|
||||
resp, err := am.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("authentication request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("authentication failed, status: %d, response: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Parse the response
|
||||
var response struct {
|
||||
Token string `json:"accessToken"`
|
||||
ExpiresIn string `json:"accessTokenExpiration"` // Token expiration datetime
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
|
||||
return fmt.Errorf("failed to decode authentication response: %w", err)
|
||||
}
|
||||
|
||||
// Store the token and expiration time
|
||||
am.token = response.Token
|
||||
expiresAt, err := time.Parse(time.RFC3339, response.ExpiresIn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse token expiration time: %w", err)
|
||||
}
|
||||
am.expiresAt = expiresAt
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetToken ensures the token is valid and returns it.
|
||||
func (am *AuthManager) GetToken(ctx context.Context) (string, error) {
|
||||
if err := am.Authenticate(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return am.token, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,654 @@
|
|||
package namecrane
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/uuid"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"io"
|
||||
"math"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnknownType = errors.New("unknown content type")
|
||||
ErrUnexpectedStatus = errors.New("unexpected status")
|
||||
ErrNoFolder = errors.New("no folder found")
|
||||
ErrNoFile = errors.New("no file found")
|
||||
)
|
||||
|
||||
const (
|
||||
defaultFileType = "application/octet-stream"
|
||||
contextFileStorage = "file-storage"
|
||||
maxChunkSize = 15 * 1024 * 1024 // 15 MB
|
||||
|
||||
apiUpload = "api/upload"
|
||||
apiFiles = "api/v1/filestorage/files"
|
||||
apiFolder = "api/v1/filestorage/folder"
|
||||
apiFolders = "api/v1/filestorage/folders"
|
||||
apiPutFolder = "api/v1/filestorage/folder-put"
|
||||
apiDeleteFolder = "api/v1/filestorage/delete-folder"
|
||||
apiFileDownload = "api/v1/filestorage/%s/download"
|
||||
)
|
||||
|
||||
// Namecrane is the Namecrane API Client implementation
|
||||
type Namecrane struct {
|
||||
apiURL string
|
||||
authManager *AuthManager
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new Namecrane Client with the specified URL and auth manager
|
||||
func NewClient(apiURL string, authManager *AuthManager) *Namecrane {
|
||||
return &Namecrane{
|
||||
apiURL: apiURL,
|
||||
authManager: authManager,
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
// defaultResponse represents a default API response, containing Success and optionally Message
|
||||
type defaultResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Response wraps an *http.Response and provides extra functionality
|
||||
type Response struct {
|
||||
*http.Response
|
||||
}
|
||||
|
||||
// Data is a quick and dirty "read this data" for debugging
|
||||
func (r *Response) Data() []byte {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// Decode only supports JSON.
|
||||
// The API is weird and returns text/plain for JSON sometimes, but it's almost always JSON.
|
||||
func (r *Response) Decode(data any) error {
|
||||
// Close by default on decode
|
||||
defer r.Close()
|
||||
|
||||
return json.NewDecoder(r.Body).Decode(data)
|
||||
}
|
||||
|
||||
// Close is a redirect to r.Body.Close for shorthand
|
||||
func (r *Response) Close() error {
|
||||
return r.Body.Close()
|
||||
}
|
||||
|
||||
// File represents a file object on the remote server, identified by `ID`
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"fileName"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size"`
|
||||
DateAdded time.Time `json:"dateAdded"`
|
||||
FolderPath string `json:"folderPath"`
|
||||
}
|
||||
|
||||
// Folder represents a folder object on the remote server
|
||||
type Folder struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
Version string `json:"version"`
|
||||
Count int `json:"count"`
|
||||
Subfolders []Folder `json:"subfolders"`
|
||||
Files []File `json:"files"`
|
||||
}
|
||||
|
||||
// Flatten takes all folders and subfolders, returning them as a single slice
|
||||
func (f Folder) Flatten() []Folder {
|
||||
folders := []Folder{f}
|
||||
|
||||
for _, folder := range f.Subfolders {
|
||||
folders = append(folders, folder.Flatten()...)
|
||||
}
|
||||
|
||||
return folders
|
||||
}
|
||||
|
||||
func (n *Namecrane) String() string {
|
||||
return "Namecrane API (Endpoint: " + n.apiURL + ")"
|
||||
}
|
||||
|
||||
// uploadChunk uploads a chunk, then waits for it to be accepted.
|
||||
// When the last chunk is uploaded, the backend will combine the file, then return a 200 with a body.
|
||||
func (n *Namecrane) uploadChunk(ctx context.Context, reader io.Reader, fileName string, fileSize, chunkSize int64, fields map[string]string) (*Response, error) {
|
||||
// Send POST request to upload
|
||||
var requestBody bytes.Buffer
|
||||
writer := multipart.NewWriter(&requestBody)
|
||||
|
||||
for key, value := range fields {
|
||||
if err := writer.WriteField(key, value); err != nil {
|
||||
return nil, fmt.Errorf("failed to write field %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add the file content for this chunk
|
||||
part, err := writer.CreateFormFile("file", fileName)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create form file: %w", err)
|
||||
}
|
||||
|
||||
if _, err = io.CopyN(part, reader, chunkSize); err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("failed to copy chunk data: %w", err)
|
||||
}
|
||||
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// --- Send the chunk ---
|
||||
|
||||
resp, err := n.doRequest(ctx, http.MethodPost, apiUpload,
|
||||
requestBody.Bytes(),
|
||||
WithContentType(writer.FormDataContentType()))
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to upload file: %w", err)
|
||||
}
|
||||
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Upload will push a file to the Namecrane API
|
||||
func (n *Namecrane) Upload(ctx context.Context, in io.Reader, filePath string, fileSize int64) (*File, error) {
|
||||
fileName := path.Base(filePath)
|
||||
|
||||
// encode brackets, fixing bug within uploader
|
||||
// fileName = url.PathEscape(fileName)
|
||||
|
||||
basePath := path.Dir(filePath)
|
||||
|
||||
if basePath == "" || basePath[0] != '/' {
|
||||
basePath = "/" + basePath
|
||||
}
|
||||
|
||||
// Prepare context data
|
||||
contextBytes, err := json.Marshal(folderRequest{
|
||||
Folder: basePath,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
contextData := string(contextBytes)
|
||||
|
||||
// Calculate total chunks
|
||||
totalChunks := int(math.Ceil(float64(fileSize) / maxChunkSize))
|
||||
|
||||
remaining := fileSize
|
||||
|
||||
id, err := uuid.NewV7()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fields := map[string]string{
|
||||
"resumableChunkSize": strconv.FormatInt(maxChunkSize, 10),
|
||||
"resumableTotalSize": strconv.FormatInt(fileSize, 10),
|
||||
"resumableIdentifier": id.String(),
|
||||
"resumableType": defaultFileType,
|
||||
"resumableFilename": fileName,
|
||||
"resumableRelativePath": fileName,
|
||||
"resumableTotalChunks": strconv.Itoa(totalChunks),
|
||||
"context": contextFileStorage,
|
||||
"contextData": contextData,
|
||||
}
|
||||
|
||||
var res *Response
|
||||
|
||||
for chunk := 1; chunk <= totalChunks; chunk++ {
|
||||
chunkSize := int64(maxChunkSize)
|
||||
|
||||
if remaining < maxChunkSize {
|
||||
chunkSize = remaining
|
||||
}
|
||||
|
||||
// strconv.FormatInt is pretty much fmt.Sprintf but without needing to parse the format, replace things, etc.
|
||||
// base 10 is the default, see strconv.Itoa
|
||||
fields["resumableChunkNumber"] = strconv.Itoa(chunk)
|
||||
fields["resumableCurrentChunkSize"] = strconv.FormatInt(chunkSize, 10)
|
||||
|
||||
// --- Prepare the chunk payload ---
|
||||
res, err = n.uploadChunk(ctx, in, fileName, fileSize, chunkSize, fields)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chunk upload failed, error: %w", err)
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
var status defaultResponse
|
||||
|
||||
if err := res.Decode(&status); err != nil {
|
||||
return nil, fmt.Errorf("chunk %d upload failed, status: %d, response: %s", chunk, res.StatusCode, string(res.Data()))
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("chunk %d upload failed, status: %d, message: %s", chunk, res.StatusCode, status.Message)
|
||||
}
|
||||
|
||||
fs.Debugf(n, "Successfully uploaded chunk %d of %d of size %d/%d for file '%s'\n", chunk, totalChunks, chunkSize, remaining, fileName)
|
||||
|
||||
if chunk == totalChunks {
|
||||
var file File
|
||||
|
||||
if err := res.Decode(&file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &file, nil
|
||||
} else {
|
||||
_ = res.Close()
|
||||
}
|
||||
|
||||
// Update progress
|
||||
remaining -= chunkSize
|
||||
}
|
||||
|
||||
fs.Errorf(n, "Received no response from last upload chunk")
|
||||
|
||||
return nil, errors.New("no response from endpoint")
|
||||
}
|
||||
|
||||
type ListResponse struct {
|
||||
Files []File `json:"files"`
|
||||
}
|
||||
|
||||
type FolderResponse struct {
|
||||
defaultResponse
|
||||
Folder Folder `json:"folder"`
|
||||
}
|
||||
|
||||
// GetFolders returns all folders at the root level
|
||||
func (n *Namecrane) GetFolders(ctx context.Context) ([]Folder, error) {
|
||||
res, err := n.doRequest(ctx, http.MethodGet, apiFolders, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var response FolderResponse
|
||||
|
||||
if err := res.Decode(&response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Root folder is response.Folder
|
||||
return response.Folder.Flatten(), nil
|
||||
}
|
||||
|
||||
// GetFolder returns a single folder
|
||||
func (n *Namecrane) GetFolder(ctx context.Context, folder string) (*Folder, error) {
|
||||
res, err := n.doRequest(ctx, http.MethodPost, apiFolder, folderRequest{
|
||||
Folder: folder,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var folderResponse FolderResponse
|
||||
|
||||
if err := res.Decode(&folderResponse); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !folderResponse.Success {
|
||||
if folderResponse.Message == "Folder not found" {
|
||||
return nil, ErrNoFolder
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("received error from API: %s", folderResponse.Message)
|
||||
}
|
||||
|
||||
return &folderResponse.Folder, nil
|
||||
}
|
||||
|
||||
// filesRequest is a struct containing the appropriate fields for making a `GetFiles` request
|
||||
type filesRequest struct {
|
||||
FileIDs []string `json:"fileIds"`
|
||||
}
|
||||
|
||||
// GetFiles returns file data of the specified files
|
||||
func (n *Namecrane) GetFiles(ctx context.Context, ids ...string) ([]File, error) {
|
||||
res, err := n.doRequest(ctx, http.MethodPost, apiFiles, filesRequest{
|
||||
FileIDs: ids,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var response ListResponse
|
||||
|
||||
if err := res.Decode(&response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return response.Files, nil
|
||||
}
|
||||
|
||||
// DeleteFiles deletes the remote files specified by ids
|
||||
func (n *Namecrane) DeleteFiles(ctx context.Context, ids ...string) error {
|
||||
res, err := n.doRequest(ctx, http.MethodPost, apiFiles, filesRequest{
|
||||
FileIDs: ids,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var response defaultResponse
|
||||
|
||||
if err := res.Decode(&response); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFile opens the specified file as an io.ReadCloser, with optional `opts` (range header, etc)
|
||||
func (n *Namecrane) DownloadFile(ctx context.Context, id string, opts ...RequestOpt) (io.ReadCloser, error) {
|
||||
res, err := n.doRequest(ctx, http.MethodGet, fmt.Sprintf(apiFileDownload, id), nil, opts...)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// GetFileID gets a file id from a specified directory and file name
|
||||
func (n *Namecrane) GetFileID(ctx context.Context, dir, fileName string) (string, error) {
|
||||
var folder *Folder
|
||||
|
||||
if dir == "" || dir == "/" {
|
||||
folders, err := n.GetFolders(ctx)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
folder = &folders[0]
|
||||
} else {
|
||||
var err error
|
||||
|
||||
folder, err = n.GetFolder(ctx, dir)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
for _, file := range folder.Files {
|
||||
if file.Name == fileName {
|
||||
return file.ID, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", ErrNoFile
|
||||
}
|
||||
|
||||
// Find uses similar methods to GetFileID, but instead checks for both files AND folders
|
||||
func (n *Namecrane) Find(ctx context.Context, file string) (*Folder, *File, error) {
|
||||
base, name := n.parsePath(file)
|
||||
|
||||
var folder *Folder
|
||||
|
||||
if base == "" || base == "/" {
|
||||
folders, err := n.GetFolders(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
folder = &folders[0]
|
||||
} else {
|
||||
var err error
|
||||
|
||||
folder, err = n.GetFolder(ctx, base)
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, file := range folder.Files {
|
||||
if file.Name == name {
|
||||
return nil, &file, nil
|
||||
}
|
||||
}
|
||||
|
||||
for _, folder := range folder.Subfolders {
|
||||
if folder.Name == name {
|
||||
return &folder, nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, ErrNoFile
|
||||
}
|
||||
|
||||
// folderRequest is used for creating and deleting folders
|
||||
type folderRequest struct {
|
||||
ParentFolder string `json:"parentFolder,omitempty"`
|
||||
Folder string `json:"folder"`
|
||||
}
|
||||
|
||||
// CreateFolder creates a new remote folder
|
||||
func (n *Namecrane) CreateFolder(ctx context.Context, folder string) (*Folder, error) {
|
||||
parent, subfolder := n.parsePath(folder)
|
||||
|
||||
res, err := n.doRequest(ctx, http.MethodPost, apiPutFolder, folderRequest{
|
||||
ParentFolder: parent,
|
||||
Folder: subfolder,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var response FolderResponse
|
||||
|
||||
if err := res.Decode(&response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !response.Success {
|
||||
return nil, fmt.Errorf("failed to create directory, status: %d, response: %s", res.StatusCode, response.Message)
|
||||
}
|
||||
|
||||
return &response.Folder, nil
|
||||
}
|
||||
|
||||
// DeleteFolder deletes a specified folder by name
|
||||
func (n *Namecrane) DeleteFolder(ctx context.Context, folder string) error {
|
||||
parent, subfolder := n.parsePath(folder)
|
||||
|
||||
res, err := n.doRequest(ctx, http.MethodPost, apiDeleteFolder, folderRequest{
|
||||
ParentFolder: parent,
|
||||
Folder: subfolder,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%w: %d", ErrUnexpectedStatus, res.StatusCode)
|
||||
}
|
||||
|
||||
var status defaultResponse
|
||||
|
||||
if err := res.Decode(&status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !status.Success {
|
||||
return fmt.Errorf("failed to remove directory, status: %d, response: %s", res.StatusCode, status.Message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiUrl joins the base API URL with the path specified
|
||||
func (n *Namecrane) apiUrl(subPath string) (string, error) {
|
||||
u, err := url.Parse(n.apiURL)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
u.Path = path.Join(u.Path, subPath)
|
||||
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// RequestOpt is a quick helper for changing request options
|
||||
type RequestOpt func(r *http.Request)
|
||||
|
||||
// WithContentType overrides specified content types
|
||||
func WithContentType(contentType string) RequestOpt {
|
||||
return func(r *http.Request) {
|
||||
r.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
}
|
||||
|
||||
// WithHeader sets header values on the request
|
||||
func WithHeader(key, value string) RequestOpt {
|
||||
return func(r *http.Request) {
|
||||
r.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Namecrane) doRequest(ctx context.Context, method, path string, body any, opts ...RequestOpt) (*Response, error) {
|
||||
token, err := n.authManager.GetToken(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to retrieve token: %w", err)
|
||||
}
|
||||
|
||||
apiUrl, err := n.apiUrl(path)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
var jsonBody bool
|
||||
|
||||
if body != nil {
|
||||
switch method {
|
||||
case http.MethodPost:
|
||||
switch v := body.(type) {
|
||||
case io.Reader:
|
||||
bodyReader = v
|
||||
case []byte:
|
||||
bodyReader = bytes.NewReader(v)
|
||||
case string:
|
||||
bodyReader = strings.NewReader(v)
|
||||
default:
|
||||
b, err := json.Marshal(body)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bodyReader = bytes.NewReader(b)
|
||||
|
||||
jsonBody = true
|
||||
}
|
||||
case http.MethodGet:
|
||||
switch v := body.(type) {
|
||||
case *url.Values:
|
||||
apiUrl += "?" + v.Encode()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create the HTTP request
|
||||
req, err := http.NewRequestWithContext(ctx, method, apiUrl, bodyReader)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create rmdir request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
if jsonBody {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
// Apply extra options like overriding content types
|
||||
for _, opt := range opts {
|
||||
opt(req)
|
||||
}
|
||||
|
||||
// Execute the HTTP request
|
||||
resp, err := n.client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute rmdir request: %w", err)
|
||||
}
|
||||
|
||||
return &Response{
|
||||
Response: resp,
|
||||
}, err
|
||||
}
|
||||
|
||||
// parsePath parses the last segment off the specified path, representing either a file or directory
|
||||
func (n *Namecrane) parsePath(path string) (basePath, lastSegment string) {
|
||||
trimmedPath := strings.Trim(path, "/")
|
||||
|
||||
segments := strings.Split(trimmedPath, "/")
|
||||
|
||||
if len(segments) > 1 {
|
||||
basePath = "/" + strings.Join(segments[:len(segments)-1], "/")
|
||||
|
||||
lastSegment = segments[len(segments)-1]
|
||||
} else {
|
||||
basePath = "/"
|
||||
lastSegment = segments[0]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package namecrane
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClient(t *testing.T) {
|
||||
fsDirPath := "../../Dockerfile"
|
||||
|
||||
fd, err := os.Open(fsDirPath)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to open directory %q: %w", fsDirPath, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for {
|
||||
var fis []os.FileInfo
|
||||
// Windows and Plan9 read the directory entries with the stat information in which
|
||||
// shouldn't fail because of unreadable entries.
|
||||
fis, err = fd.Readdir(1024)
|
||||
if err == io.EOF && len(fis) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Unable to read directory", err)
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
t.Log("Found", fi.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,492 @@
|
|||
package namecrane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
/**
|
||||
* Namecrane Storage Backend
|
||||
* Copyright (c) 2025 Namecrane LLC
|
||||
* PSA: No cranes harmed in the development of this module.
|
||||
*/
|
||||
|
||||
var (
|
||||
ErrEmptyDirectory = errors.New("directory name cannot be empty")
|
||||
)
|
||||
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
features *fs.Features
|
||||
client *Namecrane
|
||||
apiURL string
|
||||
authManager *AuthManager
|
||||
}
|
||||
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
file *File
|
||||
folder *Folder
|
||||
remote string
|
||||
}
|
||||
|
||||
type Directory struct {
|
||||
*Object
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
ApiURL string `config:"api_url"`
|
||||
Username string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "namecrane",
|
||||
Description: "Namecrane File Storage API Backend",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "api_url",
|
||||
Help: `Namecrane API URL, like https://us1.workspace.org`,
|
||||
Default: "https://us1.workspace.org",
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "username",
|
||||
Help: `Namecrane username`,
|
||||
Required: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: `Namecrane password`,
|
||||
Required: true,
|
||||
IsPassword: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
|
||||
if err := configstruct.Set(m, opt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pass, err := obscure.Reveal(opt.Password)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("NewFS decrypt password: %w", err)
|
||||
}
|
||||
|
||||
opt.Password = pass
|
||||
|
||||
if root == "" || root == "." {
|
||||
root = "/"
|
||||
}
|
||||
|
||||
authManager := NewAuthManager(http.DefaultClient, opt.ApiURL, opt.Username, opt.Password)
|
||||
|
||||
if _, err := authManager.GetToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(opt.ApiURL, authManager)
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
client: client,
|
||||
apiURL: opt.ApiURL,
|
||||
authManager: authManager,
|
||||
}
|
||||
|
||||
// Validate that the root is a directory, not a file
|
||||
_, file, err := client.Find(ctx, root)
|
||||
|
||||
// Ignore ErrNoFile as rclone will create directories for us
|
||||
if err != nil && !errors.Is(err, ErrNoFile) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
// Path is a file, not a folder. Set the root to the folder and return a special error.
|
||||
f.root = file.FolderPath
|
||||
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
func (f *Fs) String() string {
|
||||
if f.root == "" {
|
||||
return fmt.Sprintf("NameCrane backend at %s", f.apiURL)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("NameCrane backend at %s, root '%s'", f.apiURL, f.root)
|
||||
}
|
||||
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
if f.features == nil {
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(context.Background(), f)
|
||||
}
|
||||
return f.features
|
||||
}
|
||||
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
normalizedDir := path.Join(f.root, path.Clean(dir))
|
||||
|
||||
if normalizedDir == "" {
|
||||
return ErrEmptyDirectory
|
||||
}
|
||||
|
||||
// Normalize the directory path
|
||||
err := f.client.DeleteFolder(ctx, normalizedDir)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Successfully removed directory '%s'", normalizedDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
normalizedDir := path.Join(f.root, path.Clean(dir))
|
||||
|
||||
if normalizedDir == "" {
|
||||
return ErrEmptyDirectory
|
||||
}
|
||||
|
||||
res, err := f.client.CreateFolder(ctx, normalizedDir)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Successfully created directory '%s'", res.Path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) Stat(ctx context.Context, remote string) (fs.DirEntry, error) {
|
||||
// Fetch the folder path and file name from the remote path
|
||||
dir, fileName := path.Split(remote)
|
||||
|
||||
// Prepend root path
|
||||
dir = path.Join(f.root, dir)
|
||||
|
||||
if dir == "" || dir[0] != '/' {
|
||||
dir = "/" + dir
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Stat file at %s: %s -> %s", remote, dir, fileName)
|
||||
|
||||
id, err := f.client.GetFileID(ctx, dir, fileName)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files, err := f.client.GetFiles(ctx, id)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file := files[0]
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
file: &file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// Return the hash types supported by the backend.
|
||||
// If no hashing is supported, return hash.None.
|
||||
return hash.NewHashSet()
|
||||
}
|
||||
|
||||
func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
remote := path.Join(f.root, dir)
|
||||
|
||||
if remote == "" || remote[0] != '/' {
|
||||
remote = "/" + remote
|
||||
}
|
||||
|
||||
fs.Debugf(f, "List contents of %s: %s", dir, remote)
|
||||
|
||||
// If the path is a subdirectory, use GetFolder instead of GetFolders
|
||||
if remote != "/" {
|
||||
fs.Debugf(f, "Listing files in non-root directory %s", remote)
|
||||
|
||||
folder, err := f.client.GetFolder(ctx, remote)
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNoFolder) {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
fs.Errorf(f, "Unable to find directory %s", remote)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.folderToEntries(*folder), nil
|
||||
}
|
||||
|
||||
root, err := f.client.GetFolders(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// root[0] is always the root folder
|
||||
return f.folderToEntries(root[0]), nil
|
||||
}
|
||||
|
||||
func (f *Fs) folderToEntries(folder Folder) fs.DirEntries {
|
||||
var entries fs.DirEntries
|
||||
|
||||
for _, file := range folder.Files {
|
||||
entries = append(entries, &Object{
|
||||
fs: f,
|
||||
file: &file,
|
||||
})
|
||||
}
|
||||
|
||||
for _, subfolder := range folder.Subfolders {
|
||||
entries = append(entries, &Directory{
|
||||
Object: &Object{
|
||||
fs: f,
|
||||
folder: &subfolder,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
func (f *Fs) Sortable() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
fs.Debugf(f, "New object %s", remote)
|
||||
|
||||
folder, file, err := f.client.Find(ctx, remote)
|
||||
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Unable to find existing file, not necessarily a bad thing: %s", err.Error())
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
folder: folder,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
remote = path.Join(f.root, remote)
|
||||
|
||||
if remote[0] != '/' {
|
||||
remote = "/" + remote
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Put contents of %s to %s", src.Remote(), remote)
|
||||
|
||||
file, err := f.client.Upload(ctx, in, remote, src.Size())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return the uploaded object
|
||||
return &Object{
|
||||
fs: f,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file != nil {
|
||||
return o.file.DateAdded
|
||||
}
|
||||
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
func (o *Object) String() string {
|
||||
if o.file != nil {
|
||||
return strings.TrimRight(o.file.FolderPath, "/") + "/" + o.file.Name
|
||||
} else if o.folder != nil {
|
||||
return o.folder.Path
|
||||
}
|
||||
|
||||
return o.remote
|
||||
}
|
||||
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Return the time precision supported by the backend.
|
||||
// Use fs.ModTimeNotSupported if modification times are not supported.
|
||||
return fs.ModTimeNotSupported
|
||||
}
|
||||
|
||||
// Remote joins the path with the fs root
|
||||
func (o *Object) Remote() string {
|
||||
// Ensure paths are normalized and relative
|
||||
remotePath := path.Clean(o.String())
|
||||
rootPath := path.Clean(o.fs.root)
|
||||
|
||||
// Strip the root path from the remote if necessary
|
||||
remotePath = strings.TrimPrefix(remotePath, rootPath)
|
||||
|
||||
// Return the relative path
|
||||
return strings.TrimLeft(remotePath, "/")
|
||||
}
|
||||
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Size returns the size of the object in bytes.
|
||||
func (o *Object) Size() int64 {
|
||||
if o.file != nil {
|
||||
return o.file.Size
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// Stat will ensure that either folder or file is populated, then return the object to use as ObjectInfo
|
||||
func (o *Object) Stat(ctx context.Context) (fs.ObjectInfo, error) {
|
||||
if o.file != nil || o.folder != nil {
|
||||
return o, nil
|
||||
}
|
||||
|
||||
fs.Debugf(o.fs, "Stat object %s", o.remote)
|
||||
|
||||
folder, file, err := o.fs.client.Find(ctx, o.remote)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Since one of these will be nil, we're fine setting both without an if check
|
||||
o.folder = folder
|
||||
o.file = file
|
||||
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Open will open the file for reading
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
if o.file == nil {
|
||||
// Populate file from path
|
||||
_, file, err := o.fs.client.Find(ctx, o.remote)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if file == nil {
|
||||
return nil, fs.ErrorIsDir
|
||||
}
|
||||
|
||||
o.file = file
|
||||
}
|
||||
|
||||
// Support ranges (maybe, not sure if the API supports this?)
|
||||
opts := make([]RequestOpt, 0)
|
||||
|
||||
for _, opt := range options {
|
||||
key, value := opt.Header()
|
||||
|
||||
if key != "" && value != "" {
|
||||
opts = append(opts, WithHeader(key, value))
|
||||
}
|
||||
}
|
||||
|
||||
return o.fs.client.DownloadFile(ctx, o.file.ID, opts...)
|
||||
}
|
||||
|
||||
// Update pushes a file up to the backend
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
obj, err := o.fs.Put(ctx, in, src, options...)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
o.file = obj.(*Object).file
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove deletes the file represented by the object from the remote.
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.client.DeleteFiles(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// Items returns the count of items in this directory or this
|
||||
// directory and subdirectories if known, -1 for unknown
|
||||
func (d *Directory) Items() int64 {
|
||||
return int64(len(d.folder.Files))
|
||||
}
|
||||
|
||||
// ID returns the internal ID of this directory if known, or
|
||||
// "" otherwise
|
||||
func (d *Directory) ID() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Hash does nothing on a directory
|
||||
//
|
||||
// This method is implemented with the incorrect type signature to
|
||||
// stop the Directory type asserting to fs.Object or fs.ObjectInfo
|
||||
func (d *Directory) Hash() {
|
||||
// Does nothing
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.Directory = &Directory{}
|
||||
_ fs.SetModTimer = &Directory{}
|
||||
)
|
||||
Loading…
Reference in New Issue