diff --git a/backend/all/all.go b/backend/all/all.go index f097cb261..7a08f9583 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -58,5 +58,6 @@ import ( _ "github.com/rclone/rclone/backend/uptobox" _ "github.com/rclone/rclone/backend/webdav" _ "github.com/rclone/rclone/backend/yandex" + _ "github.com/rclone/rclone/backend/youtubemusic" _ "github.com/rclone/rclone/backend/zoho" ) diff --git a/backend/youtubemusic/api/types.go b/backend/youtubemusic/api/types.go new file mode 100644 index 000000000..f26f75ac6 --- /dev/null +++ b/backend/youtubemusic/api/types.go @@ -0,0 +1,37 @@ +// Package api provides types used by the YouTube Music API. +package api + +// OAuthTVAndLimitedDeviceRequest represents the JSON API object that's sent to the oauth API endpoint. +type OAuthTVAndLimitedDeviceRequest struct { + Scope string `json:"scope"` + ClientID string `json:"client_id"` +} + +// OAuthTVAndLimitedDeviceResponse represents the JSON API object that's received from the oauth API endpoint. +type OAuthTVAndLimitedDeviceResponse struct { + DeviceCode string `json:"device_code"` + UserCode string `json:"user_code"` + ExpiresIn int `json:"expires_in"` + Interval int `json:"interval"` + VerificationURL string `json:"verification_url"` + Error string `json:"error"` +} + +// OAuthTokenTVAndLimitedDeviceRequest represents the JSON API object that's sent to the oauth token API endpoint. +type OAuthTokenTVAndLimitedDeviceRequest struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + GrantType string `json:"grant_type"` + Code string `json:"code"` +} + +// OAuthTokenTVAndLimitedDeviceResponse represents the JSON API object that's received from the oauth token API endpoint. +type OAuthTokenTVAndLimitedDeviceResponse struct { + Scope string `json:"scope"` + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} diff --git a/backend/youtubemusic/fs.go b/backend/youtubemusic/fs.go new file mode 100644 index 000000000..ae22dbd36 --- /dev/null +++ b/backend/youtubemusic/fs.go @@ -0,0 +1,121 @@ +package youtubemusic + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/oauthutil" + "github.com/rclone/rclone/lib/rest" +) + +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed options + features *fs.Features // optional features + unAuth *rest.Client // unauthenticated http client + srv *rest.Client // the connection to the server + ts *oauthutil.TokenSource // token source for oauth2 + pacer *fs.Pacer // To pace the API calls + startTime time.Time // time Fs was started - used for datestamps +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("YouTube Music path '%q'", f.root) +} + +// Precision returns the precision +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + // TODO: + return nil, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + // TODO: + return nil, nil +} + +// Put the object into the bucket +// +// Copy the reader in to the new object which is returned. +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // TODO: + return nil, nil +} + +// Mkdir creates the album if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + // TODO: + return nil +} + +// Rmdir deletes the bucket if the fs is at the root +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + // TODO: + return nil +} + +// NewFs constructs an Fs from the path. +// +// The returned Fs implements fs.Fs interface. +func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + fsys := &Fs{} + + return fsys, nil +} + +// Check the interfaces are satisfied. +var _ fs.Fs = &Fs{} diff --git a/backend/youtubemusic/object.go b/backend/youtubemusic/object.go new file mode 100644 index 000000000..bba35c92f --- /dev/null +++ b/backend/youtubemusic/object.go @@ -0,0 +1,142 @@ +package youtubemusic + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/log" + "github.com/rclone/rclone/lib/rest" +) + +// Object describes a youtubemusic object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + url string // download path + id string // ID of this object + bytes int64 // Bytes in the object + modTime time.Time // Modified time of the object + mimeType string +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime(ctx context.Context) time.Time { + defer log.Trace(o, "")("") + err := o.readMetaData(ctx) + if err != nil { + fs.Debugf(o, "ModTime: Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + // TODO: + return 0 +} + +// Hash returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", hash.ErrUnsupported + +} + +// Storable returns a boolean as to whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, t time.Time) error { + return fs.ErrorCantSetModTime +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + defer log.Trace(o, "")("") + err = o.readMetaData(ctx) + if err != nil { + fs.Debugf(o, "Open: Failed to read metadata: %v", err) + return nil, err + } + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: o.downloadURL(), + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + // TODO: + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + // TODO: + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData(ctx context.Context) (err error) { + // TODO: + return nil +} + +// setMetaData sets the fs data from a storage.Object +func (o *Object) setMetaData() { + // TODO: +} + +// downloadURL returns the URL for a full bytes download for the object +func (o *Object) downloadURL() (url string) { + // TODO: + // url := o.url + "=d" + // if strings.HasPrefix(o.mimeType, "video/") { + // url += "v" + // } + return url +} + +// Check the interfaces are satisfied +var _ fs.Object = &Object{} diff --git a/backend/youtubemusic/youtubemusic.go b/backend/youtubemusic/youtubemusic.go new file mode 100644 index 000000000..2d68aefba --- /dev/null +++ b/backend/youtubemusic/youtubemusic.go @@ -0,0 +1,248 @@ +// Package youtubemusic provides the youtubemusic backend. +package youtubemusic + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/rclone/rclone/backend/youtubemusic/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/lib/oauthutil" + "github.com/rclone/rclone/lib/rest" + "github.com/skratchdot/open-golang/open" + "golang.org/x/oauth2" +) + +const ( + // The app for these API keys should be for TV and Limited-Input Device. + rcloneClientID = "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com" // TODO: update this + rcloneEncryptedClientSecret = "aCVz_k_XoJc9gc3XuuDCeq2VzXsQFd4QTKsyF8ir2Bgnj5abr28JQw" // TODO: update this + + // These 2 scopes are the only YouTube Data API scopes available for TV and Limited-Input Device. + // source: https://developers.google.com/youtube/v3/guides/auth/devices#allowedscopes + scopeYoutubeReadWrite = "https://www.googleapis.com/auth/youtube" // manage your YouTube account + scopeYoutubeReadOnly = "https://www.googleapis.com/auth/youtube.readonly" // view your YouTube account + + ytmusicDomain = "https://music.youtube.com" + ytmusicBaseAPI = ytmusicDomain + "/youtubei/v1/" + ytmusicParams = "?alt=json" + ytmusicParamsKey = "&key=AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30" + userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0" + + oauthCodeURL = "https://www.youtube.com/o/oauth2/device/code" + oauthTokenURL = "https://oauth2.googleapis.com/token" + oauthUserAgent = userAgent + " Cobalt/Version" +) + +var ( + oauthScope string + deviceCode string +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "youtube music", + Prefix: "ytmusic", + Description: "YouTube Music", + NewFs: NewFs, + Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, fmt.Errorf("couldn't parse config into struct: %w", err) + } + + switch config.State { + case "": + // Fill in the scopes + if opt.ReadOnly { + oauthScope = scopeYoutubeReadOnly + } else { + oauthScope = scopeYoutubeReadWrite + } + + // Update client_id and client_secret if they are empty + if val, _ := m.Get("client_id"); val == "" { + m.Set("client_id", rcloneClientID) + } + if val, _ := m.Get("client_secret"); val == "" { + m.Set("client_secret", obscure.MustReveal(rcloneEncryptedClientSecret)) + } + + // Post the response from oauthCodeURL + clientID, _ := m.Get("client_id") + oauthTVAndLimitedDevice, err := postOAuthTVAndLimitedDevice(ctx, api.OAuthTVAndLimitedDeviceRequest{ + Scope: oauthScope, + ClientID: clientID, + }) + if err != nil { + return nil, fmt.Errorf("failed to postOAuthCodeURL: %w", err) + } + deviceCode = oauthTVAndLimitedDevice.DeviceCode + + // Open the verification URL in the browser + url := fmt.Sprintf("%s?user_code=%s", oauthTVAndLimitedDevice.VerificationURL, oauthTVAndLimitedDevice.UserCode) + open.Start(url) + + return fs.ConfigConfirm("config_auth_do", true, "config_init", fmt.Sprintf("Go to %s, finish the login flow and press Enter when done, Ctrl-C to abort", url)) + case "config_auth_do": // Continue the authentication process + // Post the response from oauthTokenURL + clientID, _ := m.Get("client_id") + clientSecret, _ := m.Get("client_secret") + oauthTokenTVAndLimitedDevice, err := postOAuthTokenTVAndLimitedDevice(ctx, api.OAuthTokenTVAndLimitedDeviceRequest{ + ClientID: clientID, + ClientSecret: clientSecret, + GrantType: "http://oauth.net/grant_type/device/1.0", + Code: deviceCode, + }) + if err != nil { + return nil, fmt.Errorf("failed to postOAuthTokenURL: %w", err) + } + + // Save the token to the config file + err = oauthutil.PutToken(name, m, &oauth2.Token{ + AccessToken: oauthTokenTVAndLimitedDevice.AccessToken, + RefreshToken: oauthTokenTVAndLimitedDevice.RefreshToken, + TokenType: oauthTokenTVAndLimitedDevice.TokenType, + Expiry: time.Now().Add(time.Duration(oauthTokenTVAndLimitedDevice.ExpiresIn) * time.Second), + }, true) + if err != nil { + return nil, fmt.Errorf("error while saving token: %w", err) + } + + return nil, nil + } + return nil, fmt.Errorf("unknown state %q", config.State) + }, + Options: append(oauthutil.SharedOptions, []fs.Option{{ + Name: "read_only", + Default: false, + Help: `Set to make the YouTube Music backend read only. + +If you choose read only then rclone will only request read only access +to your music, otherwise rclone will request full access.`, + }, { + Name: "playlist-id", + Help: "ID of the playlist to sync. Can be found in the playlist's URL.", + }}...), + }) +} + +// Options defines the configuration for this backend +type Options struct { + ReadOnly bool `config:"read_only"` + + // ReadSize bool `config:"read_size"` + // StartYear int `config:"start_year"` + // IncludeArchived bool `config:"include_archived"` + // Enc encoder.MultiEncoder `config:"encoding"` + // BatchMode string `config:"batch_mode"` + // BatchSize int `config:"batch_size"` + // BatchTimeout fs.Duration `config:"batch_timeout"` +} + +// ------------------------------------------------------------ + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// postOAuthTVAndLimitedDevice sends a POST request to the oauthCodeURL and returns the response. +func postOAuthTVAndLimitedDevice(ctx context.Context, reqData api.OAuthTVAndLimitedDeviceRequest) (resp api.OAuthTVAndLimitedDeviceResponse, err error) { + // Create a new HTTP client with the provided context. + httpClient := fshttp.NewClient(ctx) + + // Marshal the request data into JSON. + reqStr, err := json.Marshal(reqData) + if err != nil { + return resp, fmt.Errorf("failed to marshal reqData: %w", err) + } + + // Send a POST request to the oauthCodeURL with the request data. + respHTTP, err := httpClient.Post(oauthCodeURL, "application/json", bytes.NewBuffer(reqStr)) + if err != nil { + return resp, fmt.Errorf("failed to get oauth code: %w", err) + } + defer fs.CheckClose(respHTTP.Body, &err) + + // Read the response body. + respBody, err := rest.ReadBody(respHTTP) + if err != nil { + return resp, fmt.Errorf("failed to read respBody: %w", err) + } + + // Unmarshal the response body into a struct. + response := new(api.OAuthTVAndLimitedDeviceResponse) + err = json.Unmarshal(respBody, &response) + if err != nil { + return resp, fmt.Errorf("failed to parse respBody: %w", err) + } + if response.Error != "" { + return resp, fmt.Errorf("failed to get oauth code: %s", respBody) + } + + return *response, nil +} + +// postOAuthTokenTVAndLimitedDevice sends a POST request to the oauthTokenURL and returns the response. +func postOAuthTokenTVAndLimitedDevice(ctx context.Context, reqData api.OAuthTokenTVAndLimitedDeviceRequest) (resp api.OAuthTokenTVAndLimitedDeviceResponse, err error) { + // Create a new HTTP client with the provided context. + httpClient := fshttp.NewClient(ctx) + + // Marshal the request data into JSON. + reqStr, err := json.Marshal(reqData) + if err != nil { + return resp, fmt.Errorf("failed to marshal reqData: %w", err) + } + + // Send a POST request to the oauthTokenURL with the request data. + respHTTP, err := httpClient.Post(oauthTokenURL, "application/json", bytes.NewBuffer(reqStr)) + if err != nil { + return resp, fmt.Errorf("failed to get oauth token: %w", err) + } + defer fs.CheckClose(respHTTP.Body, &err) + + // Read the response body. + respBody, err := rest.ReadBody(respHTTP) + if err != nil { + return resp, fmt.Errorf("failed to read respBody: %w", err) + } + + // Unmarshal the response body into a struct. + response := new(api.OAuthTokenTVAndLimitedDeviceResponse) + err = json.Unmarshal(respBody, &response) + if err != nil { + return resp, fmt.Errorf("failed to parse respBody: %w", err) + } + if response.Error != "" { + return resp, fmt.Errorf("failed to get oauth token: %s", respBody) + } + + return *response, nil +}