yandex: fixes the speed cap of Yandex.Disk

oauthutil: allows for a device-type authorization flow (used when no browser available)

backend/yandex/yandex.go

@@ -26,6 +26,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/oauth2"
@@ -46,12 +47,26 @@ var (
 	// Description of how to auth for this app
 	oauthConfig = &oauth2.Config{
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
-			TokenURL: "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
+			AuthURL:       "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
+			TokenURL:      "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
+			DeviceAuthURL: "https://oauth.yandex.com/device/code",
 		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
+		Scopes: []string{
+			"cloud_api:disk.app_folder",
+			"cloud_api:disk.read",
+			"cloud_api:disk.write",
+			"cloud_api:disk.info",
+			/*"yadisk:all",
+			"cloud_api.data:app_data",
+			"cloud_api.data:user_data",*/
+			/*"messenger:telemost",
+			"telemost:all",
+			"passport:bind_phone",
+			"calendar:all",*/
+		},
 	}
 )
 
@@ -63,7 +78,8 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
 			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
+				OAuth2Config:    oauthConfig,
+				DeviceGrantType: "device_code",
 			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
@@ -79,6 +95,14 @@ func init() {
 			// it doesn't seem worth making an exception for this
 			Default: (encoder.Display |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:     "user_agent",
+			Default:  "",
+			Advanced: true,
+			Hide:     fs.OptionHideCommandLine,
+			Help: `HTTP user agent used internally by client.
+
+Defaults to a dynamically generated ID used by the official client or "--user-agent" provided on command line.`,
 		}}...),
 	})
 }
@@ -88,6 +112,7 @@ type Options struct {
 	Token      string               `config:"token"`
 	HardDelete bool                 `config:"hard_delete"`
 	Enc        encoder.MultiEncoder `config:"encoding"`
+	UserAgent  string               `config:"user_agent"`
 }
 
 // Fs represents a remote yandex
@@ -254,6 +279,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
+	newCtx, ci := fs.AddConfig(ctx)
+
+	randomSessionId, _ := random.Password(128)
+
+	if opt.UserAgent == "" {
+		opt.UserAgent = `Yandex.Disk {"os":"windows","dtype":"ydisk3","vsn":"3.2.37.4977","id":"6BD01244C7A94456BBCEE7EEC990AEAD","id2":"0F370CD40C594A4783BC839C846B999C","session_id":"` +
+			randomSessionId + `"}`
+	}
+
+	globalConfig := fs.GetConfig(nil)
+
+	if ci.UserAgent == globalConfig.UserAgent {
+		ci.UserAgent = opt.UserAgent
+	}
+
 	token, err := oauthutil.GetToken(name, m)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't read OAuth token: %w", err)
@@ -269,12 +309,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 		log.Printf("Automatically upgraded OAuth config.")
 	}
-	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
+	oAuthClient, _, err := oauthutil.NewClient(newCtx, name, m, oauthConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure Yandex: %w", err)
 	}
 
-	ci := fs.GetConfig(ctx)
 	f := &Fs{
 		name:  name,
 		opt:   *opt,

fs/config/config.go

@@ -47,6 +47,9 @@ const (
 	// ConfigTokenURL is the config key used to store the token server endpoint
 	ConfigTokenURL = "token_url"
 
+	// ConfigDeviceURL is the config key used to store the device flow server endpoint
+	ConfigDeviceURL = "device_url"
+
 	// ConfigEncoding is the config key to change the encoding for a backend
 	ConfigEncoding = "encoding"
 

lib/oauthutil/oauthutil.go

@@ -102,6 +102,10 @@ var SharedOptions = []fs.Option{{
 	Name:     config.ConfigTokenURL,
 	Help:     "Token server url.\n\nLeave blank to use the provider defaults.",
 	Advanced: true,
+}, {
+	Name:     config.ConfigDeviceURL,
+	Help:     "Device grant server url.\n\nLeave blank to use the provider defaults.",
+	Advanced: true,
 }}
 
 // oldToken contains an end-user's tokens.
@@ -393,6 +397,11 @@ func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Con
 		newConfig.Endpoint.TokenURL = TokenURL
 		changed = true
 	}
+	DeviceURL, ok := m.Get(config.ConfigDeviceURL)
+	if ok && DeviceURL != "" {
+		newConfig.Endpoint.DeviceAuthURL = DeviceURL
+		changed = true
+	}
 	return newConfig, changed
 }
 
@@ -456,11 +465,12 @@ type CheckAuthFn func(*oauth2.Config, *AuthResult) error
 
 // Options for the oauth config
 type Options struct {
-	OAuth2Config *oauth2.Config          // Basic config for oauth2
-	NoOffline    bool                    // If set then "access_type=offline" parameter is not passed
-	CheckAuth    CheckAuthFn             // When the AuthResult is known the checkAuth function is called if set
-	OAuth2Opts   []oauth2.AuthCodeOption // extra oauth2 options
-	StateBlankOK bool                    // If set, state returned as "" is deemed to be OK
+	OAuth2Config    *oauth2.Config          // Basic config for oauth2
+	NoOffline       bool                    // If set then "access_type=offline" parameter is not passed
+	CheckAuth       CheckAuthFn             // When the AuthResult is known the checkAuth function is called if set
+	OAuth2Opts      []oauth2.AuthCodeOption // extra oauth2 options
+	StateBlankOK    bool                    // If set, state returned as "" is deemed to be OK
+	DeviceGrantType string                  // If set, changes the default (rfc) grant type to the one specified
 }
 
 // ConfigOut returns a config item suitable for the backend config
@@ -535,6 +545,16 @@ func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.Re
 		if err != nil {
 			return nil, err
 		}
+		if deviceCodeAuthPossible(opt.OAuth2Config) {
+			token, err := getDeviceCodeAuthURL(name, m, opt.OAuth2Config, opt)
+
+			if err != nil {
+				return nil, err
+			}
+
+			PutToken(name, m, token, false)
+			return fs.ConfigGoto(newState("*oauth-done"))
+		}
 		if noWebserverNeeded(opt.OAuth2Config) {
 			authURL, _, err := getAuthURL(name, m, opt.OAuth2Config, opt)
 			if err != nil {
@@ -647,11 +667,101 @@ func init() {
 	fs.ConfigOAuth = ConfigOAuth
 }
 
+func deviceCodeAuthPossible(oauthConfig *oauth2.Config) bool {
+	return oauthConfig.Endpoint.DeviceAuthURL != ""
+}
+
 // Return true if can run without a webserver and just entering a code
 func noWebserverNeeded(oauthConfig *oauth2.Config) bool {
 	return oauthConfig.RedirectURL == TitleBarRedirectURL
 }
 
+func getDeviceCodeAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (token *oauth2.Token, err error) {
+	oauthConfig, _ = overrideCredentials(name, m, oauthConfig)
+
+	ctx := context.Background()
+
+	deviceCode, err := oauthConfig.DeviceAuth(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Printf("Visit: %v and enter: %v\n", deviceCode.VerificationURI, deviceCode.UserCode)
+
+	token, err = waitForDeviceAuthorization(oauthConfig, opt, deviceCode)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return token, nil
+}
+
+// A tokenOrError is either an OAuth2 Token response or an error indicating why
+// such a response failed.
+type tokenOrError struct {
+	*oauth2.Token
+	Error string `json:"error,omitempty"`
+}
+
+// WaitForDeviceAuthorization polls the token URL waiting for the user to
+// authorize the app. Upon authorization, it returns the new token. If
+// authorization fails then an error is returned. If that failure was due to a
+// user explicitly denying access, the error is ErrAccessDenied.
+func waitForDeviceAuthorization(oauthConfig *oauth2.Config, opt *Options, deviceCode *oauth2.DeviceAuthResponse) (*oauth2.Token, error) {
+	client := http.DefaultClient
+
+	grantType := opt.DeviceGrantType
+	if grantType == "" {
+		grantType = "urn:ietf:params:oauth:grant-type:device_code"
+	}
+
+	for {
+		resp, err := client.PostForm(oauthConfig.Endpoint.TokenURL,
+			url.Values{
+				"client_secret": {oauthConfig.ClientSecret},
+				"client_id":     {oauthConfig.ClientID},
+				"code":          {deviceCode.DeviceCode},
+				"grant_type":    {grantType}})
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
+			return nil, fmt.Errorf("HTTP error %v (%v) when polling for OAuth token",
+				resp.StatusCode, http.StatusText(resp.StatusCode))
+		}
+
+		// Unmarshal response, checking for errors
+		var token tokenOrError
+
+		dec := json.NewDecoder(resp.Body)
+		if err := dec.Decode(&token); err != nil {
+			return nil, err
+		}
+
+		switch token.Error {
+		case "":
+
+			return token.Token, nil
+		case "authorization_pending":
+
+		case "slow_down":
+
+			deviceCode.Interval *= 2
+		case "access_denied":
+
+			return nil, errors.New("access denied by user")
+		default:
+
+			return nil, fmt.Errorf("authorization failed: %v", token.Error)
+		}
+
+		time.Sleep(time.Duration(deviceCode.Interval) * time.Second)
+	}
+}
+
 // get the URL we need to send the user to
 func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (authURL string, state string, err error) {
 	oauthConfig, _ = overrideCredentials(name, m, oauthConfig)