log: Updated filename timestamp wildcard and format method

Changed the flag for filename timestamp format from `log-file-time-format` to `file-time-format`. It now allows user to specify `WILDCARD` along with the format. Also, the formatting is now done by `go-strftime` library.

docs/content/docs.md

@@ -1119,6 +1119,22 @@ triggering follow-on actions if data was copied, or skipping if not.
 NB: Enabling this option turns a usually non-fatal error into a potentially
 fatal one - please check and adjust your scripts accordingly!
 
+### --file-time-format=WILDCARD~#~FORMAT ###
+
+The value provided for this flag is split into two parts (`WILDCARD` and `FORMAT`)
+at `~#~`. The `FORMAT` is a time stamp format as defined
+[here](https://pkg.go.dev/github.com/ncruces/go-strftime). The timestamp is converted
+into the provided format and all the `WILDCARD` in the filename,
+(e.g., `--log-file=FILE`, see [Log File Section](#--log-filefile-))
+will be replaced by the formatted timestamp.
+
+E.g. usage:
+
+Assume that the current time is `27 March 2024 10:00 P.M.` and the flag is used as
+`--file-time-format=:dt:~#~%Y-%m-%d` along with the flag `--log-file=C:\err_:dt:.log`.
+This means that the wildcard is `:dt:` and the time format is `%Y-%m-%d`.
+So the logs will be printed in the file `C:\err_2024-03-27.log`.
+
 ### --fix-case ###
 
 Normally, a sync to a case insensitive dest (such as macOS / Windows) will
@@ -1390,42 +1406,6 @@ The options mean
 
 During rmdirs it will not remove root directory, even if it's empty.
 
-### --log-file-time-format=FORMAT ###
-
-A format that is used to generate a datetime string. Any `:dt:` wildcard
-in the log filename specified via `--log-file=FILE` will be replaced with
-this datetime string.
-
-If `FORMAT` is `%Y-%m-%d`, and log-file `FILE` is `err_:dt:.log`, then
-all rclone output will be logged to the file `err_2024-03-26.log`
-assuming that the current date it `26 March 2024`.
-
-Following keywords in the `FORMAT` are recognised as below:
-
-`%A` - Weekday (E.g.: Monday)
-
-`%a` - Short Weekday (E.g.: Mon)
-
-`%Y` - Year (E.g.: 2006)
-
-`%y` - Short Year (E.g.: 06)
-
-`%B` - Month name (E.g.: January)
-
-`%b` - Short Month name (E.g.: Jan)
-
-`%m` - Month number (E.g.: 01)
-
-`%d` - Day of month (E.g.: 02)
-
-`%H` - Hour of day in 24-hour format (E.g.: 15)
-
-`%I` - Hour of day in 12-hour format (E.g.: 03)
-
-`%M` - Minute of hour (E.g.: 04)
-
-`%S` - Second of minute (E.g.: 05)
-
 ### --log-file=FILE ###
 
 Log all of rclone's output to FILE.  This is not active by default.

fs/log/log.go

@@ -11,18 +11,20 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ncruces/go-strftime"
 	"github.com/rclone/rclone/fs"
 	"github.com/sirupsen/logrus"
 )
 
 // Options contains options for controlling the logging
 type Options struct {
-	File               string // Log everything to this file
-	FilenameTimeFormat string // Date time format that is used to replace the :dt: in logging filename
-	Format             string // Comma separated list of log format options
-	UseSyslog          bool   // Use Syslog for logging
-	SyslogFacility     string // Facility for syslog, e.g. KERN,USER,...
-	LogSystemdSupport  bool   // set if using systemd logging
+	File                 string // Log everything to this file
+	FilenameTimeFormat   string // Date time format that is used to replace the :dt: in logging filename
+	FilenameTimeWildcard string // The wildcard that is replaced with formatted time in logging filenames
+	Format               string // Comma separated list of log format options
+	UseSyslog            bool   // Use Syslog for logging
+	SyslogFacility       string // Facility for syslog, e.g. KERN,USER,...
+	LogSystemdSupport    bool   // set if using systemd logging
 }
 
 // DefaultOpt is the default values used for Opt
@@ -116,31 +118,25 @@ func InitLogging() {
 
 	fs.LogPrintPid = strings.Contains(flagsStr, ",pid,")
 
+	if Opt.FilenameTimeFormat != "" {
+		tmp := strings.SplitN(Opt.FilenameTimeFormat, "~#~", 2)
+		if len(tmp) != 2 {
+			log.Fatalf("Invalid Filename Time Format provided via --file-time-format. Please refer docs for proper usage.")
+		}
+
+		Opt.FilenameTimeWildcard = tmp[0]
+		Opt.FilenameTimeFormat = tmp[1]
+
+		if Opt.FilenameTimeWildcard == "" {
+			log.Fatalf("The wildcard in the Filename Time Format cannot be empty")
+		}
+	}
+
 	// Log file output
 	if Opt.File != "" {
-		if Opt.FilenameTimeFormat != "" {
-			if strings.Index(Opt.File, ":dt:") == -1 {
-				log.Fatalf("Datetime format has been provided without using :dt: wildcard in the log filename")
-			}
-
-			currTime := time.Now()
-			replacer := strings.NewReplacer(
-				"%A", currTime.Format("Monday"),
-				"%a", currTime.Format("Mon"),
-				"%Y", currTime.Format("2006"),
-				"%y", currTime.Format("06"),
-				"%B", currTime.Format("January"),
-				"%b", currTime.Format("Jan"),
-				"%m", currTime.Format("01"),
-				"%d", currTime.Format("02"),
-				"%H", currTime.Format("15"),
-				"%I", currTime.Format("03"),
-				"%M", currTime.Format("04"),
-				"%S", currTime.Format("05"),
-			)
-
-			timeStr := replacer.Replace(Opt.FilenameTimeFormat)
-			Opt.File = strings.ReplaceAll(Opt.File, ":dt:", timeStr)
+		if Opt.FilenameTimeFormat != "" && Opt.FilenameTimeWildcard != "" {
+			timeStr := strftime.Format(Opt.FilenameTimeFormat, time.Now())
+			Opt.File = strings.ReplaceAll(Opt.File, Opt.FilenameTimeWildcard, timeStr)
 		}
 
 		f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)

fs/log/logflags/logflags.go

@@ -12,7 +12,7 @@ import (
 func AddFlags(flagSet *pflag.FlagSet) {
 	rc.AddOption("log", &log.Opt)
 
-	flags.StringVarP(flagSet, &log.Opt.FilenameTimeFormat, "log-filename-time-format", "", log.Opt.FilenameTimeFormat, "Date time format that is used to replace the :dt: in logging filename", "Logging")
+	flags.StringVarP(flagSet, &log.Opt.FilenameTimeFormat, "file-time-format", "", log.Opt.FilenameTimeFormat, "Date time format that is used to replace the provided wildcard with time in several filenames", "Logging")
 	flags.StringVarP(flagSet, &log.Opt.File, "log-file", "", log.Opt.File, "Log everything to this file", "Logging")
 	flags.StringVarP(flagSet, &log.Opt.Format, "log-format", "", log.Opt.Format, "Comma separated list of log format options", "Logging")
 	flags.BoolVarP(flagSet, &log.Opt.UseSyslog, "syslog", "", log.Opt.UseSyslog, "Use Syslog for logging", "Logging")

go.mod

@@ -46,9 +46,11 @@ require (
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/sys/mountinfo v0.7.1
+	github.com/ncruces/go-strftime v0.1.9
 	github.com/ncw/swift/v2 v2.0.2
 	github.com/oracle/oci-go-sdk/v65 v65.55.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.6
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.18.0

go.sum

@@ -414,6 +414,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
 github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=