From 708715e7058dc01dbb8f963288e19db98ba1ca45 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 13:30:46 +0200 Subject: [PATCH 001/238] Fix force-locale option for CLI and GUI. - Localization culture is set from multiple threads on the server, so use current local context for every new call instead of global context - Change locale before executing help commands if flag is present - When force-locale is set, also change locale for log messages Closes #4176 --- Duplicati/CommandLine/Commands.cs | 34 +++---- .../Localization/LocalizationService.cs | 11 ++- Duplicati/Library/Localization/Short.cs | 14 ++- Duplicati/Library/Main/Controller.cs | 54 ++--------- .../Main/Duplicati.Library.Main.csproj | 1 + Duplicati/Library/Main/LocaleChange.cs | 90 +++++++++++++++++++ 6 files changed, 126 insertions(+), 78 deletions(-) create mode 100644 Duplicati/Library/Main/LocaleChange.cs diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 74523762ce..541a5b5769 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -129,13 +129,15 @@ public void Dispose() public static int Examples(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { - Duplicati.CommandLine.Help.PrintUsage(outwriter, "example", options); + using (new Duplicati.Library.Main.LocaleChange(options)) + Duplicati.CommandLine.Help.PrintUsage(outwriter, "example", options); return 0; } public static int Help(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { - Duplicati.CommandLine.Help.PrintUsage(outwriter, args.Count > 1? args[1] : "help", options); + using (new Duplicati.Library.Main.LocaleChange(options)) + Duplicati.CommandLine.Help.PrintUsage(outwriter, args.Count > 1 ? args[1] : "help", options); return 0; } @@ -170,8 +172,8 @@ public static int Affected(TextWriter outwriter, Action @@ -254,7 +256,7 @@ private static string PrefixArgWithAsterisk(string arg) if (!containsSeparators && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and prefix with ".*/" - return $"[.*{Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; + return $"[.*{Library.Utility.Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; } else if (!containsSeparators && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -285,7 +287,7 @@ private static string SuffixArgWithAsterisk(string arg) if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and suffix with ".*" - return $"[{Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; + return $"[{Library.Utility.Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; } else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -319,7 +321,7 @@ public static int List(TextWriter outwriter, Action= 0) outwriter.WriteLine("{0}\t: {1} ({2} files, {3})", e.Version, e.Time, e.FileCount, Library.Utility.Utility.FormatSizeString(e.FileSizes)); @@ -439,10 +441,10 @@ public static int List(TextWriter outwriter, Action new { Index = a.Version, Time = a.Time, Size = b } )) + foreach (var nx in res.Filesets.Zip(e.Sizes, (a, b) => new { Index = a.Version, Time = a.Time, Size = b })) outwriter.WriteLine("{0}\t: {1} {2}", nx.Index, nx.Time, nx.Size < 0 ? " - " : Library.Utility.Utility.FormatSizeString(nx.Size)); outwriter.WriteLine(); @@ -466,7 +468,7 @@ public static int Delete(TextWriter outwriter, Action /// Gets a localization provider with the current language /// - public static ILocalizationService Current - { - get + public static ILocalizationService Current + { + get { var lc = System.Runtime.Remoting.Messaging.CallContext.LogicalGetData(LOGICAL_CONTEXT_KEY) as string; if (!string.IsNullOrWhiteSpace(lc)) return Get(new CultureInfo(lc)); - return Get(CultureInfo.CurrentCulture); - } + return Get(CultureInfo.CurrentCulture); + } } /// diff --git a/Duplicati/Library/Localization/Short.cs b/Duplicati/Library/Localization/Short.cs index 5cf29da586..2c04b506df 100644 --- a/Duplicati/Library/Localization/Short.cs +++ b/Duplicati/Library/Localization/Short.cs @@ -16,6 +16,7 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Threading; namespace Duplicati.Library.Localization.Short { @@ -27,15 +28,10 @@ public static class LC /// /// The instance for translation /// - private static ILocalizationService LS = LocalizationService.Current; - - /// - /// Sets the culture - /// - /// CultureInfo - public static void setCulture(System.Globalization.CultureInfo ci) + private static ILocalizationService LS { - LS = LocalizationService.Get(ci); + // Get up-to-date service (may be changed by temporary contexts) + get => LocalizationService.Current; } /// @@ -93,7 +89,7 @@ public static string L(string message, object arg0, object arg1, object arg2) public static string L(string message, params object[] args) { return LS.Localize(message, args); - } + } } } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 5b319fab35..48316a1166 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -64,19 +64,9 @@ public class Controller : IDisposable private System.Threading.ThreadPriority? m_resetPriority; /// - /// The localization culture to reset to + /// If not null, active locale change that needs to be reset /// - private System.Globalization.CultureInfo m_resetLocale; - - /// - /// The localization UI culture to reset to - /// - private System.Globalization.CultureInfo m_resetLocaleUI; - - /// - /// True if the locale should be reset - /// - private bool m_doResetLocale; + private LocaleChange m_localeChange = null; /// /// The multi-controller log target @@ -477,26 +467,6 @@ private T RunAction(T result, ref string[] paths, ref IFilter filter, Action< } } - /// - /// Attempts to get the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoGetLocale(out System.Globalization.CultureInfo locale, out System.Globalization.CultureInfo uiLocale) - { - locale = System.Globalization.CultureInfo.DefaultThreadCurrentCulture; - uiLocale = System.Globalization.CultureInfo.DefaultThreadCurrentUICulture; - } - - /// - /// Attempts to set the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoSetLocale(System.Globalization.CultureInfo locale, System.Globalization.CultureInfo uiLocale) - { - System.Globalization.CultureInfo.DefaultThreadCurrentCulture = locale; - System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = uiLocale; - } - private void OnOperationComplete(object result) { if (m_options != null && m_options.LoadedModules != null) @@ -520,14 +490,10 @@ private void OnOperationComplete(object result) m_resetPriority = null; } - if (m_doResetLocale) + if(m_localeChange != null) { - // Wrap the call to avoid loading issues for the setLocale method - DoSetLocale(m_resetLocale, m_resetLocaleUI); - - m_doResetLocale = false; - m_resetLocale = null; - m_resetLocaleUI = null; + m_localeChange.Dispose(); + m_localeChange = null; } if (m_logTarget != null) @@ -635,17 +601,11 @@ private void SetupCommonOptions(ISetCommonOptions result, ref string[] paths, re { try { - var locale = m_options.ForcedLocale; - DoGetLocale(out m_resetLocale, out m_resetLocaleUI); - m_doResetLocale = true; - // Wrap the call to avoid loading issues for the setLocale method - DoSetLocale(locale, locale); + m_localeChange = new LocaleChange(m_options.ForcedLocale); } - catch (Exception ex) // or only: MissingMethodException + catch (Exception ex) { Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message)); - m_doResetLocale = false; - m_resetLocale = m_resetLocaleUI = null; } } diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 2788134558..8db76b9e91 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -142,6 +142,7 @@ + diff --git a/Duplicati/Library/Main/LocaleChange.cs b/Duplicati/Library/Main/LocaleChange.cs new file mode 100644 index 0000000000..5d3952b308 --- /dev/null +++ b/Duplicati/Library/Main/LocaleChange.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Main +{ + public class LocaleChange : IDisposable + { + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + private bool m_doResetLocale = false; + private System.Globalization.CultureInfo m_resetLocale = null; + private System.Globalization.CultureInfo m_resetLocaleUI = null; + + private IDisposable m_localizationContext = null; + + public LocaleChange(System.Globalization.CultureInfo newLocale) + { + if (newLocale == null) + { + return; + } + try + { + DoGetLocale(out m_resetLocale, out m_resetLocaleUI); + m_doResetLocale = true; + // Wrap the call to avoid loading issues for the setLocale method + DoSetLocale(newLocale, newLocale); + m_localizationContext = Localization.LocalizationService.TemporaryContext(newLocale); + } + catch (MissingMethodException ex) + { + m_doResetLocale = false; + m_resetLocale = m_resetLocaleUI = null; + + Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message)); + } + } + + public LocaleChange(Options options) + : this(options.HasForcedLocale ? options.ForcedLocale : null) + { + } + public LocaleChange(Dictionary options) + : this(new Options(options)) + { + } + + public void Dispose() + { + if (m_doResetLocale) + { + // Wrap the call to avoid loading issues for the setLocale method + DoSetLocale(m_resetLocale, m_resetLocaleUI); + + m_doResetLocale = false; + m_resetLocale = null; + m_resetLocaleUI = null; + } + if(m_localizationContext != null) + { + m_localizationContext.Dispose(); + m_localizationContext = null; + } + } + + /// + /// Attempts to get the locale, but delays linking to the calls as they are missing in some environments + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void DoGetLocale(out System.Globalization.CultureInfo locale, out System.Globalization.CultureInfo uiLocale) + { + locale = System.Globalization.CultureInfo.DefaultThreadCurrentCulture; + uiLocale = System.Globalization.CultureInfo.DefaultThreadCurrentUICulture; + } + + /// + /// Attempts to set the locale, but delays linking to the calls as they are missing in some environments + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void DoSetLocale(System.Globalization.CultureInfo locale, System.Globalization.CultureInfo uiLocale) + { + System.Globalization.CultureInfo.DefaultThreadCurrentCulture = locale; + System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = uiLocale; + } + + } +} From b6643fa47322b4391c478dbbee4cbd2d9bb2220a Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:22:36 +0200 Subject: [PATCH 002/238] Add documentation comments for LocaleChange. --- Duplicati/Library/Main/LocaleChange.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Duplicati/Library/Main/LocaleChange.cs b/Duplicati/Library/Main/LocaleChange.cs index 5d3952b308..402d240e1f 100644 --- a/Duplicati/Library/Main/LocaleChange.cs +++ b/Duplicati/Library/Main/LocaleChange.cs @@ -6,6 +6,10 @@ namespace Duplicati.Library.Main { + /// + /// Changes the current locale for all threads and for log messages in the current context. + /// When disposed the changes are reset. + /// public class LocaleChange : IDisposable { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -16,6 +20,9 @@ public class LocaleChange : IDisposable private IDisposable m_localizationContext = null; + /// + /// Change locale if newLocale is not null + /// public LocaleChange(System.Globalization.CultureInfo newLocale) { if (newLocale == null) @@ -39,10 +46,24 @@ public LocaleChange(System.Globalization.CultureInfo newLocale) } } + /// + /// Change locale if ForcedLocale is specified in options + /// + /// Command line options + /// + /// Thrown if specified locale was not found. + /// public LocaleChange(Options options) : this(options.HasForcedLocale ? options.ForcedLocale : null) { } + /// + /// Change locale if 'force-locale' is specified in options dictionary + /// + /// Command line options + /// + /// Thrown if specified locale was not found. + /// public LocaleChange(Dictionary options) : this(new Options(options)) { From b9d4be60ca9b916a12529a8a8a74b490718ef3f2 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 16:34:43 +0200 Subject: [PATCH 003/238] Do not set advanced options with empty string to true. There are advanced options such as force-local that expect an empty string in some cases. While saving, these were replaced with 'true' even though there was an equals sign. Now only options without an equals sign are set to true. Closes #4757 --- Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index 0817690d28..ba90444a37 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -280,8 +280,6 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer if (eqpos > 0) { key = line.substr(0, eqpos).trim(); value = line.substr(eqpos + 1).trim(); - if (value == '') - value = true; } if (validateCallback) From 0f5b4b97070271935a0d8bc8ce8dfed069d2959a Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Mon, 10 Jul 2023 16:44:24 +0200 Subject: [PATCH 004/238] Fix restore of empty files with recovery tool. Empty files do not have data blocks stored, so don't try to get a block for empty files. --- Duplicati/CommandLine/RecoveryTool/Restore.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/CommandLine/RecoveryTool/Restore.cs b/Duplicati/CommandLine/RecoveryTool/Restore.cs index b2fc5282e5..e7e6e44621 100644 --- a/Duplicati/CommandLine/RecoveryTool/Restore.cs +++ b/Duplicati/CommandLine/RecoveryTool/Restore.cs @@ -171,7 +171,8 @@ public static int Run(List args, Dictionary options, Lib { if (f.BlocklistHashes == null) { - lookup.WriteHash(sw, f.Hash); + if(f.Size > 0) + lookup.WriteHash(sw, f.Hash); } else { From 6b5623651c87137af5946ef420731e101a94f194 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:32:35 +0200 Subject: [PATCH 005/238] Fix recovery tool restore paths on different OS. On Linux, paths were created with \ instead of / separators if backup was originally from Windows (#4884). Also fix bug that created one unnecessary level of directories in the restore path. --- Duplicati/CommandLine/RecoveryTool/Restore.cs | 111 +++++++++++------- 1 file changed, 70 insertions(+), 41 deletions(-) diff --git a/Duplicati/CommandLine/RecoveryTool/Restore.cs b/Duplicati/CommandLine/RecoveryTool/Restore.cs index e7e6e44621..c95b7aa2ff 100644 --- a/Duplicati/CommandLine/RecoveryTool/Restore.cs +++ b/Duplicati/CommandLine/RecoveryTool/Restore.cs @@ -104,53 +104,19 @@ public static int Run(List args, Dictionary options, Lib using (var mru = new CompressedFileMRUCache(options)) { Console.WriteLine("Building lookup table for file hashes"); - using (HashLookupHelper lookup = new HashLookupHelper(ixfile, mru, (int) blocksize, blockhasher.HashSize / 8)) + // Source OS can have different directory separator + string sourceDirsep = null; + using (HashLookupHelper lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8)) { var filecount = 0L; - string largestprefix = null; - string[] largestprefixparts = null; if (!string.IsNullOrWhiteSpace(targetpath)) Console.WriteLine("Computing restore path"); - foreach (var f in List.EnumerateFilesInDList(filelist, filter, options)) - { - if (largestprefix == null) - { - largestprefix = f.Path; - largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar }); - } - else if (largestprefix.Length > 1) - { - var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar }); - - var ni = 0; - for (; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++) - if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni])) - break; - - if (ni != largestprefixparts.Length) - { - if (ni == 0) - { - largestprefixparts = new string[0]; - largestprefix = string.Empty; - } - else - { - Array.Resize(ref largestprefixparts, ni - 1); - largestprefix = string.Join(Util.DirectorySeparatorString, largestprefixparts); - } - } - } - filecount++; - } + string largestprefix = GetLargestPrefix(from f in List.EnumerateFilesInDList(filelist, filter, options) select f.Path, out sourceDirsep, out filecount); Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath); - if (Platform.IsClientPosix || largestprefix.Length > 0) - largestprefix = Util.AppendDirSeparator(largestprefix); - if (!string.IsNullOrEmpty(largestprefix)) Console.WriteLine("Removing common prefix {0} from files", largestprefix); @@ -159,7 +125,7 @@ public static int Run(List args, Dictionary options, Lib { try { - var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath); + var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath, sourceDirsep); if (!systemIO.DirectoryExists(systemIO.PathGetDirectoryName(targetfile))) systemIO.DirectoryCreate(systemIO.PathGetDirectoryName(targetfile)); @@ -171,7 +137,7 @@ public static int Run(List args, Dictionary options, Lib { if (f.BlocklistHashes == null) { - if(f.Size > 0) + if (f.Size > 0) lookup.WriteHash(sw, f.Hash); } else @@ -244,8 +210,71 @@ public static int Run(List args, Dictionary options, Lib return 0; } - private static string MapToRestorePath(string path, string prefixpath, string restorepath) + public static string GetLargestPrefix(IEnumerable filePaths, out string sourceDirsep, out long filecount) { + // Get dir separator like in LocalRestoreDatabase.GetLargestPrefix(): + string largestprefix = filePaths.OrderByDescending(p => p.Length).FirstOrDefault() ?? string.Empty; + sourceDirsep = Util.GuessDirSeparator(largestprefix); + + + string[] dirsepSplit = new string[] { sourceDirsep }; + string[] largestprefixparts = largestprefix.Split(dirsepSplit, StringSplitOptions.None); + + // Because only files are in the list, need to remove filename from prefix + // Otherwise, in case of a single file the prefix is not a directory + if (largestprefixparts.Length > 0) + { + Array.Resize(ref largestprefixparts, largestprefixparts.Length - 1); + } + largestprefix = string.Join(sourceDirsep, largestprefixparts); + + + + filecount = 0; + foreach (var path in filePaths) + { + if (largestprefix.Length > 1) + { + // Unix paths starting with / have an empty string in paths[0] + // Completely empty strings should not combine with that, so should have no parts at all + var parts = path.Length == 0 ? new string[0] : path.Split(dirsepSplit, StringSplitOptions.None); + + var ni = 0; + for (; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++) + if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni])) + break; + + if (ni != largestprefixparts.Length) + { + if (ni == 0) + { + largestprefixparts = new string[0]; + largestprefix = string.Empty; + } + else + { + // Only the first ni parts match + Array.Resize(ref largestprefixparts, ni); + largestprefix = string.Join(sourceDirsep, largestprefixparts); + } + } + } + filecount++; + } + return largestprefixparts.Length == 0 ? "" : Util.AppendDirSeparator(largestprefix, sourceDirsep); + } + + public static string MapToRestorePath(string path, string prefixpath, string restorepath, string sourceDirsep) + { + if (sourceDirsep != null && sourceDirsep != Util.DirectorySeparatorString && sourceDirsep != Util.AltDirectorySeparatorString) + { + // Replace directory separator in source and prefix path + path = path.Replace(sourceDirsep, Util.DirectorySeparatorString); + if (!string.IsNullOrWhiteSpace(prefixpath)) + { + prefixpath = prefixpath.Replace(sourceDirsep, Util.DirectorySeparatorString); + } + } if (string.IsNullOrWhiteSpace(restorepath)) return path; From cff1049347d569dd30866f375ed8188a6e02719e Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:35:14 +0200 Subject: [PATCH 006/238] Add unit tests for recovery tool restore paths. Test that paths are generated correctly from a different source OS. Also test that empty files are restored by recovery tool. --- Duplicati/UnitTest/RecoveryToolTests.cs | 170 ++++++++++++++++++++++-- 1 file changed, 160 insertions(+), 10 deletions(-) diff --git a/Duplicati/UnitTest/RecoveryToolTests.cs b/Duplicati/UnitTest/RecoveryToolTests.cs index b81d7b25ae..a1f524ad39 100644 --- a/Duplicati/UnitTest/RecoveryToolTests.cs +++ b/Duplicati/UnitTest/RecoveryToolTests.cs @@ -33,6 +33,160 @@ public override void TearDown() base.TearDown(); } + [Test] + [Category("RecoveryTool")] + public void RecoveryRestorePrefix() + { + string sourceDirsep; + long fileCount; + string largestprefix; + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + // sourceDirsep unspecified + Assert.AreEqual(0, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { "" }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + // sourceDirsep unspecified + Assert.AreEqual(1, fileCount); + + // Windows paths + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\User\\Pictures\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(1, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Users\\User\\Pictures\\c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\User\\Pictures\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Users\\User2\\Pictures\\b.jpg" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Data\\a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "D:\\Users\\User\\Pictures\\a.jpg" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(2, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(2, fileCount); + + // Unix paths + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/user/pictures/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(1, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/home/user/pictures/b.jpg", + "/home/user/pictures/c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/user/pictures/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/home/user/pictures/b.jpg", + "/home/user2/pictures/c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/media/data/a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(2, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(2, fileCount); + } + + [Test] + [Category("RecoveryTool")] + public void RecoveryRestorePathMap() + { + // Windows source paths + string restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\User\\Pictures\\", "restore", "\\"); + // Replace the alt directory separator with normal directory separator to compare paths, + // otherwise the path returned by Path.Combine does not match + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\", "restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "User", "Pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\", "restore" + Path.DirectorySeparatorChar, "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "User", "Pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "", "restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "C", "Users", "User", "Pictures", "a.png"), restorePath); + // This is an absolute restore path on Windows and a relative path with backslashes on Unix + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "", "C:\\Users\\User\\restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("C:\\Users\\User\\restore", "C", "Users", "User", "Pictures", "a.png"), restorePath); + + // Unix source paths + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/user/pictures/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/user/pictures/", "restore" + Path.DirectorySeparatorChar, "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "user", "pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "home", "user", "pictures", "a.png"), restorePath); + // This is an absolute restore path on Windows and a relative path with backslashes on Unix + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/", "C:\\Users\\User\\restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("C:\\Users\\User\\restore", "home", "user", "pictures", "a.png"), restorePath); + } + [Test] [Category("RecoveryTool")] [TestCase("false")] @@ -40,7 +194,7 @@ public override void TearDown() public void Recover(string buildIndexWithFiles) { // Files to create in MB. - int[] fileSizes = {10, 20, 30}; + int[] fileSizes = { 0, 10, 20, 30 }; foreach (int size in fileSizes) { byte[] data = new byte[size * 1024 * 1024]; @@ -65,7 +219,7 @@ public void Recover(string buildIndexWithFiles) string backendURL = "file://" + this.TARGETFOLDER; using (Controller c = new Controller(backendURL, options, null)) { - IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}); + IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(0, backupResults.Warnings.Count()); } @@ -73,24 +227,20 @@ public void Recover(string buildIndexWithFiles) // Download the backend files. string downloadFolder = Path.Combine(this.RESTOREFOLDER, "downloadedFiles"); Directory.CreateDirectory(downloadFolder); - int status = CommandLine.RecoveryTool.Program.RealMain(new[] {"download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}"}); + int status = CommandLine.RecoveryTool.Program.RealMain(new[] { "download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}" }); Assert.AreEqual(0, status); // Create the index. - status = CommandLine.RecoveryTool.Program.RealMain(new[] {"index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}"}); + status = CommandLine.RecoveryTool.Program.RealMain(new[] { "index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}" }); Assert.AreEqual(0, status); // Restore to a different folder. string restoreFolder = Path.Combine(this.RESTOREFOLDER, "restoredFiles"); Directory.CreateDirectory(restoreFolder); - status = CommandLine.RecoveryTool.Program.RealMain(new[] {"restore", $"{downloadFolder}", $"--targetpath={restoreFolder}"}); + status = CommandLine.RecoveryTool.Program.RealMain(new[] { "restore", $"{downloadFolder}", $"--targetpath={restoreFolder}" }); Assert.AreEqual(0, status); - // Since this.DATAFOLDER is a folder, Path.GetFileName will return the name of the - // last folder in the path. - string baseFolder = Path.GetFileName(this.DATAFOLDER); - - TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, Path.Combine(restoreFolder, baseFolder), false, "Verifying restore using RecoveryTool."); + TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, restoreFolder, false, "Verifying restore using RecoveryTool."); } } } \ No newline at end of file From 8af1dbb6fd084c04628b722e5b5f7d21db92a732 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:16:11 +0200 Subject: [PATCH 007/238] Add --quota-disable option to disable reported backend quota. If set, the backend is treated as if it does not report a quota at all. No warnings and errors are created and no quota values are set in the backup log. This option is intended to be used if a backend reports a wrong quota, so it needs to be disabled. --- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Main/Operation/FilelistProcessor.cs | 2 +- Duplicati/Library/Main/Options.cs | 120 ++++++++++-------- Duplicati/Library/Main/Strings.cs | 2 + 4 files changed, 70 insertions(+), 56 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index ef12b94e8d..004e99c37c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -329,7 +329,7 @@ private void UpdateStorageStatsFromDatabase() // TODO: If we have a BackendManager, we should query through that using (var backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions)) { - if (backend is IQuotaEnabledBackend enabledBackend) + if (backend is IQuotaEnabledBackend enabledBackend && !m_options.QuotaDisable) { Library.Interface.IQuotaInfo quota = enabledBackend.Quota; if (quota != null) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 0652020d82..5df8770dff 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -234,7 +234,7 @@ public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Op // TODO: We should query through the backendmanager using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) - if (bk is IQuotaEnabledBackend enabledBackend) + if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) { Library.Interface.IQuotaInfo quota = enabledBackend.Quota; if (quota != null) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 1c638ddc4d..75a5447f35 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -34,7 +34,7 @@ public class Options { private const string DEFAULT_BLOCK_HASH_ALGORITHM = "SHA256"; private const string DEFAULT_FILE_HASH_ALGORITHM = "SHA256"; - + /// /// The default block size /// @@ -44,17 +44,17 @@ public class Options /// The default threshold value /// private const long DEFAULT_THRESHOLD = 25; - + /// /// The default value for maximum number of small files /// private const long DEFAULT_SMALL_FILE_MAX_COUNT = 20; - + /// /// Default size of volumes /// private const string DEFAULT_VOLUME_SIZE = "50mb"; - + /// /// Default value for keep-versions /// @@ -74,7 +74,7 @@ public class Options /// The default number of hasher instances /// private readonly int DEFAULT_BLOCK_HASHERS = Math.Max(1, Environment.ProcessorCount / 2); - + /// /// The default threshold for warning about coming close to quota /// @@ -133,7 +133,7 @@ public enum HardlinkStrategy /// Process only the first hardlink /// First, - + /// /// Process all hardlinks /// @@ -144,7 +144,7 @@ public enum HardlinkStrategy /// None } - + /// /// The possible settings for index file usage /// @@ -154,17 +154,17 @@ public enum IndexFileStrategy /// Disables usage of index files /// None, - + /// /// Stores only block lookup information in the index files /// Lookup, - + /// /// Stores both block lookup and block lists in the index files /// Full - + } private static string[] GetSupportedHashes() @@ -182,7 +182,7 @@ private static string[] GetSupportedHashes() { } } - + return r.ToArray(); } @@ -286,13 +286,13 @@ public IList SupportedCommands new CommandLineArgument("asynchronous-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousuploadlimitShort, Strings.Options.AsynchronousuploadlimitLong, "4"), new CommandLineArgument("asynchronous-concurrent-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousconcurrentuploadlimitShort, Strings.Options.AsynchronousconcurrentuploadlimitLong, "4"), new CommandLineArgument("asynchronous-upload-folder", CommandLineArgument.ArgumentType.Path, Strings.Options.AsynchronousuploadfolderShort, Strings.Options.AsynchronousuploadfolderLong, System.IO.Path.GetTempPath()), - + new CommandLineArgument("disable-streaming-transfers", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableStreamingShort, Strings.Options.DisableStreamingLong, "false"), new CommandLineArgument("throttle-upload", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottleuploadShort, Strings.Options.ThrottleuploadLong, "0kb"), new CommandLineArgument("throttle-download", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottledownloadShort, Strings.Options.ThrottledownloadLong, "0kb"), new CommandLineArgument("skip-files-larger-than", CommandLineArgument.ArgumentType.Size, Strings.Options.SkipfileslargerthanShort, Strings.Options.SkipfileslargerthanLong), - + new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"), new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), @@ -327,6 +327,7 @@ public IList SupportedCommands new CommandLineArgument("quota-size", CommandLineArgument.ArgumentType.Size, Strings.Options.QuotasizeShort, Strings.Options.QuotasizeLong), new CommandLineArgument("quota-warning-threshold", CommandLineArgument.ArgumentType.Integer, Strings.Options.QuotaWarningThresholdShort, Strings.Options.QuotaWarningThresholdLong, DEFAULT_QUOTA_WARNING_THRESHOLD.ToString()), + new CommandLineArgument("quota-disable", CommandLineArgument.ArgumentType.Boolean, Strings.Options.QuotaDisableShort, Strings.Options.QuotaDisableLong("quota-size"), "false"), new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, Strings.Options.SymlinkpolicyLong("store", "ignore", "follow"), Enum.GetName(typeof(SymlinkStrategy), SymlinkStrategy.Store), null, Enum.GetNames(typeof(SymlinkStrategy))), new CommandLineArgument("hardlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.HardlinkpolicyShort, Strings.Options.HardlinkpolicyLong("first", "all", "none"), Enum.GetName(typeof(HardlinkStrategy), HardlinkStrategy.All), null, Enum.GetNames(typeof(HardlinkStrategy))), @@ -390,7 +391,7 @@ public IList SupportedCommands new CommandLineArgument("concurrency-max-threads", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencymaxthreadsShort, Strings.Options.ConcurrencymaxthreadsLong, "0"), new CommandLineArgument("concurrency-block-hashers", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencyblockhashersShort, Strings.Options.ConcurrencyblockhashersLong, DEFAULT_BLOCK_HASHERS.ToString()), new CommandLineArgument("concurrency-compressors", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencycompressorsShort, Strings.Options.ConcurrencycompressorsLong, DEFAULT_COMPRESSORS.ToString()), - + new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"), new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"), new CommandLineArgument("disable-filelist-consistency-checks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilelistconsistencychecksShort, Strings.Options.DisablefilelistconsistencychecksLong, "false"), @@ -412,7 +413,7 @@ public IList SupportedCommands /// /// Gets or sets the current main action of the instance /// - public OperationMode MainAction + public OperationMode MainAction { get { return (OperationMode)Enum.Parse(typeof(OperationMode), m_options["main-action"]); } set { m_options["main-action"] = value.ToString(); } @@ -499,7 +500,7 @@ public DateTime Time return Library.Utility.Timeparser.ParseTimeInterval(m_options["time"], DateTime.Now); } } - + /// /// Gets the versions the restore or list operation is limited to /// @@ -648,7 +649,7 @@ public string Prefix m_options.TryGetValue("prefix", out v); if (!string.IsNullOrEmpty(v)) return v; - + return "duplicati"; } } @@ -664,7 +665,7 @@ public int KeepVersions m_options.TryGetValue("keep-versions", out v); if (string.IsNullOrEmpty(v)) return DEFAULT_KEEP_VERSIONS; - + return Math.Max(0, int.Parse(v)); } } @@ -678,7 +679,7 @@ public DateTime KeepTime { string v; m_options.TryGetValue("keep-time", out v); - + if (string.IsNullOrEmpty(v)) return new DateTime(0); @@ -696,12 +697,14 @@ public DateTime KeepTime /// public List RetentionPolicy { - get { + get + { var retentionPolicyConfig = new List(); string v; m_options.TryGetValue("retention-policy", out v); - if (string.IsNullOrEmpty(v)) { + if (string.IsNullOrEmpty(v)) + { return retentionPolicyConfig; } @@ -1061,7 +1064,7 @@ public string AsynchronousUploadFolder return value; } } - + /// /// Gets the logfile filename /// @@ -1233,6 +1236,15 @@ public int QuotaWarningThreshold } } + /// + /// Gets a flag indicating that backup quota reported by the backend should be ignored + /// + /// This is necessary because in some cases the backend might report a wrong quota (especially with some Linux mounts). + public bool QuotaDisable + { + get { return Library.Utility.Utility.ParseBoolOption(m_options, "quota-disable"); } + } + /// /// Gets the display name of the backup /// @@ -1289,15 +1301,15 @@ public int Blocksize long blocksize = Library.Utility.Sizeparser.ParseSize(tmp, "kb"); if (blocksize > int.MaxValue || blocksize < 1024) throw new ArgumentOutOfRangeException(nameof(blocksize), string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue)); - + return (int)blocksize; } } - - /// + + /// /// Cache for the block hash size value, to avoid creating new hash instances just to get the size /// - private KeyValuePair m_cachedBlockHashSize; + private KeyValuePair m_cachedBlockHashSize; /// /// Gets the size of the blockhash in bytes. @@ -1307,10 +1319,10 @@ public int BlockhashSize { get { - if (m_cachedBlockHashSize.Key != BlockHashAlgorithm) - m_cachedBlockHashSize = new KeyValuePair(BlockHashAlgorithm, Duplicati.Library.Utility.HashAlgorithmHelper.Create(BlockHashAlgorithm).HashSize / 8); - - return m_cachedBlockHashSize.Value; + if (m_cachedBlockHashSize.Key != BlockHashAlgorithm) + m_cachedBlockHashSize = new KeyValuePair(BlockHashAlgorithm, Duplicati.Library.Utility.HashAlgorithmHelper.Create(BlockHashAlgorithm).HashSize / 8); + + return m_cachedBlockHashSize.Value; } } @@ -1374,8 +1386,8 @@ public bool UseBlockCache return Library.Utility.Utility.ParseBoolOption(m_options, "use-block-cache"); } } - - + + /// /// Gets the compact threshold /// @@ -1407,7 +1419,7 @@ public long SmallFileSize return Library.Utility.Sizeparser.ParseSize(v, "mb"); } } - + /// /// Gets the maximum number of small volumes /// @@ -1423,7 +1435,7 @@ public long SmallFileMaxCount return Convert.ToInt64(v); } } - + /// /// List of files to check for changes /// @@ -1483,26 +1495,26 @@ public string Restorepath return v; } } - + /// /// Gets the index file usage method /// public IndexFileStrategy IndexfilePolicy { - get - { + get + { string strategy; if (!m_options.TryGetValue("index-file-policy", out strategy)) strategy = ""; - + IndexFileStrategy res; if (!Enum.TryParse(strategy, true, out res)) res = IndexFileStrategy.Full; - + return res; } } - + /// /// Gets a flag indicating if the check for files on the remote storage should be omitted /// @@ -1510,7 +1522,7 @@ public bool NoBackendverification { get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-backend-verification"); } } - + /// /// Gets the percentage of samples to test during a backup operation /// @@ -1548,17 +1560,17 @@ public long BackupTestPercentage /// public long BackupTestSampleCount { - get - { + get + { string s; m_options.TryGetValue("backup-test-samples", out s); if (string.IsNullOrEmpty(s)) return 1; - + return long.Parse(s); } } - + /// /// Gets a flag indicating if compacting should not be done automatically /// @@ -1588,21 +1600,21 @@ public bool AllowMissingSource { get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-missing-source"); } } - + /// /// Gets a value indicating if a verification file should be uploaded after changing the remote store /// public bool UploadVerificationFile { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "upload-verification-file"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "upload-verification-file"); } } - + /// /// Gets a value indicating if a passphrase change is allowed /// public bool AllowPassphraseChange { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-passphrase-change"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-passphrase-change"); } } /// @@ -1610,12 +1622,12 @@ public bool AllowPassphraseChange /// public bool Dryrun { - get + get { if (m_options.ContainsKey("dry-run")) - return Library.Utility.Utility.ParseBoolOption(m_options, "dry-run"); + return Library.Utility.Utility.ParseBoolOption(m_options, "dry-run"); else - return Library.Utility.Utility.ParseBoolOption(m_options, "dryrun"); + return Library.Utility.Utility.ParseBoolOption(m_options, "dryrun"); } } @@ -1626,7 +1638,7 @@ public bool FullRemoteVerification { get { return Library.Utility.Utility.ParseBoolOption(m_options, "full-remote-verification"); } } - + /// /// The block hash algorithm to use /// @@ -1673,7 +1685,7 @@ public bool PatchWithLocalBlocks /// true if no local blocks; otherwise, false. public bool NoLocalBlocks { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-local-blocks"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-local-blocks"); } } /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 9ba71054fd..51979e3d27 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -127,6 +127,8 @@ internal static class Options public static string QuotasizeShort { get { return LC.L(@"A reported maximum storage"); } } public static string QuotaWarningThresholdLong { get { return LC.L(@"Sets a threshold for when to warn about the backend quota being nearly exceeded. It is given as a percentage, and a warning is generated if the amount of available quota is less than this percentage of the total backup size. If the backend does not report the quota information, this value will be ignored"); } } public static string QuotaWarningThresholdShort { get { return LC.L(@"Threshold for warning about low quota"); } } + public static string QuotaDisableLong(string optionname) { return LC.L(@"Disable the quota reported by the backend. --{0} can still be used to set a manual quota", optionname); } + public static string QuotaDisableShort { get { return LC.L(@"Disable backend quota"); } } public static string SymlinkpolicyShort { get { return LC.L(@"Symlink handling"); } } public static string SymlinkpolicyLong(string store, string ignore, string follow) { return LC.L(@"Use this option to handle symlinks differently. The ""{0}"" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option ""{1}"" to ignore all symlinks and not store any information about them. The option ""{2}"" will cause the symlinked target to be backed up and restored as a normal file with the symlink name. Early versions of Duplicati did not support this option and bevhaved as if ""{2}"" was specified.", store, ignore, follow); } public static string HardlinkpolicyShort { get { return LC.L(@"Hardlink handling"); } } From 0754a5cd72bf6040234e14430b2eafeb46ffd0d0 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:24:37 +0200 Subject: [PATCH 008/238] Ignore File backend quota if total size is zero. Mono sometimes reports the total size of unknown file systems as zero. Because it should be obvious when the size is actually zero, ignore the quota in this case. --- Duplicati/Library/Backend/File/FileBackend.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index a2b0054582..79c410e263 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -234,7 +234,7 @@ public Task PutAsync(string targetFilename, string sourceFilePath, CancellationT } else { - systemIO.FileCopy(sourceFilePath, targetFilePath, true); + systemIO.FileCopy(sourceFilePath, targetFilePath, true); if (m_verifyDestinationLength) VerifyMatchingSize(targetFilePath, sourceFilePath); } @@ -347,7 +347,12 @@ public IQuotaInfo Quota System.IO.DriveInfo driveInfo = this.GetDrive(); if (driveInfo != null) { - return new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace); + // Check that the total space is above 0, because Mono sometimes reports 0 for unknown file systems + // If the drive actually has a total size of 0, this should be obvious immediately due to write errors + if (driveInfo.TotalSize > 0) + { + return new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace); + } } if (Platform.IsClientWindows) From aa365f7f34ec7104249d656c036c602e4098ac07 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:49:54 +0200 Subject: [PATCH 009/238] Update --quota-size option to assign a backup size. The assigned quota size is checked in addition to the reported backend quota (if available). It can be used to limit the backup size on the remote. Warnings are created according to the same quota warning threshold. Previously, this option had no function. --- .../Main/Operation/FilelistProcessor.cs | 95 ++++++++++++------- Duplicati/Library/Main/Options.cs | 2 +- Duplicati/Library/Main/Strings.cs | 4 +- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 5df8770dff..f2d02ea14c 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -232,42 +232,9 @@ public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Op log.BackupListCount = database.FilesetTimes.Count(); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); - // TODO: We should query through the backendmanager - using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) - if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) - { - Library.Interface.IQuotaInfo quota = enabledBackend.Quota; - if (quota != null) - { - log.TotalQuotaSpace = quota.TotalQuotaSpace; - log.FreeQuotaSpace = quota.FreeQuotaSpace; - - // Check to see if there should be a warning or error about the quota - // Since this processor may be called multiple times during a backup - // (both at the start and end, for example), the log keeps track of - // whether a quota error or warning has been sent already. - // Note that an error can still be sent later even if a warning was sent earlier. - if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) - { - log.ReportedQuotaError = true; - Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - } - else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info - { - // Warnings are sent if the available free space is less than the given percentage of the total backup size. - double warningThreshold = options.QuotaWarningThreshold / (double)100; - if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) - { - log.ReportedQuotaWarning = true; - Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - } - } - } - } + CheckQuota(backend, options, log, knownFileSize); - log.AssignedQuotaSpace = options.QuotaSize; - - foreach(var s in remotelist) + foreach (var s in remotelist) lookup[s.File.Name] = s; var missing = new List(); @@ -401,5 +368,63 @@ public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Op VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }; } + + private static void CheckQuota(BackendManager backend, Options options, IBackendWriter log, long knownFileSize) + { + // TODO: We should query through the backendmanager + using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) + if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) + { + Library.Interface.IQuotaInfo quota = enabledBackend.Quota; + if (quota != null) + { + log.TotalQuotaSpace = quota.TotalQuotaSpace; + log.FreeQuotaSpace = quota.FreeQuotaSpace; + + // Check to see if there should be a warning or error about the quota + // Since this processor may be called multiple times during a backup + // (both at the start and end, for example), the log keeps track of + // whether a quota error or warning has been sent already. + // Note that an error can still be sent later even if a warning was sent earlier. + if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) + { + log.ReportedQuotaError = true; + Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + } + else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info + { + // Warnings are sent if the available free space is less than the given percentage of the total backup size. + double warningThreshold = options.QuotaWarningThreshold / (double)100; + if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) + { + log.ReportedQuotaWarning = true; + Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + } + } + } + } + + log.AssignedQuotaSpace = options.QuotaSize; + if (log.AssignedQuotaSpace != -1) + { + // Check assigned quota + if (!log.ReportedQuotaError && knownFileSize > log.AssignedQuotaSpace) + { + log.ReportedQuotaError = true; + Logging.Log.WriteErrorMessage(LOGTAG, "AssignedQuotaExceeded", null, "Assigned quota has been exceeded: Using {0} of {1}", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(log.AssignedQuotaSpace)); + } + else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError) + { + // Warnings are sent if the available free space is less than the given percentage of the total backup size. + double warningThreshold = options.QuotaWarningThreshold / (double)100; + long freeSpace = log.AssignedQuotaSpace - knownFileSize; + if (freeSpace < warningThreshold * knownFileSize) + { + log.ReportedQuotaWarning = true; + Logging.Log.WriteWarningMessage(LOGTAG, "AssignedQuotaNear", null, "Assigned quota is close to being exceeded: Using {0} of {1}", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(log.AssignedQuotaSpace)); + } + } + } + } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 75a5447f35..9639448c5e 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1200,7 +1200,7 @@ public System.IO.FileAttributes FileAttributeFilter public bool Overwrite { get { return GetBool("overwrite"); } } /// - /// Gets the total size in bytes that the backend supports, returns -1 if there is no upper limit + /// Gets the total size in bytes that the backup should use, returns -1 if there is no upper limit /// public long QuotaSize { diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 51979e3d27..2eb0db080f 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -123,8 +123,8 @@ internal static class Options public static string DebugretryerrorsShort { get { return LC.L(@"Show error messages when a retry is performed"); } } public static string UploadUnchangedBackupsLong { get { return LC.L(@"If no files have changed, Duplicati will not upload a backup set. If the backup data is used to verify that a backup was executed, this option will make Duplicati upload a backupset even if it is empty"); } } public static string UploadUnchangedBackupsShort { get { return LC.L(@"Upload empty backup files"); } } - public static string QuotasizeLong { get { return LC.L(@"This value can be used to set a known upper limit on the amount of space a backend has. If the backend reports the size itself, this value is ignored"); } } - public static string QuotasizeShort { get { return LC.L(@"A reported maximum storage"); } } + public static string QuotasizeLong { get { return LC.L(@"Set a limit to the amount of storage used on the backend (by this backup). This is in addition to the full backend quota, if available. Note: Backups will continue past the quota. This only creates warnings and error messages."); } } + public static string QuotasizeShort { get { return LC.L(@"Limit storage use"); } } public static string QuotaWarningThresholdLong { get { return LC.L(@"Sets a threshold for when to warn about the backend quota being nearly exceeded. It is given as a percentage, and a warning is generated if the amount of available quota is less than this percentage of the total backup size. If the backend does not report the quota information, this value will be ignored"); } } public static string QuotaWarningThresholdShort { get { return LC.L(@"Threshold for warning about low quota"); } } public static string QuotaDisableLong(string optionname) { return LC.L(@"Disable the quota reported by the backend. --{0} can still be used to set a manual quota", optionname); } From c9e9b0eeb7f1fb757a75c6769169f2432a8dc2df Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 15:00:14 +0200 Subject: [PATCH 010/238] Scroll to source data path on double click. --- .../scripts/directives/sourceFolderPicker.js | 77 +++++++++++++++++-- .../ngax/templates/sourcefolderpicker.html | 4 +- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index afa88466c1..0436aec68c 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -1,7 +1,7 @@ backupApp.directive('sourceFolderPicker', function() { return { restrict: 'E', - require: ['ngSources', 'ngFilters'], + require: ['ngSources', 'ngFilters', '$anchorScroll'], scope: { ngSources: '=', ngFilters: '=', @@ -9,13 +9,14 @@ backupApp.directive('sourceFolderPicker', function() { }, templateUrl: 'templates/sourcefolderpicker.html', - controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog) { + controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog, $anchorScroll) { var scope = $scope; scope.systeminfo = SystemInfo.watch($scope); var sourceNodeChildren = null; - $scope.treedata = { }; + $scope.treedata = {}; + $scope.expandedPath = null; var sourcemap = {}; var excludemap = {}; @@ -331,8 +332,13 @@ backupApp.directive('sourceFolderPicker', function() { } }; + function shouldExpand(path, expandedPath) { + return expandedPath.indexOf(path) == 0 && path.length < expandedPath.length; + } + $scope.toggleExpanded = function(node) { node.expanded = !node.expanded; + self = this; if (node.root || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' || node.iconCls == 'x-tree-icon-hyperv' || node.iconCls == 'x-tree-icon-hypervmachine' @@ -347,8 +353,19 @@ backupApp.directive('sourceFolderPicker', function() { node.loading = false; if (node.children != null) - for(var i in node.children) - setEntryType(node.children[i]); + for (var i in node.children) { + var child = node.children[i]; + setEntryType(child); + if (self.expandedPath != null) { + var childPath = compareablePath(child.id); + if (shouldExpand(childPath, self.expandedPath)) { + self.toggleExpanded(child); + } else if (childPath == self.expandedPath) { + self.expandedPath = null; + self.scrollId = child.id; + } + } + } updateIncludeFlags(node, node.include); @@ -360,6 +377,49 @@ backupApp.directive('sourceFolderPicker', function() { } }; + $scope.expandPath = function(path) { + cPath = compareablePath(path); + this.expandedPath = cPath; + traversenodes(function (n, p) { + if (n.root) { + return null; + } + var nodePath = compareablePath(n.id); + if (nodePath == cPath && !n.other) { + // Scroll to node + scope.scrollId = n.id; + // Cancel traverse + return false; + } + + if (shouldExpand(nodePath, cPath)) { + if (!p.expanded) { + // Handle root nodes + scope.toggleExpanded(p); + } + if (!n.expanded) { + scope.toggleExpanded(n); + } + // Continue traverse + return null; + } else { + // Do not continue this subtree + return true; + } + }, this.treedata); + } + + $scope.$watch('scrollId', function (scrollId, oldVal, scope) { + // Scroll to node + if (scrollId != null) { + scope.scrollId = null; + // Need to wait until all nodes are processed + $timeout(function () { + $anchorScroll('node-' + scrollId); + }, 100); + } + }); + $scope.toggleSelected = function(node) { if (scope.selectednode != null) scope.selectednode.selected = false; @@ -368,6 +428,13 @@ backupApp.directive('sourceFolderPicker', function() { scope.selectednode.selected = true; }; + $scope.doubleClick = function (node) { + if (sourceNodeChildren.indexOf(node) != -1) { + // Open folder in file picker + scope.expandPath(node.id); + } + }; + scope.treedata.children = []; AppService.post('/filesystem?onlyfolders=false&showhidden=true', {path: '/'}).then(function(data) { diff --git a/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html b/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html index 5ea095042a..f5926b6637 100644 --- a/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html +++ b/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html @@ -1,11 +1,11 @@
    -
  • +
  • - {{ node.text }} + {{ node.text }}
      From 9dbdbb1b2fffe9fde686c3af86d85cf001d3d9a2 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:08:03 +0200 Subject: [PATCH 011/238] Show files excluded via 'Exclude' section in source picker. Add more metadata to the file list returned by server, so that the source file picker can display filters based on file attributes. Correctly display 'X' in file list if a file is excluded by attributes and not included as a source (#4751). Fix directory separator issue when loading the page and prevent opening folders in source file list. --- Duplicati/Server/Serializable/TreeNode.cs | 12 ++ .../WebServer/RESTMethods/Filesystem.cs | 27 +++- .../controllers/EditBackupController.js | 18 ++- .../scripts/directives/sourceFolderPicker.js | 125 ++++++++++++++---- .../webroot/ngax/scripts/services/AppUtils.js | 17 ++- .../webroot/ngax/templates/addoredit.html | 4 +- 6 files changed, 163 insertions(+), 40 deletions(-) diff --git a/Duplicati/Server/Serializable/TreeNode.cs b/Duplicati/Server/Serializable/TreeNode.cs index 39c49d05e2..cf8d3c8e8f 100644 --- a/Duplicati/Server/Serializable/TreeNode.cs +++ b/Duplicati/Server/Serializable/TreeNode.cs @@ -43,9 +43,21 @@ public class TreeNode ///
public bool hidden { get; set; } /// + /// True if the element has the system file attribute + /// + public bool systemFile { get; set; } + /// + /// True if the element is marked as temporary + /// + public bool temporary { get; set; } + /// /// True if the element is a symlink /// public bool symlink { get; set; } + /// + /// Size of the file. -1 if directory or inaccessible + /// + public long fileSize { get; set; } /// /// Constructs a new TreeNode diff --git a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs index 3c2b25aedd..5c26570e5e 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs @@ -19,9 +19,9 @@ using System.Linq; using System.IO; using Duplicati.Library.Snapshots; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Common; - +using Duplicati.Library.Common.IO; +using Duplicati.Library.Common; + namespace Duplicati.Server.WebServer.RESTMethods { public class Filesystem : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented @@ -130,7 +130,8 @@ private void Process(string command, string path, RequestInfo info) if (specialtoken != null) { - res = res.Select(x => { + res = res.Select(x => + { x.resolvedpath = x.id; x.id = specialtoken + x.id.Substring(specialpath.Length); return x; @@ -208,6 +209,13 @@ private static Serializable.TreeNode TryCreateTreeNodeForDrive(DriveInfo driveIn return false; }; + Func getFileSize = (p) => + { + try { return new FileInfo(p).Length; } + catch { } + return -1; + }; + foreach (var s in SystemIO.IO_OS.EnumerateFileSystemEntries(entrypath) // Group directories first .OrderByDescending(f => SystemIO.IO_OS.GetFileAttributes(f) & FileAttributes.Directory) @@ -222,6 +230,9 @@ private static Serializable.TreeNode TryCreateTreeNodeForDrive(DriveInfo driveIn var isFolder = (attr & FileAttributes.Directory) != 0; var isFile = !isFolder; var isHidden = (attr & FileAttributes.Hidden) != 0; + bool isSystem = (attr & FileAttributes.System) != 0; + bool isTemporary = (attr & FileAttributes.Temporary) != 0; + long fileSize = -1; var accessible = isFile || canAccess(s); var isLeaf = isFile || !accessible || isEmptyFolder(s); @@ -233,12 +244,20 @@ private static Serializable.TreeNode TryCreateTreeNodeForDrive(DriveInfo driveIn if (!showHidden && isHidden) continue; + if (isFile) + { + fileSize = getFileSize(s); + } + tn = new Serializable.TreeNode() { id = rawid, text = SystemIO.IO_OS.PathGetFileName(s), hidden = isHidden, symlink = isSymlink, + temporary = isTemporary, + systemFile = isSystem, + fileSize = fileSize, iconCls = isFolder ? (accessible ? (isSymlink ? "x-tree-icon-symlink" : "x-tree-icon-parent") : "x-tree-icon-locked") : "x-tree-icon-leaf", leaf = isLeaf }; diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js index 32b61c0ef2..0d14a25f54 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js @@ -14,6 +14,7 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.ExcludeAttributes = []; $scope.ExcludeLargeFiles = false; + $scope.ExcludeFileSize = null; $scope.fileAttributes = [ {'name': gettextCatalog.getString('Hidden files'), 'value': 'hidden'}, @@ -676,10 +677,19 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.$watch("Options['--compression-module']", reloadOptionsList); $scope.$watch("Backup.TargetURL", reloadOptionsList); $scope.$on('systeminfochanged', reloadOptionsList); - $scope.$watch('ExcludeLargeFiles', function() { - if ($scope.Options != null && $scope.Options['--skip-files-larger-than'] == null) - $scope.Options['--skip-files-larger-than'] = '100MB'; - }); + $scope.$watch('ExcludeLargeFiles', function(enabled) { + if ($scope.Options != null) { + if ($scope.Options['--skip-files-larger-than'] == null) { + $scope.Options['--skip-files-larger-than'] = '100MB'; + } + $scope.ExcludeFileSize = enabled ? AppUtils.parseSizeString($scope.Options['--skip-files-larger-than']) : null; + } + }, true); + $scope.$watch("Options['--skip-files-larger-than']", function (value) { + if ($scope.ExcludeLargeFiles) { + $scope.ExcludeFileSize = AppUtils.parseSizeString(value); + } + }, true); $scope.$watch("Schedule.AllowedDays", checkAllowedDaysConfig, true); if ($routeParams.backupid == null) { diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index 0436aec68c..ddfb50afa3 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -5,11 +5,13 @@ backupApp.directive('sourceFolderPicker', function() { scope: { ngSources: '=', ngFilters: '=', - ngShowHidden: '=' + ngShowHidden: '=', + ngExcludeAttributes: '=', + ngExcludeSize: '=' }, templateUrl: 'templates/sourcefolderpicker.html', - controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog, $anchorScroll) { + controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, DialogService, gettextCatalog, $anchorScroll) { var scope = $scope; scope.systeminfo = SystemInfo.watch($scope); @@ -25,11 +27,11 @@ backupApp.directive('sourceFolderPicker', function() { var filterList = null; var displayMap = {}; - var dirsep = scope.systeminfo.DirectorySeparator || '/'; + scope.dirsep = null; function compareablePath(path) { if (path.substr(0, 1) == '%' && path.substr(path.length - 1, 1) == '%') - path += dirsep; + path += scope.dirsep; return scope.systeminfo.CaseSensitiveFilesystem ? path : path.toLowerCase(); } @@ -66,7 +68,7 @@ backupApp.directive('sourceFolderPicker', function() { n.iconCls = 'x-tree-icon-mssql'; else if (defunctmap[cp]) n.iconCls = 'x-tree-icon-broken'; - else if (cp.substr(cp.length - 1, 1) != dirsep) + else if (cp.substr(cp.length - 1, 1) != scope.dirsep) n.iconCls = 'x-tree-icon-leaf'; setEntryType(n); @@ -118,20 +120,18 @@ backupApp.directive('sourceFolderPicker', function() { } function buildidlookup(sources, map) { - var dirsep = scope.systeminfo.DirectorySeparator || '/'; - map = map || {}; for(var n in sources) { - var parts = compareablePath(n).split(dirsep); + var parts = compareablePath(n).split(scope.dirsep); var p = []; for(var pi in parts) { p.push(parts[pi]); - var r = p.join(dirsep); + var r = p.join(scope.dirsep); var l = r.substr(r.length - 1, 1); - if (l != dirsep) - r += dirsep; + if (l != scope.dirsep) + r += scope.dirsep; map[r] = true; } } @@ -139,6 +139,64 @@ backupApp.directive('sourceFolderPicker', function() { return map; } + function nodeExcludedByAttributes(n) { + // Check ExcludeAttributes + if (scope.ngExcludeAttributes != null && scope.ngExcludeAttributes.length > 0) { + if (scope.ngExcludeAttributes.indexOf('hidden') != -1 && n.hidden) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('system') != -1 && n.systemFile) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('temporary') != -1 && n.temporary) { + return true; + } + } + return false + } + + function nodeExcludedBySize(n) { + if(scope.ngExcludeSize != null) { + return n.fileSize > scope.ngExcludeSize; + } + } + + function shouldIncludeNode(n, checkSize) { + if (checkSize === undefined) { + checkSize = true; + } + + // ExcludeSize overrides source paths + if (checkSize && nodeExcludedBySize(n)) { + return false; + } + + // Check if explicitly included in sources + if (sourcemap[compareablePath(n.id)]) { + return true; + } + + // Result is true if included, false if excluded, null if none match + var result = null; + + // Check filter expression + if (filterList == null) + result = excludemap[compareablePath(n.id)] ? false : null; + else { + result = AppUtils.evalFilter(n.id, filterList, null); + } + + if (result !== false && nodeExcludedByAttributes(n)) { + result = false; + } + + if (result === null) { + // Include by default + result = true; + } + return result; + } + function updateIncludeFlags(root, parentFlag) { if (root != null) root = {children: [root], include: parentFlag}; @@ -147,13 +205,10 @@ backupApp.directive('sourceFolderPicker', function() { if (n.root) return null; - if (sourcemap[compareablePath(n.id)]) + if (sourcemap[compareablePath(n.id)] && !nodeExcludedBySize(n)) n.include = '+'; else if (p != null && p.include == '+') { - if (filterList == null) - n.include = excludemap[compareablePath(n.id)] ? '-' : '+'; - else - n.include = AppUtils.evalFilter(n.id, filterList) ? '+' : '-'; + n.include = shouldIncludeNode(n) ? '+' : '-'; } else if (p != null && p.include == '-') n.include = '-'; @@ -176,15 +231,15 @@ backupApp.directive('sourceFolderPicker', function() { var anySpecials = false; - for(var i = 0; i < (scope.ngFilters || []).length; i++) { - var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + for (var i = 0; i < (scope.ngFilters || []).length; i++) { + var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (f != null) { - if (f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) + if (f[0].indexOf('+') == 0 || f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) anySpecials = true; else if (f[0] == '-path') excludemap[compareablePath(f[1])] = true; else if (f[0] == '-folder') - excludemap[compareablePath(f[1] + dirsep)] = true; + excludemap[compareablePath(f[1] + scope.dirsep)] = true; else anySpecials = true; } @@ -192,6 +247,14 @@ backupApp.directive('sourceFolderPicker', function() { if (anySpecials) filterList = AppUtils.filterListToRegexps(scope.ngFilters, scope.systeminfo.CaseSensitiveFilesystem); + } + + function syncTreeWithLists() { + if (scope.ngSources == null || sourceNodeChildren == null || scope.dirsep == null) + return; + + sourcemap = {}; + updateFilterList(); sourceNodeChildren.length = 0; @@ -236,7 +299,7 @@ backupApp.directive('sourceFolderPicker', function() { var p = scope.ngSources[i]; if (p.substr(0, 1) == '%' && p.substr(p.length - 1, 1) == '%') - p += dirsep; + p += scope.dirsep; AppService.post('/filesystem/validate', {path: p}).then(function(data) { defunctmap[compareablePath(data.config.data.path)] = false; @@ -259,6 +322,14 @@ backupApp.directive('sourceFolderPicker', function() { $scope.$watch('ngSources', syncTreeWithLists, true); $scope.$watch('ngFilters', syncTreeWithLists, true); + $scope.$watch('ngExcludeAttributes', syncTreeWithLists, true); + $scope.$watch('ngExcludeSize', syncTreeWithLists, true); + $scope.$watch('systeminfo.DirectorySeparator', function (val, oldVal) { + if (val != null) { + scope.dirsep = val; + syncTreeWithLists(); + } + }, true); function findParent(id) { var r = {}; @@ -282,10 +353,8 @@ backupApp.directive('sourceFolderPicker', function() { } $scope.toggleCheck = function(node) { - dirsep = scope.systeminfo.DirectorySeparator || '/'; - var c = compareablePath(node.id); - var c_is_dir = c.substr(c.length - 1, 1) == dirsep; + var c_is_dir = c.substr(c.length - 1, 1) == scope.dirsep; if (node.include == null || node.include == ' ') { @@ -296,7 +365,7 @@ backupApp.directive('sourceFolderPicker', function() { if (s == c) return; - if (s.substr(s.length - 1, 1) == dirsep && c.indexOf(s) == 0) { + if (s.substr(s.length - 1, 1) == scope.dirsep && c.indexOf(s) == 0) { return; } else if (s.indexOf(c) == 0) { scope.ngSources.splice(i, 1); @@ -311,11 +380,11 @@ backupApp.directive('sourceFolderPicker', function() { removePathFromArray(scope.ngSources, node.id); for(var i = scope.ngFilters.length - 1; i >= 0; i--) { - var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (n != null) { if (c_is_dir) { if (n[0] == '-path' || n[0] == '-folder') { - if (compareablePath(n[1] + (n[0] == '-folder' ? dirsep : '')).indexOf(c) == 0) + if (compareablePath(n[1] + (n[0] == '-folder' ? scope.dirsep : '')).indexOf(c) == 0) scope.ngFilters.splice(i, 1); } } else { @@ -340,7 +409,7 @@ backupApp.directive('sourceFolderPicker', function() { node.expanded = !node.expanded; self = this; - if (node.root || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' + if (node.root || node.leaf || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' || node.iconCls == 'x-tree-icon-hyperv' || node.iconCls == 'x-tree-icon-hypervmachine' || node.iconCls == 'x-tree-icon-mssql' || node.iconCls == 'x-tree-icon-mssqldb') return; diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index ed76c72a1e..47bd07a29f 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -228,14 +228,27 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer }; - this.splitSizeString = function(val) { + this.splitSizeString = function (val) { var m = (/(\d*)(\w*)/mg).exec(val); var mul = null; if (!m) return [parseInt(val), null]; else return [parseInt(m[1]), m[2]]; - } + }; + + this.parseSizeString = function (val) { + if (val == null) { + return null; + } + var split = this.splitSizeString(val); + var formatSizes = ['B', 'KB', 'MB', 'GB', 'TB']; + var idx = formatSizes.indexOf((split[1]||'').toUpperCase()); + if (idx == -1) { + idx = 0; + } + return split[0] * Math.pow(1024, idx); + }; this.toDisplayDateAndTime = function(dt) { diff --git a/Duplicati/Server/webroot/ngax/templates/addoredit.html b/Duplicati/Server/webroot/ngax/templates/addoredit.html index 084c602f0d..cdaa981c35 100644 --- a/Duplicati/Server/webroot/ngax/templates/addoredit.html +++ b/Duplicati/Server/webroot/ngax/templates/addoredit.html @@ -143,7 +143,7 @@

Source data

{{'Show hidden folders' | translate}}
- +
@@ -237,7 +237,7 @@

Exclude

-
From 5acf4dbaad72bc1dbf36ef0817e2dc1c0e947204 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:50:21 +0200 Subject: [PATCH 012/238] Improve file in-/excluding logic in source file picker. Add filters to the beginning instead of the end of the list, so they are not overridden by more general ones. If a file is excluded, add an include filter to include the file. --- .../scripts/directives/sourceFolderPicker.js | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index ddfb50afa3..effac4940a 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -219,16 +219,9 @@ backupApp.directive('sourceFolderPicker', function() { }, root); } - function syncTreeWithLists() { - if (scope.ngSources == null || sourceNodeChildren == null) - return; - - dirsep = scope.systeminfo.DirectorySeparator || '/'; - - sourcemap = {}; + function updateFilterList() { excludemap = {}; filterList = null; - var anySpecials = false; for (var i = 0; i < (scope.ngFilters || []).length; i++) { @@ -394,10 +387,28 @@ backupApp.directive('sourceFolderPicker', function() { } } } else { - scope.ngFilters.push("-" + node.id); + removePathFromArray(scope.ngFilters, '+' + node.id); + updateFilterList(); + if (shouldIncludeNode(node, false)) { + // No explicit include filter, add exclude filter to start of list + scope.ngFilters.unshift("-" + node.id); + } } } else if (node.include == '-') { removePathFromArray(scope.ngFilters, '-' + node.id); + updateFilterList(); + if (nodeExcludedByAttributes(node) + && indexOfPathInArray(scope.ngSources, node.id) == -1) { + // Node is excluded by attributes, have to add as source to override + scope.ngSources.push(node.id); + } else if (!shouldIncludeNode(node, false)) { + // No explicit exclude filter, add include filter to start of list + scope.ngFilters.unshift('+' + node.id); + } + if (nodeExcludedBySize(node)) { + DialogService.dialog(gettextCatalog.getString('Cannot include "{{text}}"', node), + gettextCatalog.getString('The file size is {{size}}, larger than the maximum specified size. If the file size decreases, it will be included in future backups.', { size: AppUtils.formatSizeString(node.fileSize) })); + } } }; From 74db90924ef0180a22ea7c8d79cd3b96599223f9 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 21:49:32 +0200 Subject: [PATCH 013/238] Fix processing files repeatedly if present in multiple sources. This can happen if a source is added to force including a system folder. --- .../Library/Main/Operation/Backup/FileEnumerationProcess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index e816fbbabd..993776d254 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -105,7 +105,7 @@ public static Task Run(IEnumerable sources, Snapshots.ISnapshotService s worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) => { Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath); - }); + }).Distinct(Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); } if (token.IsCancellationRequested) From a1349b1af9e5a566a8c97998ae28cdeb817d10e9 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 25 Jun 2023 00:13:05 +0200 Subject: [PATCH 014/238] Show filter group rules in source file picker. Add REST handler /systeminfo/filtergroups that returns the filter strings for all filter groups on the system. Use these filter strings to apply filter groups to the source file picker. --- Duplicati/Library/Utility/FilterGroups.cs | 17 +++++++ .../WebServer/RESTMethods/SystemInfo.cs | 29 ++++++++--- .../scripts/directives/sourceFolderPicker.js | 3 ++ .../webroot/ngax/scripts/services/AppUtils.js | 49 ++++++++++++++++--- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Utility/FilterGroups.cs b/Duplicati/Library/Utility/FilterGroups.cs index e59255453e..fdfd85fdb6 100644 --- a/Duplicati/Library/Utility/FilterGroups.cs +++ b/Duplicati/Library/Utility/FilterGroups.cs @@ -249,6 +249,23 @@ public static IEnumerable GetFilterStrings(FilterGroup group) .Distinct(Utility.ClientFilenameStringComparer); } + public static Dictionary> GetFilterStringMap() + { + // Flag messes with ToString(), so have to do this manually + Dictionary> filterMap = new Dictionary>() + { + { nameof(FilterGroup.None), GetFilterStrings(FilterGroup.None).ToList() }, + { nameof(FilterGroup.SystemFiles), GetFilterStrings(FilterGroup.SystemFiles).ToList() }, + { nameof(FilterGroup.OperatingSystem), GetFilterStrings(FilterGroup.OperatingSystem).ToList() }, + { nameof(FilterGroup.CacheFiles), GetFilterStrings(FilterGroup.CacheFiles).ToList() }, + { nameof(FilterGroup.TemporaryFiles), GetFilterStrings(FilterGroup.TemporaryFiles).ToList() }, + { nameof(FilterGroup.Applications), GetFilterStrings(FilterGroup.Applications).ToList() }, + { nameof(FilterGroup.DefaultExcludes), GetFilterStrings(FilterGroup.DefaultExcludes).ToList() }, + { nameof(FilterGroup.DefaultIncludes), GetFilterStrings(FilterGroup.DefaultIncludes).ToList() } + }; + return filterMap; + } + /// /// Filters all items that have a prefix /// diff --git a/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs b/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs index 3932e895a6..bb322288af 100644 --- a/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs +++ b/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs @@ -26,7 +26,7 @@ public class SystemInfo : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented { public string Description { get { return "Gets various system properties"; } } public IEnumerable> Types - { + { get { return new KeyValuePair[] { @@ -37,9 +37,19 @@ public IEnumerable> Types public void GET(string key, RequestInfo info) { - info.BodyWriter.OutputOK(SystemData(info)); + if (string.IsNullOrWhiteSpace(key)) + { + info.BodyWriter.OutputOK(SystemData(info)); + } + else if (key.Equals("filtergroups", StringComparison.OrdinalIgnoreCase)) + { + info.BodyWriter.OutputOK(FilterGroups()); + } + else + { + info.OutputError(code: System.Net.HttpStatusCode.NotFound, reason: "Not found"); + } } - public void POST(string key, RequestInfo info) { switch ((key ?? "").ToLowerInvariant()) @@ -110,17 +120,22 @@ private static object SystemData(RequestInfo info) EnglishName = browserlanguage.EnglishName, DisplayName = browserlanguage.NativeName }, - SupportedLocales = + SupportedLocales = Library.Localization.LocalizationService.SupportedCultures - .Select(x => new { - Code = x, + .Select(x => new { + Code = x, EnglishName = new System.Globalization.CultureInfo(x).EnglishName, DisplayName = new System.Globalization.CultureInfo(x).NativeName - } + } ), BrowserLocaleSupported = Library.Localization.LocalizationService.isCultureSupported(browserlanguage) }; } + + private static object FilterGroups() + { + return new { FilterGroups = Library.Utility.FilterGroups.GetFilterStringMap() }; + } } } diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index effac4940a..52d405a849 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -517,6 +517,9 @@ backupApp.directive('sourceFolderPicker', function() { scope.treedata.children = []; + // Load filter groups + AppUtils.loadFilterGroups(); + AppService.post('/filesystem?onlyfolders=false&showhidden=true', {path: '/'}).then(function(data) { var usernode = { diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index 47bd07a29f..0f96817586 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -1,4 +1,4 @@ -backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogService, gettextCatalog) { +backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, AppService, DialogService, gettextCatalog) { var apputils = this; @@ -214,6 +214,17 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer $rootScope.$broadcast('apputillookupschanged'); }; + apputils.filterGroupMap = null; + apputils.loadFilterGroups = function (reload) { + if (reload || apputils.filterGroupMap === null) { + AppService.get('/systeminfo/filtergroups').then(function (data) { + apputils.filterGroupMap = angular.copy(data.data.FilterGroups); + + $rootScope.$broadcast('apputillookupschanged'); + }, apputils.connectionError); + } + } + reloadTexts(); $rootScope.$on('gettextLanguageChanged', reloadTexts); @@ -644,6 +655,34 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer return pre + body + suf; }; + this.filterToRegexpStr = function (filter) { + var firstChar = filter.substr(0, 1); + var lastChar = filter.substr(filter.length - 1, 1); + var isFilterGroup = firstChar == '{' && lastChar == '}'; + if (isFilterGroup && this.filterGroupMap !== null) { + // Replace filter groups with filter strings + var filterGroups = filter.substr(1,filter.length - 2).split(','); + var filterStrings = []; + for (var i = 0; i < filterGroups.length; ++i) { + filterStrings = filterStrings.concat(this.filterGroupMap[filterGroups[i].trim()] || []); + } + + filter = filterStrings.map(function (s) { + return '(?:' + apputils.filterToRegexpStr(s) + ')' + }).join('|'); + + } else { + var rx = firstChar == '[' && lastChar == ']'; + if (rx) + filter = filter.substr(1, filter.length - 2); + else { + filter = this.globToRegexp(filter); + } + } + + return filter; + }; + this.filterListToRegexps = function(filters, caseSensitive) { var res = []; @@ -655,14 +694,10 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer var flag = f.substr(0, 1); var filter = f.substr(1); - var rx = filter.substr(0, 1) == '[' && filter.substr(filter.length - 1, 1) == ']'; - if (rx) - filter = filter.substr(1, filter.length - 2); - else - filter = this.globToRegexp(filter); try { - res.push([flag == '+', new RegExp(filter, caseSensitive ? 'g' : 'gi')]); + var regexp = this.filterToRegexpStr(filter); + res.push([flag == '+', new RegExp(regexp, caseSensitive ? 'g' : 'gi')]); } catch (e) { } } From 32e852a392ee094a3f680e7a213000d8b6bc4b00 Mon Sep 17 00:00:00 2001 From: natan Date: Fri, 15 Mar 2024 16:51:01 +0100 Subject: [PATCH 015/238] WIP --- .../LicenseUpdater/LicenseUpdater.csproj | 22 +-- .../Abstractions/IScheduler.cs | 45 +++++ .../Abstractions/IWorkerThreadsManager.cs | 15 ++ .../Database/Connection.cs | 14 +- .../Database/ServerSettings.cs | 6 +- .../Duplicati.Library.RestAPI.csproj | 7 + Duplicati.Library.RestAPI/FIXMEGlobal.cs | 38 +++-- Duplicati.Library.RestAPI/LiveControls.cs | 18 +- .../NotificationUpdateService.cs | 50 ++++++ .../RESTMethods/Backups.cs | 2 +- .../RESTMethods/RequestInfo.cs | 4 +- .../RESTMethods/ServerState.cs | 7 +- Duplicati.Library.RestAPI/Runner.cs | 4 +- Duplicati.Library.RestAPI/Scheduler.cs | 135 +++++++++------ .../Serializable/ServerStatus.cs | 5 +- Duplicati.Library.RestAPI/UpdatePollThread.cs | 16 +- .../WorkerThreadsManager.cs | 34 ++++ .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 2 +- .../AutoUpdater/IUpdateManagerAccessor.cs | 6 + .../AutoUpdater/UpdateManagerAccessor.cs | 6 + Duplicati/Library/Common/ILiveControls.cs | 6 + .../Utility/Abstractions/IBoolParser.cs | 6 + Duplicati/Library/Utility/BoolParser.cs | 11 ++ Duplicati/Library/Utility/WorkerThread.cs | 42 ++--- .../Serializer.cs | 70 ++++---- Duplicati/Server/Program.cs | 133 ++++++--------- Duplicati/UnitTest/ImportExportTests.cs | 4 +- .../WebserverCore/Abstractions/FileEntry.cs | 10 ++ .../Abstractions/ISettingsService.cs | 6 + .../Abstractions/IStatusService.cs | 8 + .../Abstractions/IUpdateService.cs | 6 + .../WebserverCore/Abstractions/IV1Endpoint.cs | 6 + .../Abstractions/ServerSettings.cs | 29 ++++ .../WebserverCore/Abstractions/UpdateInfo.cs | 17 ++ .../WebserverCore/ApplicationPartsLogger.cs | 32 ++++ .../Configuration/OptionConfiguration.cs | 14 ++ .../WebserverCore/Database/Entities/Option.cs | 9 + .../WebserverCore/Database/MainDbContext.cs | 14 ++ Duplicati/WebserverCore/Dto/ServerStatus.cs | 24 +++ .../Duplicati.WebserverCore.csproj | 5 + Duplicati/WebserverCore/DuplicatiWebserver.cs | 159 ++++++++++-------- .../Endpoints/ServerStateIv1Endpoint.cs | 11 ++ .../ServiceCollectionsExtensions.cs | 41 +++++ .../Extensions/WebApplicationExtensions.cs | 37 ++++ .../WebserverCore/LegacyHttpRequestShim.cs | 12 +- .../WebserverCore/LegacyHttpResponseShim.cs | 7 +- .../WebserverCore/LegacyHttpSessionShim.cs | 8 +- .../Middlewares/StaticFilesMiddleware.cs | 28 +++ .../Services/Settings/SettingsService.cs | 110 ++++++++++++ .../WebserverCore/Services/StatusService.cs | 92 ++++++++++ .../WebserverCore/Services/UpdateService.cs | 29 ++++ .../Services/UpdateServiceLogger.cs | 11 ++ .../WebserverCore/TemporaryIoCAccessor.cs | 6 + .../appsettings.Development.json | 8 + Duplicati/WebserverCore/appsettings.json | 28 +++ 55 files changed, 1140 insertions(+), 335 deletions(-) create mode 100644 Duplicati.Library.RestAPI/Abstractions/IScheduler.cs create mode 100644 Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs create mode 100644 Duplicati.Library.RestAPI/NotificationUpdateService.cs create mode 100644 Duplicati.Library.RestAPI/WorkerThreadsManager.cs create mode 100644 Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs create mode 100644 Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs create mode 100644 Duplicati/Library/Common/ILiveControls.cs create mode 100644 Duplicati/Library/Utility/Abstractions/IBoolParser.cs create mode 100644 Duplicati/Library/Utility/BoolParser.cs create mode 100644 Duplicati/WebserverCore/Abstractions/FileEntry.cs create mode 100644 Duplicati/WebserverCore/Abstractions/ISettingsService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IStatusService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IUpdateService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs create mode 100644 Duplicati/WebserverCore/Abstractions/ServerSettings.cs create mode 100644 Duplicati/WebserverCore/Abstractions/UpdateInfo.cs create mode 100644 Duplicati/WebserverCore/ApplicationPartsLogger.cs create mode 100644 Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs create mode 100644 Duplicati/WebserverCore/Database/Entities/Option.cs create mode 100644 Duplicati/WebserverCore/Database/MainDbContext.cs create mode 100644 Duplicati/WebserverCore/Dto/ServerStatus.cs create mode 100644 Duplicati/WebserverCore/Endpoints/ServerStateIv1Endpoint.cs create mode 100644 Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs create mode 100644 Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs create mode 100644 Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs create mode 100644 Duplicati/WebserverCore/Services/Settings/SettingsService.cs create mode 100644 Duplicati/WebserverCore/Services/StatusService.cs create mode 100644 Duplicati/WebserverCore/Services/UpdateService.cs create mode 100644 Duplicati/WebserverCore/Services/UpdateServiceLogger.cs create mode 100644 Duplicati/WebserverCore/TemporaryIoCAccessor.cs create mode 100644 Duplicati/WebserverCore/appsettings.Development.json create mode 100644 Duplicati/WebserverCore/appsettings.json diff --git a/BuildTools/LicenseUpdater/LicenseUpdater.csproj b/BuildTools/LicenseUpdater/LicenseUpdater.csproj index a44450dc46..551de1203b 100755 --- a/BuildTools/LicenseUpdater/LicenseUpdater.csproj +++ b/BuildTools/LicenseUpdater/LicenseUpdater.csproj @@ -1,11 +1,11 @@ - - - - Exe - net8.0 - license_upgrader - enable - enable - - - + + + + Exe + net8.0 + license_upgrader + enable + enable + + + diff --git a/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs b/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs new file mode 100644 index 0000000000..0d7016e78c --- /dev/null +++ b/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.Server.Serialization.Interface; + +namespace Duplicati.WebserverCore.Abstractions; + +public interface IScheduler +{ + /// + /// Initializes scheduler + /// + /// The worker thread + void Init(WorkerThread worker); + + IList> GetSchedulerQueueIds(); + + /// + /// Terminates the thread. Any items still in queue will be removed + /// + /// True if the call should block until the thread has exited, false otherwise + void Terminate(bool wait); + + /// + /// An event that is raised when the schedule changes + /// + event EventHandler NewSchedule; + + /// + /// A snapshot copy of the current schedule list + /// + List> Schedule { get; } + + /// + /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution + /// + List WorkerQueue { get; } + + /// + /// Forces the scheduler to re-evaluate the order. + /// Call this method if something changes + /// + void Reschedule(); +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs b/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs new file mode 100644 index 0000000000..6e4aa3b15c --- /dev/null +++ b/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs @@ -0,0 +1,15 @@ +#nullable enable +using System; +using Duplicati.Library.Utility; +using Duplicati.Server; + +namespace Duplicati.Library.RestAPI.Abstractions; + +public interface IWorkerThreadsManager +{ + void Spawn(Action item); + + Tuple? CurrentTask { get; } + WorkerThread? WorkerThread { get; } + void UpdateThrottleSpeeds(); +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Database/Connection.cs b/Duplicati.Library.RestAPI/Database/Connection.cs index 4fd7b1f90d..fb35401d76 100644 --- a/Duplicati.Library.RestAPI/Database/Connection.cs +++ b/Duplicati.Library.RestAPI/Database/Connection.cs @@ -496,7 +496,7 @@ internal void UpdateBackupDBPath(IBackup item, string path) } } - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -598,7 +598,7 @@ private void AddOrUpdateBackup(IBackup item, bool updateSchedule, ISchedule sche } tr.Commit(); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } @@ -611,7 +611,7 @@ internal void AddOrUpdateSchedule(ISchedule item) { AddOrUpdateSchedule(item, tr); tr.Commit(); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } @@ -674,7 +674,7 @@ public void DeleteBackup(long ID) } } - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -694,7 +694,7 @@ public void DeleteSchedule(long ID) lock(m_lock) DeleteFromDb("Schedule", ID); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -782,7 +782,7 @@ public bool DismissNotification(long id) FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); } - FIXMEGlobal.IncrementLastNotificationUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); return true; @@ -822,7 +822,7 @@ public void RegisterNotification(Serialization.NotificationType type, string tit FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = true; } - FIXMEGlobal.IncrementLastNotificationUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } diff --git a/Duplicati.Library.RestAPI/Database/ServerSettings.cs b/Duplicati.Library.RestAPI/Database/ServerSettings.cs index 92b02bcfab..7f6fe21947 100644 --- a/Duplicati.Library.RestAPI/Database/ServerSettings.cs +++ b/Duplicati.Library.RestAPI/Database/ServerSettings.cs @@ -27,7 +27,7 @@ namespace Duplicati.Server.Database { public class ServerSettings { - private static class CONST + public static class CONST { public const string STARTUP_DELAY = "startup-delay"; public const string DOWNLOAD_SPEED_LIMIT = "max-download-speed"; @@ -113,13 +113,13 @@ from n in settings Value = n.Value }, Database.Connection.SERVER_SETTINGS_ID); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); // In case the usage reporter is enabled or disabled, refresh now FIXMEGlobal.StartOrStopUsageReporter(); // If throttle options were changed, update now - FIXMEGlobal.UpdateThrottleSpeeds(); + FIXMEGlobal.WorkerThreadsManager.UpdateThrottleSpeeds(); } public string StartupDelayDuration diff --git a/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj b/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj index e10690aa5d..3e01d4bd58 100644 --- a/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj +++ b/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj @@ -17,6 +17,9 @@ + + ..\Executables\net8\Duplicati.Server\bin\Debug\net8.0\Duplicati.WebserverCore.dll + ..\thirdparty\HttpServer\HttpServer.dll @@ -32,4 +35,8 @@ + + + + diff --git a/Duplicati.Library.RestAPI/FIXMEGlobal.cs b/Duplicati.Library.RestAPI/FIXMEGlobal.cs index 2a8fc31482..3cb688507f 100644 --- a/Duplicati.Library.RestAPI/FIXMEGlobal.cs +++ b/Duplicati.Library.RestAPI/FIXMEGlobal.cs @@ -2,6 +2,11 @@ using Duplicati.Server; using System; using System.Collections.Generic; +using Duplicati.Library.IO; +using Duplicati.Library.RestAPI.Abstractions; +using Duplicati.Library.Utility; +using Duplicati.WebserverCore.Abstractions; +using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Library.RestAPI { @@ -11,7 +16,8 @@ namespace Duplicati.Library.RestAPI */ public static class FIXMEGlobal { - + public static IServiceProvider Provider { get; set; } + /// /// This is the only access to the database /// @@ -20,7 +26,7 @@ public static class FIXMEGlobal /// /// The controller interface for pause/resume and throttle options /// - public static LiveControls LiveControl; + public static LiveControls LiveControl => Provider.GetRequiredService(); /// /// A delegate method for creating a copy of the current progress state @@ -30,24 +36,24 @@ public static class FIXMEGlobal /// /// The status event signaler, used to control long polling of status updates /// - public static readonly EventPollNotify StatusEventNotifyer = new EventPollNotify(); + public static EventPollNotify StatusEventNotifyer => Provider.GetRequiredService(); /// - /// This is the working thread + /// For keeping and incrementing last last events Ids of db save and last notification /// - public static Duplicati.Library.Utility.WorkerThread WorkThread; + public static INotificationUpdateService NotificationUpdateService => Provider.GetRequiredService(); - public static Func PeekLastDataUpdateID; - public static Func PeekLastNotificationUpdateID; - - public static Action IncrementLastDataUpdateID; + /// + /// This is the working thread + /// + public static WorkerThread WorkThread => + Provider.GetRequiredService().WorkerThread; - public static Action IncrementLastNotificationUpdateID; + public static IWorkerThreadsManager WorkerThreadsManager => + Provider.GetRequiredService(); public static Action StartOrStopUsageReporter; - public static Action UpdateThrottleSpeeds; - /// /// Gets the folder where Duplicati data is stored /// @@ -56,19 +62,19 @@ public static class FIXMEGlobal /// /// This is the scheduling thread /// - public static Scheduler Scheduler; + public static IScheduler Scheduler => Provider.GetRequiredService(); /// /// The log redirect handler /// public static readonly LogWriteHandler LogHandler = new LogWriteHandler(); - public static Func, Server.Database.Connection> GetDatabaseConnection; + public static Func, Server.Database.Connection> GetDatabaseConnection; /// /// The update poll thread. /// - public static UpdatePollThread UpdatePoller; + public static UpdatePollThread UpdatePoller => Provider.GetRequiredService(); /// @@ -93,5 +99,7 @@ public static class FIXMEGlobal /// This is the lock to be used before manipulating the shared resources /// public static readonly object MainLock = new object(); + + private static LiveControls _liveControl; } } diff --git a/Duplicati.Library.RestAPI/LiveControls.cs b/Duplicati.Library.RestAPI/LiveControls.cs index 1a51636a3d..c3d0d24d05 100644 --- a/Duplicati.Library.RestAPI/LiveControls.cs +++ b/Duplicati.Library.RestAPI/LiveControls.cs @@ -18,9 +18,8 @@ // #endregion using System; -using System.Collections.Generic; -using System.Text; using Duplicati.Library.Common; +using Duplicati.Library.IO; using Duplicati.Library.RestAPI; namespace Duplicati.Server @@ -29,7 +28,7 @@ namespace Duplicati.Server /// This class keeps track of the users modifications regarding /// throttling and pause/resume /// - public class LiveControls + public class LiveControls : ILiveControls { /// /// The tag used for logging @@ -86,6 +85,8 @@ public enum LiveControlState /// public LiveControlState State { get { return m_state; } } + public bool IsPaused => State == LiveControlState.Paused; + /// /// The internal variable that tracks the the priority /// @@ -160,7 +161,7 @@ public long? DownloadLimit /// /// The timer that is activated after a pause period. /// - private readonly System.Threading.Timer m_waitTimer; + private System.Threading.Timer m_waitTimer; /// /// The time that the current pause is expected to expire @@ -170,7 +171,14 @@ public long? DownloadLimit /// /// Constructs a new instance of the LiveControl /// - public LiveControls(Database.ServerSettings settings) + public LiveControls() + { + } + + /// + /// Constructs a new instance of the LiveControl + /// + public void Init(Database.ServerSettings settings) { m_state = LiveControlState.Running; m_waitTimer = new System.Threading.Timer(m_waitTimer_Tick, this, System.Threading.Timeout.Infinite, System.Threading.Timeout.Infinite); diff --git a/Duplicati.Library.RestAPI/NotificationUpdateService.cs b/Duplicati.Library.RestAPI/NotificationUpdateService.cs new file mode 100644 index 0000000000..2bbec0ce3c --- /dev/null +++ b/Duplicati.Library.RestAPI/NotificationUpdateService.cs @@ -0,0 +1,50 @@ +namespace Duplicati.Library.RestAPI; + +public interface INotificationUpdateService +{ + /// + /// An event ID that increases whenever the database is updated + /// + long LastDataUpdateId { get; } + + /// + /// An event ID that increases whenever a notification is updated + /// + long LastNotificationUpdateId { get; } + + void IncrementLastDataUpdateId(); + void IncrementLastNotificationUpdateId(); +} + +public class NotificationUpdateService : INotificationUpdateService +{ + /// + /// An event ID that increases whenever the database is updated + /// + public long LastDataUpdateId { get; private set; } = 0; + + private readonly object _lastDataUpdateIdLock = new(); + + /// + /// An event ID that increases whenever a notification is updated + /// + public long LastNotificationUpdateId { get; private set; } = 0; + + private readonly object _lastNotificationUpdateIdLock = new(); + + public void IncrementLastDataUpdateId() + { + lock (_lastDataUpdateIdLock) + { + LastDataUpdateId++; + } + } + + public void IncrementLastNotificationUpdateId() + { + lock (_lastNotificationUpdateIdLock) + { + LastNotificationUpdateId++; + } + } +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/RESTMethods/Backups.cs b/Duplicati.Library.RestAPI/RESTMethods/Backups.cs index bae68fa575..2c06f365d7 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/Backups.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/Backups.cs @@ -132,7 +132,7 @@ public static Serializable.ImportExportStructure ImportBackup(string configurati Serializable.ImportExportStructure importedStructure = Backups.LoadConfiguration(configurationFile, importMetadata, getPassword); // This will create the Duplicati-server.sqlite database file if it doesn't exist. - using (Duplicati.Server.Database.Connection connection = FIXMEGlobal.GetDatabaseConnection(advancedOptions)) + using (Duplicati.Server.Database.Connection connection = FIXMEGlobal.GetDatabaseConnection(null, advancedOptions)) { if (connection.Backups.Any(x => x.Name.Equals(importedStructure.Backup.Name, StringComparison.OrdinalIgnoreCase))) { diff --git a/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs b/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs index 2069b6cf93..f0c511ef45 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs @@ -24,8 +24,8 @@ namespace Duplicati.Server.WebServer.RESTMethods { public class RequestInfo : IDisposable { - public HttpServer.IHttpRequest Request { get; private set; } - public HttpServer.IHttpResponse Response { get; private set; } + public HttpServer.IHttpRequest Request { get; } + public HttpServer.IHttpResponse Response { get; } public HttpServer.Sessions.IHttpSession Session { get; private set; } public BodyWriter BodyWriter { get; private set; } public RequestInfo(HttpServer.IHttpRequest request, HttpServer.IHttpResponse response, HttpServer.Sessions.IHttpSession session) diff --git a/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs b/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs index bd1fa914a5..70c382b1d0 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs @@ -24,7 +24,7 @@ namespace Duplicati.Server.WebServer.RESTMethods { - public class ServerState : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented + public class ServerState : IRESTMethodPOST, IRESTMethodDocumented { public void GET(string key, RequestInfo info) { @@ -32,16 +32,17 @@ public void GET(string key, RequestInfo info) long id = 0; long.TryParse(key, out id); + var serverStatus = new Serializable.ServerStatus(); if (info.LongPollCheck(FIXMEGlobal.StatusEventNotifyer, ref id, out isError)) { //Make sure we do not report a higher number than the eventnotifier says - var st = new Serializable.ServerStatus(); + var st = serverStatus; st.LastEventID = id; info.OutputOK(st); } else if (!isError) { - info.OutputOK(new Serializable.ServerStatus()); + info.OutputOK(serverStatus); } } diff --git a/Duplicati.Library.RestAPI/Runner.cs b/Duplicati.Library.RestAPI/Runner.cs index 1dcdc267cc..76554701b5 100644 --- a/Duplicati.Library.RestAPI/Runner.cs +++ b/Duplicati.Library.RestAPI/Runner.cs @@ -694,7 +694,7 @@ private static void UpdateMetadataError(Duplicati.Server.Serialization.Interface if (ex is UserInformationException exception) messageid = exception.HelpID; - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.DataConnection.RegisterNotification( NotificationType.Error, backup.IsTemporary ? @@ -876,7 +876,7 @@ private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBac if (!backup.IsTemporary) FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } diff --git a/Duplicati.Library.RestAPI/Scheduler.cs b/Duplicati.Library.RestAPI/Scheduler.cs index 81a8cdf419..7544bb7c2c 100644 --- a/Duplicati.Library.RestAPI/Scheduler.cs +++ b/Duplicati.Library.RestAPI/Scheduler.cs @@ -1,4 +1,5 @@ #region Disclaimer / License + // Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // @@ -18,8 +19,8 @@ // using Duplicati.Server.Serialization.Interface; - #endregion + using System; using System.Collections.Generic; using System.Text; @@ -27,32 +28,37 @@ using System.Threading; using Duplicati.Library.Utility; using Duplicati.Library.RestAPI; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.Server { /// /// This class handles scheduled runs of backups /// - public class Scheduler + public class Scheduler : IScheduler { private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); /// /// The thread that runs the scheduler /// - private readonly Thread m_thread; + private Thread m_thread; + /// /// A termination flag /// private volatile bool m_terminate; + /// /// The worker thread that is invoked to do work /// - private readonly WorkerThread m_worker; + private WorkerThread m_worker; + /// /// The wait event /// - private readonly AutoResetEvent m_event; + private AutoResetEvent m_event; + /// /// The data synchronization lock /// @@ -67,17 +73,24 @@ public class Scheduler /// The currently scheduled items /// private KeyValuePair[] m_schedule; - + /// /// List of update tasks, used to set the timestamp on the schedule once completed /// - private readonly Dictionary> m_updateTasks; + private Dictionary> m_updateTasks; /// /// Constructs a new scheduler /// + public Scheduler() + { + } + + /// + /// Initializes scheduler + /// /// The worker thread - public Scheduler(WorkerThread worker) + public void Init(WorkerThread worker) { m_thread = new Thread(new ThreadStart(Runner)); m_worker = worker; @@ -92,6 +105,13 @@ public Scheduler(WorkerThread worker) m_thread.Start(); } + public IList> GetSchedulerQueueIds() + { + return (from n in WorkerQueue + where n.Backup != null + select new Tuple(n.TaskID, n.Backup.ID)).ToList(); + } + /// /// Forces the scheduler to re-evaluate the order. /// Call this method if something changes @@ -104,13 +124,13 @@ public void Reschedule() /// /// A snapshot copy of the current schedule list /// - public List> Schedule - { - get + public List> Schedule + { + get { lock (m_lock) return m_schedule.ToList(); - } + } } /// @@ -118,10 +138,7 @@ public List> Schedule /// public List WorkerQueue { - get - { - return (from t in m_worker.CurrentTasks where t != null select t).ToList(); - } + get { return (from t in m_worker.CurrentTasks where t != null select t).ToList(); } } /// @@ -135,8 +152,13 @@ public void Terminate(bool wait) if (wait) { - try { m_thread.Join(); } - catch { } + try + { + m_thread.Join(); + } + catch + { + } } } @@ -148,7 +170,8 @@ public void Terminate(bool wait) /// The repetition interval /// The days the backup is allowed to run /// The next valid date, or throws an exception if no such date can be found - public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, string repetition, DayOfWeek[] allowedDays) + public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, string repetition, + DayOfWeek[] allowedDays) { var res = basetime; @@ -195,26 +218,25 @@ public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, s throw new Exception(Strings.Scheduler.InvalidTimeSetupError(basetime, repetition, sb.ToString())); } - + return res; } - + private void OnCompleted(WorkerThread worker, Runner.IRunnerData task) { Tuple t = null; - lock(m_lock) + lock (m_lock) { if (task != null && m_updateTasks.TryGetValue(task, out t)) m_updateTasks.Remove(task); } - + if (t != null) { t.Item1.Time = t.Item2; t.Item1.LastRun = t.Item3; FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); } - } private void OnStartingWork(WorkerThread worker, Runner.IRunnerData task) @@ -223,8 +245,8 @@ private void OnStartingWork(WorkerThread worker, Runner.IRun { return; } - - lock(m_lock) + + lock (m_lock) { if (m_updateTasks.TryGetValue(task, out Tuple scheduleInfo)) { @@ -245,10 +267,10 @@ private void Runner() { //TODO: As this is executed repeatedly we should cache it // to avoid frequent db lookups - + //Determine schedule list var lst = FIXMEGlobal.DataConnection.Schedules; - foreach(var sc in lst) + foreach (var sc in lst) { if (!string.IsNullOrEmpty(sc.Repeat)) { @@ -267,7 +289,7 @@ private void Runner() { start = startkey.Value; } - + try { // Recover from timedrift issues by overriding the dates if the last run date is in the future. @@ -276,11 +298,13 @@ private void Runner() start = DateTime.UtcNow; last = DateTime.UtcNow; } + start = GetNextValidTime(start, last, sc.Repeat, sc.AllowedDays); } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); + FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", + ex); } //If time is exceeded, run it now @@ -288,15 +312,17 @@ private void Runner() { var jobsToRun = new List(); //TODO: Cache this to avoid frequent lookups - foreach(var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct().Select(x => x.ToString())) + foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct() + .Select(x => x.ToString())) { //See if it is already queued var tmplst = from n in m_worker.CurrentTasks - where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup - select n.Backup; + where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup + select n.Backup; var tastTemp = m_worker.CurrentTask; - if (tastTemp != null && tastTemp.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup) - tmplst = tmplst.Union(new [] { tastTemp.Backup }); + if (tastTemp != null && tastTemp.Operation == + Duplicati.Server.Serialization.DuplicatiOperation.Backup) + tmplst = tmplst.Union(new[] { tastTemp.Backup }); //If it is not already in queue, put it there if (!tmplst.Any(x => x.ID == id)) @@ -306,13 +332,18 @@ private void Runner() { Dictionary options = Duplicati.Server.Runner.GetCommonOptions(); Duplicati.Server.Runner.ApplyOptions(entry, options); - if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) + if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && + (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == + Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "BackupDisabledOnBattery", "Scheduled backup disabled while on battery power."); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, + "BackupDisabledOnBattery", + "Scheduled backup disabled while on battery power."); } else { - jobsToRun.Add(Server.Runner.CreateTask(Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); + jobsToRun.Add(Server.Runner.CreateTask( + Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); } } } @@ -321,27 +352,32 @@ private void Runner() // Calculate next time, by finding the first entry later than now try { - start = GetNextValidTime(start, new DateTime(Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), DateTimeKind.Utc), sc.Repeat, sc.AllowedDays); + start = GetNextValidTime(start, + new DateTime( + Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), + DateTimeKind.Utc), sc.Repeat, sc.AllowedDays); } - catch(Exception ex) + catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); + FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), + "Scheduler failed to find next date", ex); continue; } - + Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault(); if (lastJob != null) { lock (m_lock) { // The actual last run time will be updated when the StartingWork event is raised. - m_updateTasks[lastJob] = new Tuple(sc, start, DateTime.UtcNow); + m_updateTasks[lastJob] = + new Tuple(sc, start, DateTime.UtcNow); } } foreach (var job in jobsToRun) m_worker.AddTask(job); - + if (start < DateTime.UtcNow) { //TODO: Report this somehow @@ -349,20 +385,20 @@ private void Runner() } } - scheduled[sc.ID] = new KeyValuePair(scticks, start); + scheduled[sc.ID] = new KeyValuePair(scticks, start); } } var existing = lst.ToDictionary(x => x.ID); //Sort them, lock as we assign the m_schedule variable - lock(m_lock) + lock (m_lock) m_schedule = (from n in scheduled where existing.ContainsKey(n.Key) orderby n.Value.Value select new KeyValuePair(n.Value.Value, existing[n.Key])).ToArray(); // Remove unused entries - foreach(var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) + foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) scheduled.Remove(c); //Raise event if needed @@ -406,8 +442,7 @@ private static bool IsDateAllowed(DateTime time, DayOfWeek[] allowedDays) if (allowedDays == null || allowedDays.Length == 0) return true; else - return Array.IndexOf(allowedDays, localTime.DayOfWeek) >= 0; + return Array.IndexOf(allowedDays, localTime.DayOfWeek) >= 0; } - } -} +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs b/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs index c77ae60275..6f08430185 100644 --- a/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs +++ b/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs @@ -139,10 +139,9 @@ public long LastEventID set { m_lastEventID = value; } } - public long LastDataUpdateID { get { return FIXMEGlobal.PeekLastDataUpdateID(); } } - - public long LastNotificationUpdateID { get { return FIXMEGlobal.PeekLastNotificationUpdateID(); } } + public long LastDataUpdateID => FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; + public long LastNotificationUpdateID => FIXMEGlobal.NotificationUpdateService.LastNotificationUpdateId; } } diff --git a/Duplicati.Library.RestAPI/UpdatePollThread.cs b/Duplicati.Library.RestAPI/UpdatePollThread.cs index c82bd8d2d2..869f992d08 100644 --- a/Duplicati.Library.RestAPI/UpdatePollThread.cs +++ b/Duplicati.Library.RestAPI/UpdatePollThread.cs @@ -28,12 +28,12 @@ namespace Duplicati.Server /// public class UpdatePollThread { - private readonly Thread m_thread; + private Thread m_thread; private volatile bool m_terminated = false; private volatile bool m_download = false; private volatile bool m_forceCheck = false; private readonly object m_lock = new object(); - private readonly AutoResetEvent m_waitSignal; + private AutoResetEvent m_waitSignal; private double m_downloadProgress; public bool IsUpdateRequested { get; private set; } = false; @@ -51,14 +51,16 @@ private set FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } - - public UpdatePollThread() + + public void Init() { m_waitSignal = new AutoResetEvent(false); ThreadState = UpdatePollerStates.Waiting; - m_thread = new Thread(Run); - m_thread.IsBackground = true; - m_thread.Name = "UpdatePollThread"; + m_thread = new Thread(Run) + { + IsBackground = true, + Name = "UpdatePollThread" + }; m_thread.Start(); } diff --git a/Duplicati.Library.RestAPI/WorkerThreadsManager.cs b/Duplicati.Library.RestAPI/WorkerThreadsManager.cs new file mode 100644 index 0000000000..79caca6953 --- /dev/null +++ b/Duplicati.Library.RestAPI/WorkerThreadsManager.cs @@ -0,0 +1,34 @@ +#nullable enable +using System; +using Duplicati.Library.IO; +using Duplicati.Library.RestAPI.Abstractions; +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.Library.RestAPI; + +public class WorkerThreadsManager(ILiveControls liveControls, IScheduler scheduler) : IWorkerThreadsManager +{ + public WorkerThread? WorkerThread { get; private set; } + + public void Spawn(Action item) + { + WorkerThread = new WorkerThread(item, liveControls.IsPaused); + scheduler.Init(WorkerThread); + } + + public Tuple? CurrentTask + { + get + { + var t = WorkerThread?.CurrentTask; + return t == null ? null : new Tuple(t.TaskID, t.Backup.ID); + } + } + + public void UpdateThrottleSpeeds() + { + WorkerThread?.CurrentTask?.UpdateThrottleSpeed(); + } +} \ No newline at end of file diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 857abfeba3..db69443ada 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -140,7 +140,7 @@ public static void RealMain(string[] _args) } else if (Library.Utility.Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION)) { - databaseConnection = Server.Program.GetDatabaseConnection(options); + databaseConnection = Server.Program.GetDatabaseConnection(null, options); if (databaseConnection != null) { diff --git a/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs b/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs new file mode 100644 index 0000000000..26763743aa --- /dev/null +++ b/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.AutoUpdater; + +public interface IUpdateManagerAccessor +{ + bool HasUpdateInstalled { get; } +} \ No newline at end of file diff --git a/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs b/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs new file mode 100644 index 0000000000..cc1cae43c4 --- /dev/null +++ b/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.AutoUpdater; + +public class UpdateManagerAccessor : IUpdateManagerAccessor +{ + public bool HasUpdateInstalled => UpdaterManager.HasUpdateInstalled; +} diff --git a/Duplicati/Library/Common/ILiveControls.cs b/Duplicati/Library/Common/ILiveControls.cs new file mode 100644 index 0000000000..cef107b119 --- /dev/null +++ b/Duplicati/Library/Common/ILiveControls.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.IO; + +public interface ILiveControls +{ + bool IsPaused { get; } +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/Abstractions/IBoolParser.cs b/Duplicati/Library/Utility/Abstractions/IBoolParser.cs new file mode 100644 index 0000000000..782b80a761 --- /dev/null +++ b/Duplicati/Library/Utility/Abstractions/IBoolParser.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.Utility.Abstractions; + +public interface IBoolParser +{ + bool ParseBool(string value, bool @default = false); +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/BoolParser.cs b/Duplicati/Library/Utility/BoolParser.cs new file mode 100644 index 0000000000..1f4f0d7572 --- /dev/null +++ b/Duplicati/Library/Utility/BoolParser.cs @@ -0,0 +1,11 @@ +using Duplicati.Library.Utility.Abstractions; + +namespace Duplicati.Library.Utility; + +public class BoolParser : IBoolParser +{ + public bool ParseBool(string value, bool @default = false) + { + return Utility.ParseBool(value, @default); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/WorkerThread.cs b/Duplicati/Library/Utility/WorkerThread.cs index dfdd99f2a5..267b40e89b 100644 --- a/Duplicati/Library/Utility/WorkerThread.cs +++ b/Duplicati/Library/Utility/WorkerThread.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2024, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -391,4 +391,4 @@ public bool Join(int millisecondTimeout) return true; } } -} +} \ No newline at end of file diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs b/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs index 14c410ee7b..79ab5492d6 100644 --- a/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2024, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; @@ -31,26 +31,28 @@ namespace Duplicati.Server.Serialization { public class Serializer { - protected static readonly JsonSerializerSettings m_jsonSettings; + public static JsonSerializerSettings JsonSettings { get; } protected static readonly Formatting m_jsonFormatting = Formatting.Indented; static Serializer() { - m_jsonSettings = new JsonSerializerSettings(); - m_jsonSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; - m_jsonSettings.Converters = new JsonConverter[] { - new DayOfWeekConcerter(), - new StringEnumConverter(), - new SerializableStatusCreator(), - new SettingsCreator(), - new FilterCreator(), - new NotificationCreator(), - }.ToList(); + JsonSettings = new JsonSerializerSettings + { + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + Converters = new JsonConverter[] { + new DayOfWeekConcerter(), + new StringEnumConverter(), + new SerializableStatusCreator(), + new SettingsCreator(), + new FilterCreator(), + new NotificationCreator(), + }.ToList() + }; } public static void SerializeJson(System.IO.TextWriter sw, object o, bool preventDispose = false) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); var jsonWriter = new JsonTextWriter(sw); using (preventDispose ? null : jsonWriter) { @@ -63,7 +65,7 @@ public static void SerializeJson(System.IO.TextWriter sw, object o, bool prevent public static async Task SerializeJsonAsync(System.IO.TextWriter tw, object o, bool preventDispose = false) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); StringBuilder sb = new StringBuilder(); StringWriter sw = new StringWriter(sb); var jsonWriter = new JsonTextWriter(sw); @@ -80,7 +82,7 @@ public static async Task SerializeJsonAsync(System.IO.TextWriter tw, object o, b public static T Deserialize(System.IO.TextReader sr) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); using (var jsonReader = new JsonTextReader(sr)) { jsonReader.Culture = System.Globalization.CultureInfo.InvariantCulture; diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 82dcf0db81..28b2ef77bf 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -24,8 +24,13 @@ using System.Linq; using Duplicati.Library.Common; using Duplicati.Library.Common.IO; +using Duplicati.Library.IO; using Duplicati.Library.RestAPI; using Duplicati.WebserverCore; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Database; +using Microsoft.Extensions.DependencyInjection; +using Sharp.Xmpp.Extensions.Dataforms; namespace Duplicati.Server { @@ -78,12 +83,7 @@ public class Program /// /// This is the scheduling thread /// - public static Scheduler Scheduler { get => FIXMEGlobal.Scheduler; set => FIXMEGlobal.Scheduler = value; } - - /// - /// This is the working thread - /// - public static Duplicati.Library.Utility.WorkerThread WorkThread { get => FIXMEGlobal.WorkThread; set => FIXMEGlobal.WorkThread = value; } + public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; } /// /// List of completed task results @@ -108,7 +108,7 @@ public class Program /// /// The controller interface for pause/resume and throttle options /// - public static LiveControls LiveControl { get => FIXMEGlobal.LiveControl; set => FIXMEGlobal.LiveControl = value; } + public static LiveControls LiveControl { get => DuplicatiWebserver.Provider.GetRequiredService() ; } /// /// The application exit event @@ -120,15 +120,23 @@ public class Program /// private static WebServer.Server WebServer; + /// + /// Duplicati webserver instance + /// + public static DuplicatiWebserver DuplicatiWebserver { get; set; } + /// /// Callback to shutdown the modern webserver /// - private static Action ShutdownModernWebserver; + private static void ShutdownModernWebserver() + { + DuplicatiWebserver.Stop().GetAwaiter().GetResult(); + } /// /// The update poll thread. /// - public static UpdatePollThread UpdatePoller { get => FIXMEGlobal.UpdatePoller; set => FIXMEGlobal.UpdatePoller = value; } + public static UpdatePollThread UpdatePoller => FIXMEGlobal.UpdatePoller; /// /// An event that is set once the server is ready to respond to requests @@ -138,23 +146,13 @@ public class Program /// /// The status event signaler, used to control long polling of status updates /// - public static EventPollNotify StatusEventNotifyer { get => FIXMEGlobal.StatusEventNotifyer; } + public static EventPollNotify StatusEventNotifyer => FIXMEGlobal.Provider.GetRequiredService(); /// /// A delegate method for creating a copy of the current progress state /// public static Func GenerateProgressState { get => FIXMEGlobal.GenerateProgressState; set => FIXMEGlobal.GenerateProgressState = value; } - /// - /// An event ID that increases whenever the database is updated - /// - public static long LastDataUpdateID = 0; - - /// - /// An event ID that increases whenever a notification is updated - /// - public static long LastNotificationUpdateID = 0; - /// /// The log redirect handler /// @@ -193,25 +191,10 @@ public static bool ServerPortChanged set { DataConnection.ApplicationSettings.ServerPortChanged = value; } } - public static void IncrementLastDataUpdateID() - { - System.Threading.Interlocked.Increment(ref Program.LastDataUpdateID); - } - - public static void IncrementLastNotificationUpdateID() - { - System.Threading.Interlocked.Increment(ref Program.LastNotificationUpdateID); - } - static Program() { - FIXMEGlobal.IncrementLastDataUpdateID = Program.IncrementLastDataUpdateID; - FIXMEGlobal.PeekLastDataUpdateID = () => Program.LastDataUpdateID; - FIXMEGlobal.IncrementLastNotificationUpdateID = Program.IncrementLastNotificationUpdateID; - FIXMEGlobal.PeekLastNotificationUpdateID = () => Program.LastNotificationUpdateID; FIXMEGlobal.GetDatabaseConnection = Program.GetDatabaseConnection; FIXMEGlobal.StartOrStopUsageReporter = Program.StartOrStopUsageReporter; - FIXMEGlobal.UpdateThrottleSpeeds = Program.UpdateThrottleSpeeds; } /// @@ -220,6 +203,9 @@ static Program() [STAThread] public static int Main(string[] args) { + // var methodInfo = typeof(TemporaryIoCAccessor).Assembly.EntryPoint; + // var program = Activator.CreateInstance(methodInfo!.DeclaringType!); + // methodInfo.Invoke(program, [Array.Empty()]); return Duplicati.Library.AutoUpdater.UpdaterManager.RunFromMostRecent(typeof(Program).GetMethod("RealMain"), args, Duplicati.Library.AutoUpdater.AutoUpdateStrategy.Never); } @@ -278,8 +264,10 @@ public static int RealMain(string[] _args) try { - - DataConnection = GetDatabaseConnection(commandlineOptions); + DuplicatiWebserver = new DuplicatiWebserver(); + DuplicatiWebserver.InitWebServer(); + FIXMEGlobal.Provider = DuplicatiWebserver.Provider; + DataConnection = GetDatabaseConnection(DuplicatiWebserver, commandlineOptions); if (!DataConnection.ApplicationSettings.FixedInvalidBackupId) DataConnection.FixInvalidBackupId(); @@ -292,12 +280,12 @@ public static int RealMain(string[] _args) ApplicationExitEvent = new System.Threading.ManualResetEvent(false); - Library.AutoUpdater.UpdaterManager.OnError += (Exception obj) => + Library.AutoUpdater.UpdaterManager.OnError += obj => { DataConnection.LogError(null, "Error in updater", obj); }; - UpdatePoller = new UpdatePollThread(); + UpdatePoller.Init(); SetPurgeTempFilesTimer(commandlineOptions); @@ -305,7 +293,7 @@ public static int RealMain(string[] _args) SetWorkerThread(); - StartWebServer(commandlineOptions); + StartWebServer(DuplicatiWebserver, commandlineOptions); if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, "ping-pong-keepalive")) { @@ -342,7 +330,7 @@ public static int RealMain(string[] _args) ShutdownModernWebserver(); UpdatePoller?.Terminate(); Scheduler?.Terminate(true); - WorkThread?.Terminate(true); + FIXMEGlobal.WorkThread?.Terminate(true); ApplicationInstance?.Dispose(); PurgeTempFilesTimer?.Dispose(); @@ -360,37 +348,33 @@ public static int RealMain(string[] _args) return 0; } - private static void StartWebServer(Dictionary commandlineOptions) + private static void StartWebServer(DuplicatiWebserver webserver, Dictionary commandlineOptions) { WebServer = new WebServer.Server(commandlineOptions); ServerPortChanged |= WebServer.Port != DataConnection.ApplicationSettings.LastWebserverPort; DataConnection.ApplicationSettings.LastWebserverPort = WebServer.Port; - - var server = new DuplicatiWebserver(); - ShutdownModernWebserver = server.Foo(); + + webserver.Start().GetAwaiter().GetResult(); } private static void SetWorkerThread() { - WorkThread = new Duplicati.Library.Utility.WorkerThread((x) => { Runner.Run(x, true); }, - LiveControl.State == LiveControls.LiveControlState.Paused); - Scheduler = new Scheduler(WorkThread); - - WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; - WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; - WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; - Scheduler.NewSchedule += new EventHandler(SignalNewEvent); - WorkThread.OnError += (worker, task, exception) => + FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); }); + FIXMEGlobal.WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; + FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; + FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; + FIXMEGlobal.Scheduler.NewSchedule += new EventHandler(SignalNewEvent); + FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { Program.DataConnection.LogError(task?.BackupID, "Error in worker", exception); }; - var lastScheduleId = LastDataUpdateID; + var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; Program.StatusEventNotifyer.NewEvent += (sender, e) => { - if (lastScheduleId == LastDataUpdateID) return; - lastScheduleId = LastDataUpdateID; + if (lastScheduleId == FIXMEGlobal.NotificationUpdateService.LastDataUpdateId) return; + lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; Program.Scheduler.Reschedule(); }; @@ -413,13 +397,13 @@ void RegisterTaskResult(long id, Exception ex) } } - Program.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; - Program.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; + FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; + FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; } private static void SetLiveControls() { - LiveControl = new LiveControls(DataConnection.ApplicationSettings); + LiveControl.Init(DataConnection.ApplicationSettings); LiveControl.StateChanged += LiveControl_StateChanged; LiveControl.ThreadPriorityChanged += LiveControl_ThreadPriorityChanged; LiveControl.ThrottleSpeedChanged += LiveControl_ThrottleSpeedChanged; @@ -589,7 +573,7 @@ private static int ShowHelp(bool writeConsole) throw new Exception("Server invoked with --help"); } - public static Database.Connection GetDatabaseConnection(Dictionary commandlineOptions) + public static Database.Connection GetDatabaseConnection(object webserver, Dictionary commandlineOptions) { var dbPassword = Environment.GetEnvironmentVariable(DB_KEY_ENV_NAME); @@ -726,22 +710,11 @@ public static void StartOrStopUsageReporter() Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter); } - public static void UpdateThrottleSpeeds() - { - if (Program.WorkThread == null) - return; - - var cur = Program.WorkThread.CurrentTask; - if (cur != null) - cur.UpdateThrottleSpeed(); - } - private static void SignalNewEvent(object sender, EventArgs e) { StatusEventNotifyer.SignalNewEvent(); } - /// /// Handles a change in the LiveControl and updates the Runner /// @@ -765,29 +738,33 @@ private static void LiveControl_ThrottleSpeedChanged(object sender, EventArgs e) /// /// This event handler updates the trayicon menu with the current state of the runner. /// - static void LiveControl_StateChanged(object sender, EventArgs e) + /// + private static void LiveControl_StateChanged(object sender, EventArgs e) { + var worker = FIXMEGlobal.WorkThread; switch (LiveControl.State) { case LiveControls.LiveControlState.Paused: { - WorkThread.Pause(); - var t = WorkThread.CurrentTask; + worker.Pause(); + var t = worker.CurrentTask; t?.Pause(); break; } case LiveControls.LiveControlState.Running: { - WorkThread.Resume(); - var t = WorkThread.CurrentTask; + worker.Resume(); + var t = worker.CurrentTask; t?.Resume(); break; } + default: + throw new InvalidOperationException($"State of {nameof(LiveControl)} was not recognized!"); } StatusEventNotifyer.SignalNewEvent(); } - + /// /// Simple method for tracking if the server has crashed /// diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index d976ee7c8e..0761b5fa25 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -105,7 +105,7 @@ public void ExportToJSONEncoding(bool removePasswords) } byte[] jsonByteArray; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions)) + using (Program.DataConnection = Program.GetDatabaseConnection(null, advancedOptions)) { jsonByteArray = Server.WebServer.RESTMethods.Backup.ExportToJSON(backup, null); } @@ -127,7 +127,7 @@ public void RoundTrip() { Dictionary metadata = new Dictionary {{"SourceFilesCount", "1"}}; Dictionary advancedOptions = new Dictionary {{"server-datafolder", this.serverDatafolder}}; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions)) + using (Program.DataConnection = Program.GetDatabaseConnection(null, advancedOptions)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); diff --git a/Duplicati/WebserverCore/Abstractions/FileEntry.cs b/Duplicati/WebserverCore/Abstractions/FileEntry.cs new file mode 100644 index 0000000000..320ff6842f --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/FileEntry.cs @@ -0,0 +1,10 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class FileEntry +{ + public string Path { get; set; } = ""; + public string MD5 { get; set; } = ""; + public string SHA256 { get; set; } = ""; + public DateTime? LastWriteTime { get; set; } + public bool Ignore { get; set; } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/ISettingsService.cs b/Duplicati/WebserverCore/Abstractions/ISettingsService.cs new file mode 100644 index 0000000000..06b07e0a28 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/ISettingsService.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface ISettingsService +{ + ServerSettings GetSettings(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IStatusService.cs b/Duplicati/WebserverCore/Abstractions/IStatusService.cs new file mode 100644 index 0000000000..0bbe7c564c --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IStatusService.cs @@ -0,0 +1,8 @@ +using Duplicati.WebserverCore.Dto; + +namespace Duplicati.WebserverCore.Abstractions; + +public interface IStatusService +{ + ServerStatusDto GetStatus(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IUpdateService.cs b/Duplicati/WebserverCore/Abstractions/IUpdateService.cs new file mode 100644 index 0000000000..d19d9486af --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IUpdateService.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface IUpdateService +{ + UpdateInfo? GetUpdateInfo(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs b/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs new file mode 100644 index 0000000000..057f5d4dcf --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface IV1Endpoint +{ + public static abstract void Map(RouteGroupBuilder group); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/ServerSettings.cs b/Duplicati/WebserverCore/Abstractions/ServerSettings.cs new file mode 100644 index 0000000000..647788d988 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/ServerSettings.cs @@ -0,0 +1,29 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class ServerSettings +{ + public string StartupDelay { get; set; } = ""; + public string DownloadSpeedLimit { get; set; }= ""; + public string UploadSpeedLimit { get; set; }= ""; + public string ThreadPriority { get; set; }= ""; + public string LastWebserverPort { get; set; }= ""; + public string IsFirstRun { get; set; }= ""; + public string ServerPortChanged { get; set; }= ""; + public string ServerPassphrase { get; set; }= ""; + public string ServerPassphraseSalt { get; set; }= ""; + public string ServerPassphraseTrayIcon { get; set; }= ""; + public string ServerPassphraseTrayIconHash { get; set; }= ""; + public string UpdateCheckLast { get; set; }= ""; + public string UpdateCheckInterval { get; set; }= ""; + public string UpdateCheckNewVersion { get; set; }= ""; + public bool UnackedError { get; set; } + public bool UnackedWarning { get; set; } + public string ServerListenInterface { get; set; }= ""; + public string ServerSslCertificate { get; set; }= ""; + public string HasFixedInvalidBackupId { get; set; }= ""; + public string UpdateChannel { get; set; }= ""; + public string UsageReporterLevel { get; set; }= ""; + public string HasAskedForPasswordProtection { get; set; }= ""; + public string DisableTrayIconLogin { get; set; }= ""; + public string ServerAllowedHostnames { get; set; }= ""; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs b/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs new file mode 100644 index 0000000000..b3554bf955 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs @@ -0,0 +1,17 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class UpdateInfo +{ + public string Displayname { get; set; } = ""; + public string Version { get; set; } = ""; + public DateTime? ReleaseTime { get; set; } + public string ReleaseType { get; set; } = ""; + public string UpdateSeverity { get; set; } = ""; + public string ChangeInfo { get; set; } = ""; + public long CompressedSize { get; set; } + public long UncompressedSize { get; set; } + public string SHA256 { get; set; } = ""; + public string MD5 { get; set; } = ""; + public string[] RemoteURLS { get; set; } = []; + public FileEntry[] Files { get; set; } = []; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/ApplicationPartsLogger.cs b/Duplicati/WebserverCore/ApplicationPartsLogger.cs new file mode 100644 index 0000000000..7cd5f64c84 --- /dev/null +++ b/Duplicati/WebserverCore/ApplicationPartsLogger.cs @@ -0,0 +1,32 @@ +using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Controllers; + +namespace Duplicati.WebserverCore; + +//Useful for debugging ASP.net magically loading controllers +public class ApplicationPartsLogger(ILogger logger, ApplicationPartManager partManager) + : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + // Get the names of all the application parts. This is the short assembly name for AssemblyParts + var applicationParts = partManager.ApplicationParts.Select(x => x.Name); + + // Create a controller feature, and populate it from the application parts + var controllerFeature = new ControllerFeature(); + partManager.PopulateFeature(controllerFeature); + + // Get the names of all of the controllers + var controllers = controllerFeature.Controllers.Select(x => x.Name); + + // Log the application parts and controllers + logger.LogInformation( + "Found the following application parts: '{ApplicationParts}' with the following controllers: '{Controllers}'", + string.Join(", ", applicationParts), string.Join(", ", controllers)); + + return Task.CompletedTask; + } + + // Required by the interface + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs b/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs new file mode 100644 index 0000000000..23a5deb77b --- /dev/null +++ b/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs @@ -0,0 +1,14 @@ +using Duplicati.WebserverCore.Database.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Duplicati.WebserverCore.Database.Configuration; + +public class OptionConfiguration : IEntityTypeConfiguration