From 5925aac859ee493fd7f6b10026c84a6a22626c79 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Wed, 30 Jun 2010 00:10:44 +0100 Subject: [PATCH] Add --merge switch to load iar. When this switch is used, iar folders are merged with existing same-name user inventory folders. This makes it a little easier to back and restore entire individual user inventories, among other things Added unit test to check behaviour --- .../Serialization/ArchiveConstants.cs | 29 ++++- .../Archiver/InventoryArchiveReadRequest.cs | 112 ++++++++++-------- .../Archiver/InventoryArchiverModule.cs | 60 +++++----- .../Archiver/Tests/InventoryArchiverTests.cs | 50 +++++++- .../Framework/Library/LibraryModule.cs | 8 +- .../Shared/Api/Implementation/LS_Api.cs | 2 +- 6 files changed, 170 insertions(+), 91 deletions(-) diff --git a/OpenSim/Framework/Serialization/ArchiveConstants.cs b/OpenSim/Framework/Serialization/ArchiveConstants.cs index 475a9de0cb..3143e3bec7 100644 --- a/OpenSim/Framework/Serialization/ArchiveConstants.cs +++ b/OpenSim/Framework/Serialization/ArchiveConstants.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Text; using OpenMetaverse; namespace OpenSim.Framework.Serialization @@ -171,6 +172,30 @@ namespace OpenSim.Framework.Serialization public static string CreateOarObjectPath(string objectName, UUID uuid, Vector3 pos) { return OBJECTS_PATH + CreateOarObjectFilename(objectName, uuid, pos); - } + } + + /// + /// Extract a plain path from an IAR path + /// + /// + /// + public static string ExtractPlainPathFromIarPath(string iarPath) + { + List plainDirs = new List(); + + string[] iarDirs = iarPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + foreach (string iarDir in iarDirs) + { + if (!iarDir.Contains(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR)) + plainDirs.Add(iarDir); + + int i = iarDir.LastIndexOf(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR); + + plainDirs.Add(iarDir.Remove(i)); + } + + return string.Join("/", plainDirs.ToArray()); + } } -} +} \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveReadRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveReadRequest.cs index 9996074860..f130b3e0ee 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveReadRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveReadRequest.cs @@ -54,6 +54,11 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver private UserAccount m_userInfo; private string m_invPath; + + /// + /// Do we want to merge this load with existing inventory? + /// + protected bool m_merge; /// /// We only use this to request modules @@ -66,19 +71,21 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver private Stream m_loadStream; public InventoryArchiveReadRequest( - Scene scene, UserAccount userInfo, string invPath, string loadPath) + Scene scene, UserAccount userInfo, string invPath, string loadPath, bool merge) : this( scene, userInfo, invPath, - new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress)) + new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress), + merge) { } public InventoryArchiveReadRequest( - Scene scene, UserAccount userInfo, string invPath, Stream loadStream) + Scene scene, UserAccount userInfo, string invPath, Stream loadStream, bool merge) { m_scene = scene; + m_merge = merge; m_userInfo = userInfo; m_invPath = invPath; m_loadStream = loadStream; @@ -91,14 +98,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver /// A list of the inventory nodes loaded. If folders were loaded then only the root folders are /// returned /// - public List Execute() + public HashSet Execute() { string filePath = "ERROR"; int successfulAssetRestores = 0; int failedAssetRestores = 0; int successfulItemRestores = 0; - List loadedNodes = new List(); + HashSet loadedNodes = new HashSet(); List folderCandidates = InventoryArchiveUtils.FindFolderByPath( @@ -158,9 +165,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver { successfulItemRestores++; - // If we're loading an item directly into the given destination folder then we need to record - // it separately from any loaded root folders - if (rootDestinationFolder == foundFolder) + // If we aren't loading the folder containing the item then well need to update the + // viewer separately for that item. + if (!loadedNodes.Contains(foundFolder)) loadedNodes.Add(item); } } @@ -203,14 +210,15 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver string iarPath, InventoryFolderBase rootDestFolder, Dictionary resolvedFolders, - List loadedNodes) + HashSet loadedNodes) { string iarPathExisting = iarPath; // m_log.DebugFormat( // "[INVENTORY ARCHIVER]: Loading folder {0} {1}", rootDestFolder.Name, rootDestFolder.ID); - InventoryFolderBase destFolder = ResolveDestinationFolder(rootDestFolder, ref iarPathExisting, resolvedFolders); + InventoryFolderBase destFolder + = ResolveDestinationFolder(rootDestFolder, ref iarPathExisting, resolvedFolders); m_log.DebugFormat( "[INVENTORY ARCHIVER]: originalArchivePath [{0}], section already loaded [{1}]", @@ -249,46 +257,55 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver { string originalArchivePath = archivePath; - InventoryFolderBase destFolder = null; - - if (archivePath.Length > 0) + while (archivePath.Length > 0) { - while (null == destFolder && archivePath.Length > 0) + m_log.DebugFormat("[INVENTORY ARCHIVER]: Trying to resolve destination folder {0}", archivePath); + + if (resolvedFolders.ContainsKey(archivePath)) { - m_log.DebugFormat("[INVENTORY ARCHIVER]: Trying to resolve destination folder {0}", archivePath); - - if (resolvedFolders.ContainsKey(archivePath)) + m_log.DebugFormat( + "[INVENTORY ARCHIVER]: Found previously created folder from archive path {0}", archivePath); + return resolvedFolders[archivePath]; + } + else + { + if (m_merge) { - m_log.DebugFormat( - "[INVENTORY ARCHIVER]: Found previously created folder from archive path {0}", archivePath); - destFolder = resolvedFolders[archivePath]; + // TODO: Using m_invPath is totally wrong - what we need to do is strip the uuid from the + // iar name and try to find that instead. + string plainPath = ArchiveConstants.ExtractPlainPathFromIarPath(archivePath); + List folderCandidates + = InventoryArchiveUtils.FindFolderByPath( + m_scene.InventoryService, m_userInfo.PrincipalID, plainPath); + + if (folderCandidates.Count != 0) + { + InventoryFolderBase destFolder = folderCandidates[0]; + resolvedFolders[archivePath] = destFolder; + return destFolder; + } + } + + // Don't include the last slash so find the penultimate one + int penultimateSlashIndex = archivePath.LastIndexOf("/", archivePath.Length - 2); + + if (penultimateSlashIndex >= 0) + { + // Remove the last section of path so that we can see if we've already resolved the parent + archivePath = archivePath.Remove(penultimateSlashIndex + 1); } else { - // Don't include the last slash so find the penultimate one - int penultimateSlashIndex = archivePath.LastIndexOf("/", archivePath.Length - 2); - - if (penultimateSlashIndex >= 0) - { - // Remove the last section of path so that we can see if we've already resolved the parent - archivePath = archivePath.Remove(penultimateSlashIndex + 1); - } - else - { - m_log.DebugFormat( - "[INVENTORY ARCHIVER]: Found no previously created folder for archive path {0}", - originalArchivePath); - archivePath = string.Empty; - destFolder = rootDestFolder; - } + m_log.DebugFormat( + "[INVENTORY ARCHIVER]: Found no previously created folder for archive path {0}", + originalArchivePath); + archivePath = string.Empty; + return rootDestFolder; } } } - if (null == destFolder) - destFolder = rootDestFolder; - - return destFolder; + return rootDestFolder; } /// @@ -314,24 +331,21 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver string iarPathExisting, string iarPathToReplicate, Dictionary resolvedFolders, - List loadedNodes) + HashSet loadedNodes) { string[] rawDirsToCreate = iarPathToReplicate.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - int i = 0; - while (i < rawDirsToCreate.Length) + for (int i = 0; i < rawDirsToCreate.Length; i++) { // m_log.DebugFormat("[INVENTORY ARCHIVER]: Creating folder {0} from IAR", rawDirsToCreate[i]); + if (!rawDirsToCreate[i].Contains(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR)) + continue; + int identicalNameIdentifierIndex = rawDirsToCreate[i].LastIndexOf( ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR); - if (identicalNameIdentifierIndex < 0) - { - i++; - continue; - } string newFolderName = rawDirsToCreate[i].Remove(identicalNameIdentifierIndex); newFolderName = InventoryArchiveUtils.UnescapeArchivePath(newFolderName); @@ -354,8 +368,6 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (0 == i) loadedNodes.Add(destFolder); - - i++; } } diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiverModule.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiverModule.cs index cfefbe9d3b..668c344d39 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiverModule.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiverModule.cs @@ -91,12 +91,12 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver scene.AddCommand( this, "load iar", - "load iar []", + "load iar []", //"load iar [--merge] []", "Load user inventory archive (IAR).", - //"--merge is an option which merges the loaded IAR with existing inventory folders where possible, rather than always creating new ones" + //"--merge is an option which merges the loaded IAR with existing inventory folders where possible, rather than always creating new ones" //+ " is user's first name." + Environment.NewLine - " is user's first name." + Environment.NewLine + " is user's first name." + Environment.NewLine + " is user's last name." + Environment.NewLine + " is the path inside the user's inventory where the IAR should be loaded." + Environment.NewLine + " is the user's password." + Environment.NewLine @@ -136,16 +136,16 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (handlerInventoryArchiveSaved != null) handlerInventoryArchiveSaved(id, succeeded, userInfo, invPath, saveStream, reportedException); } - + public bool ArchiveInventory( - Guid id, string firstName, string lastName, string invPath, string pass, Stream saveStream) - { - return ArchiveInventory(id, firstName, lastName, invPath, pass, saveStream, new Dictionary()); - } + Guid id, string firstName, string lastName, string invPath, string pass, Stream saveStream) + { + return ArchiveInventory(id, firstName, lastName, invPath, pass, saveStream, new Dictionary()); + } public bool ArchiveInventory( - Guid id, string firstName, string lastName, string invPath, string pass, Stream saveStream, - Dictionary options) + Guid id, string firstName, string lastName, string invPath, string pass, Stream saveStream, + Dictionary options) { if (m_scenes.Count > 0) { @@ -184,8 +184,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver } public bool ArchiveInventory( - Guid id, string firstName, string lastName, string invPath, string pass, string savePath, - Dictionary options) + Guid id, string firstName, string lastName, string invPath, string pass, string savePath, + Dictionary options) { if (m_scenes.Count > 0) { @@ -224,13 +224,13 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver } public bool DearchiveInventory(string firstName, string lastName, string invPath, string pass, Stream loadStream) - { - return DearchiveInventory(firstName, lastName, invPath, pass, loadStream, new Dictionary()); - } - + { + return DearchiveInventory(firstName, lastName, invPath, pass, loadStream, new Dictionary()); + } + public bool DearchiveInventory( - string firstName, string lastName, string invPath, string pass, Stream loadStream, - Dictionary options) + string firstName, string lastName, string invPath, string pass, Stream loadStream, + Dictionary options) { if (m_scenes.Count > 0) { @@ -241,10 +241,11 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (CheckPresence(userInfo.PrincipalID)) { InventoryArchiveReadRequest request; + bool merge = (options.ContainsKey("merge") ? (bool)options["merge"] : false); try { - request = new InventoryArchiveReadRequest(m_aScene, userInfo, invPath, loadStream); + request = new InventoryArchiveReadRequest(m_aScene, userInfo, invPath, loadStream, merge); } catch (EntryPointNotFoundException e) { @@ -273,8 +274,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver } public bool DearchiveInventory( - string firstName, string lastName, string invPath, string pass, string loadPath, - Dictionary options) + string firstName, string lastName, string invPath, string pass, string loadPath, + Dictionary options) { if (m_scenes.Count > 0) { @@ -285,10 +286,11 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (CheckPresence(userInfo.PrincipalID)) { InventoryArchiveReadRequest request; + bool merge = (options.ContainsKey("merge") ? (bool)options["merge"] : false); try - { - request = new InventoryArchiveReadRequest(m_aScene, userInfo, invPath, loadPath); + { + request = new InventoryArchiveReadRequest(m_aScene, userInfo, invPath, loadPath, merge); } catch (EntryPointNotFoundException e) { @@ -322,13 +324,13 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver /// protected void HandleLoadInvConsoleCommand(string module, string[] cmdparams) { - m_log.Info("[INVENTORY ARCHIVER]: PLEASE NOTE THAT THIS FACILITY IS EXPERIMENTAL. BUG REPORTS WELCOME."); - - Dictionary options = new Dictionary(); + m_log.Info("[INVENTORY ARCHIVER]: PLEASE NOTE THAT THIS FACILITY IS EXPERIMENTAL. BUG REPORTS WELCOME."); + + Dictionary options = new Dictionary(); OptionSet optionSet = new OptionSet().Add("m|merge", delegate (string v) { options["merge"] = v != null; }); List mainParams = optionSet.Parse(cmdparams); - + if (mainParams.Count < 6) { m_log.Error( @@ -349,7 +351,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (DearchiveInventory(firstName, lastName, invPath, pass, loadPath, options)) m_log.InfoFormat( "[INVENTORY ARCHIVER]: Loaded archive {0} for {1} {2}", - loadPath, firstName, lastName); + loadPath, firstName, lastName); } /// @@ -454,7 +456,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver /// Notify the client of loaded nodes if they are logged in /// /// Can be empty. In which case, nothing happens - private void UpdateClientWithLoadedNodes(UserAccount userInfo, List loadedNodes) + private void UpdateClientWithLoadedNodes(UserAccount userInfo, HashSet loadedNodes) { if (loadedNodes.Count == 0) return; diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/Tests/InventoryArchiverTests.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/Tests/InventoryArchiverTests.cs index 5130fa5522..5fad0a9e86 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/Tests/InventoryArchiverTests.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/Tests/InventoryArchiverTests.cs @@ -514,7 +514,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver.Tests UserAccount ua1 = UserProfileTestUtils.CreateUserWithInventory(scene); Dictionary foldersCreated = new Dictionary(); - List nodesLoaded = new List(); + HashSet nodesLoaded = new HashSet(); string folder1Name = "1"; string folder2aName = "2a"; @@ -529,7 +529,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver.Tests { // Test replication of path1 - new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null) + new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null, false) .ReplicateArchivePathToUserInventory( iarPath1, scene.InventoryService.GetRootFolder(ua1.PrincipalID), foldersCreated, nodesLoaded); @@ -546,7 +546,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver.Tests { // Test replication of path2 - new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null) + new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null, false) .ReplicateArchivePathToUserInventory( iarPath2, scene.InventoryService.GetRootFolder(ua1.PrincipalID), foldersCreated, nodesLoaded); @@ -592,10 +592,10 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver.Tests string itemArchivePath = string.Join("", new string[] { folder1ArchiveName, folder2ArchiveName }); - new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null) + new InventoryArchiveReadRequest(scene, ua1, null, (Stream)null, false) .ReplicateArchivePathToUserInventory( itemArchivePath, scene.InventoryService.GetRootFolder(ua1.PrincipalID), - new Dictionary(), new List()); + new Dictionary(), new HashSet()); List folder1PostCandidates = InventoryArchiveUtils.FindFolderByPath(scene.InventoryService, ua1.PrincipalID, folder1ExistingName); @@ -617,5 +617,45 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver.Tests = InventoryArchiveUtils.FindFolderByPath(scene.InventoryService, folder1Post, "b"); Assert.That(folder2PostCandidates.Count, Is.EqualTo(1)); } + + /// + /// Test replication of a partly existing archive path to the user's inventory. This should create + /// a merged path. + /// + [Test] + public void TestMergeIarPath() + { + TestHelper.InMethod(); + log4net.Config.XmlConfigurator.Configure(); + + Scene scene = SceneSetupHelpers.SetupScene("inventory"); + UserAccount ua1 = UserProfileTestUtils.CreateUserWithInventory(scene); + + string folder1ExistingName = "a"; + string folder2Name = "b"; + + InventoryFolderBase folder1 + = UserInventoryTestUtils.CreateInventoryFolder( + scene.InventoryService, ua1.PrincipalID, folder1ExistingName); + + string folder1ArchiveName = InventoryArchiveWriteRequest.CreateArchiveFolderName(folder1ExistingName, UUID.Random()); + string folder2ArchiveName = InventoryArchiveWriteRequest.CreateArchiveFolderName(folder2Name, UUID.Random()); + + string itemArchivePath = string.Join("", new string[] { folder1ArchiveName, folder2ArchiveName }); + + new InventoryArchiveReadRequest(scene, ua1, folder1ExistingName, (Stream)null, true) + .ReplicateArchivePathToUserInventory( + itemArchivePath, scene.InventoryService.GetRootFolder(ua1.PrincipalID), + new Dictionary(), new HashSet()); + + List folder1PostCandidates + = InventoryArchiveUtils.FindFolderByPath(scene.InventoryService, ua1.PrincipalID, folder1ExistingName); + Assert.That(folder1PostCandidates.Count, Is.EqualTo(1)); + Assert.That(folder1PostCandidates[0].ID, Is.EqualTo(folder1.ID)); + + List folder2PostCandidates + = InventoryArchiveUtils.FindFolderByPath(scene.InventoryService, folder1PostCandidates[0], "b"); + Assert.That(folder2PostCandidates.Count, Is.EqualTo(1)); + } } } \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/Framework/Library/LibraryModule.cs b/OpenSim/Region/CoreModules/Framework/Library/LibraryModule.cs index 36dae6ba9f..9c20d68ce8 100644 --- a/OpenSim/Region/CoreModules/Framework/Library/LibraryModule.cs +++ b/OpenSim/Region/CoreModules/Framework/Library/LibraryModule.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * @@ -173,16 +173,16 @@ namespace OpenSim.Region.CoreModules.Framework.Library m_log.InfoFormat("[LIBRARY MODULE]: Loading library archive {0} ({1})...", iarFileName, simpleName); simpleName = GetInventoryPathFromName(simpleName); - InventoryArchiveReadRequest archread = new InventoryArchiveReadRequest(m_MockScene, uinfo, simpleName, iarFileName); + InventoryArchiveReadRequest archread = new InventoryArchiveReadRequest(m_MockScene, uinfo, simpleName, iarFileName, false); try { - List nodes = archread.Execute(); + HashSet nodes = archread.Execute(); if (nodes != null && nodes.Count == 0) { // didn't find the subfolder with the given name; place it on the top m_log.InfoFormat("[LIBRARY MODULE]: Didn't find {0} in library. Placing archive on the top level", simpleName); archread.Close(); - archread = new InventoryArchiveReadRequest(m_MockScene, uinfo, "/", iarFileName); + archread = new InventoryArchiveReadRequest(m_MockScene, uinfo, "/", iarFileName, false); archread.Execute(); } foreach (InventoryNodeBase node in nodes) diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LS_Api.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LS_Api.cs index fe71ed54cd..5ae64395e7 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LS_Api.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LS_Api.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. *