From 190f9c258b6cd1efda214b2e188903f571e1c6e4 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Thu, 13 Sep 2012 10:00:29 -0700 Subject: [PATCH 1/6] Restarting to work on HGSuitcaseInventoryService: added the ability for the outside world to retrieve appearance items. Not ACLed yet. --- .../HGInventoryAccessModule.cs | 2 + .../HGSuitcaseInventoryService.cs | 60 +++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGInventoryAccessModule.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGInventoryAccessModule.cs index cf72b58832..a0cad40754 100644 --- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGInventoryAccessModule.cs +++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGInventoryAccessModule.cs @@ -308,6 +308,8 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess protected override InventoryItemBase GetItem(UUID agentID, UUID itemID) { InventoryItemBase item = base.GetItem(agentID, itemID); + if (item == null) + return null; string userAssetServer = string.Empty; if (IsForeignUser(agentID, out userAssetServer)) diff --git a/OpenSim/Services/HypergridService/HGSuitcaseInventoryService.cs b/OpenSim/Services/HypergridService/HGSuitcaseInventoryService.cs index 6e4b68c8bd..91cc6ebc66 100644 --- a/OpenSim/Services/HypergridService/HGSuitcaseInventoryService.cs +++ b/OpenSim/Services/HypergridService/HGSuitcaseInventoryService.cs @@ -56,10 +56,12 @@ namespace OpenSim.Services.HypergridService private string m_HomeURL; private IUserAccountService m_UserAccountService; + private IAvatarService m_AvatarService; // private UserAccountCache m_Cache; private ExpiringCache> m_SuitcaseTrees = new ExpiringCache>(); + private ExpiringCache m_Appearances = new ExpiringCache(); public HGSuitcaseInventoryService(IConfigSource config, string configName) : base(config, configName) @@ -77,7 +79,6 @@ namespace OpenSim.Services.HypergridService IConfig invConfig = config.Configs[m_ConfigName]; if (invConfig != null) { - // realm = authConfig.GetString("Realm", realm); string userAccountsDll = invConfig.GetString("UserAccountsService", string.Empty); if (userAccountsDll == string.Empty) throw new Exception("Please specify UserAccountsService in HGInventoryService configuration"); @@ -87,8 +88,14 @@ namespace OpenSim.Services.HypergridService if (m_UserAccountService == null) throw new Exception(String.Format("Unable to create UserAccountService from {0}", userAccountsDll)); - // legacy configuration [obsolete] - m_HomeURL = invConfig.GetString("ProfileServerURI", string.Empty); + string avatarDll = invConfig.GetString("AvatarService", string.Empty); + if (avatarDll == string.Empty) + throw new Exception("Please specify AvatarService in HGInventoryService configuration"); + + m_AvatarService = ServerUtils.LoadPlugin(avatarDll, args); + if (m_AvatarService == null) + throw new Exception(String.Format("Unable to create m_AvatarService from {0}", avatarDll)); + // Preferred m_HomeURL = invConfig.GetString("HomeURI", m_HomeURL); @@ -394,7 +401,7 @@ namespace OpenSim.Services.HypergridService return null; } - if (!IsWithinSuitcaseTree(it.Owner, it.Folder)) + if (!IsWithinSuitcaseTree(it.Owner, it.Folder) && !IsPartOfAppearance(it.Owner, it.ID)) { m_log.DebugFormat("[HG SUITCASE INVENTORY SERVICE]: Item {0} (folder {1}) is not within Suitcase", it.Name, it.Folder); @@ -549,6 +556,51 @@ namespace OpenSim.Services.HypergridService else return true; } #endregion + + #region Avatar Appearance + + private AvatarAppearance GetAppearance(UUID principalID) + { + AvatarAppearance a = null; + if (m_Appearances.TryGetValue(principalID, out a)) + return a; + + a = m_AvatarService.GetAppearance(principalID); + m_Appearances.AddOrUpdate(principalID, a, 5 * 60); // 5minutes + return a; + } + + private bool IsPartOfAppearance(UUID principalID, UUID itemID) + { + AvatarAppearance a = GetAppearance(principalID); + if (a == null) + return false; + + // Check wearables (body parts and clothes) + for (int i = 0; i < a.Wearables.Length; i++) + { + for (int j = 0; j < a.Wearables[i].Count; j++) + { + if (a.Wearables[i][j].ItemID == itemID) + { + m_log.DebugFormat("[HG SUITCASE INVENTORY SERVICE]: item {0} is a wearable", itemID); + return true; + } + } + } + + // Check attachments + if (a.GetAttachmentForItem(itemID) != null) + { + m_log.DebugFormat("[HG SUITCASE INVENTORY SERVICE]: item {0} is an attachment", itemID); + return true; + } + + return false; + } + + #endregion + } } From 6f7825e31046fd3ea4df72147b4d28d8a7a071ce Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Fri, 14 Sep 2012 00:07:39 +0100 Subject: [PATCH 2/6] Don't store the unnecessary VERSIONMIN. VERSIONMAX, METHOD or UserID (present as column PrincipalID) parameters in the Avatars table. These are used to invoke avatar service calls but are not in themselves persistable avatar data. --- .../Handlers/Avatar/AvatarServerPostHandler.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/OpenSim/Server/Handlers/Avatar/AvatarServerPostHandler.cs b/OpenSim/Server/Handlers/Avatar/AvatarServerPostHandler.cs index 393584e932..8cd747ee17 100644 --- a/OpenSim/Server/Handlers/Avatar/AvatarServerPostHandler.cs +++ b/OpenSim/Server/Handlers/Avatar/AvatarServerPostHandler.cs @@ -137,6 +137,8 @@ namespace OpenSim.Server.Handlers.Avatar if (!UUID.TryParse(request["UserID"].ToString(), out user)) return FailureResult(); + RemoveRequestParamsNotForStorage(request); + AvatarData avatar = new AvatarData(request); if (m_AvatarService.SetAvatar(user, avatar)) return SuccessResult(); @@ -153,11 +155,25 @@ namespace OpenSim.Server.Handlers.Avatar if (!UUID.TryParse(request["UserID"].ToString(), out user)) return FailureResult(); + RemoveRequestParamsNotForStorage(request); + if (m_AvatarService.ResetAvatar(user)) return SuccessResult(); return FailureResult(); } + + /// + /// Remove parameters that were used to invoke the method and should not in themselves be persisted. + /// + /// + private void RemoveRequestParamsNotForStorage(Dictionary request) + { + request.Remove("VERSIONMAX"); + request.Remove("VERSIONMIN"); + request.Remove("METHOD"); + request.Remove("UserID"); + } byte[] SetItems(Dictionary request) { @@ -173,6 +189,8 @@ namespace OpenSim.Server.Handlers.Avatar if (!(request["Names"] is List || request["Values"] is List)) return FailureResult(); + RemoveRequestParamsNotForStorage(request); + List _names = (List)request["Names"]; names = _names.ToArray(); List _values = (List)request["Values"]; From ce468215d576cc301a261d85bee9baa68a246ce6 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 24 Jul 2012 19:48:08 +0300 Subject: [PATCH 3/6] Support multi-region OAR files Merged ArchiveWriteRequestPreparation.cs and ArchiveWriteRequestExecution.cs. This simplifies the code, and it's faster to write each scene to the archive as it's found rather than all at once at the end. --- .../RemoteController/RemoteAdminPlugin.cs | 7 + .../Serialization/ArchiveConstants.cs | 5 + OpenSim/Region/Application/OpenSim.cs | 3 +- .../World/Archiver/ArchiveReadRequest.cs | 365 ++++++++++--- .../World/Archiver/ArchiveScenesGroup.cs | 176 ++++++ .../Archiver/ArchiveWriteRequestExecution.cs | 153 ------ .../ArchiveWriteRequestPreparation.cs | 508 ++++++++++++------ .../World/Archiver/ArchiverModule.cs | 5 +- .../World/Archiver/AssetsRequest.cs | 6 + .../World/Archiver/DearchiveScenesGroup.cs | 232 ++++++++ .../World/Archiver/Tests/ArchiverTests.cs | 23 +- .../Framework/Interfaces/IEstateModule.cs | 5 + .../Region/Framework/Scenes/EventManager.cs | 6 +- .../RegionReadyModule/RegionReadyModule.cs | 2 +- 14 files changed, 1088 insertions(+), 408 deletions(-) create mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs delete mode 100644 OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs create mode 100644 OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs diff --git a/OpenSim/ApplicationPlugins/RemoteController/RemoteAdminPlugin.cs b/OpenSim/ApplicationPlugins/RemoteController/RemoteAdminPlugin.cs index 24570d617a..a5e56f9e19 100644 --- a/OpenSim/ApplicationPlugins/RemoteController/RemoteAdminPlugin.cs +++ b/OpenSim/ApplicationPlugins/RemoteController/RemoteAdminPlugin.cs @@ -1369,6 +1369,8 @@ namespace OpenSim.ApplicationPlugins.RemoteController /// profile url /// noassets /// true if no assets should be saved + /// all + /// true to save all the regions in the simulator /// perm /// C and/or T /// @@ -1425,6 +1427,11 @@ namespace OpenSim.ApplicationPlugins.RemoteController options["checkPermissions"] = (string)requestData["perm"]; } + if ((string)requestData["all"] == "true") + { + options["all"] = (string)requestData["all"]; + } + IRegionArchiverModule archiver = scene.RequestModuleInterface(); if (archiver != null) diff --git a/OpenSim/Framework/Serialization/ArchiveConstants.cs b/OpenSim/Framework/Serialization/ArchiveConstants.cs index 2c5e0018ec..48f1c4f94f 100644 --- a/OpenSim/Framework/Serialization/ArchiveConstants.cs +++ b/OpenSim/Framework/Serialization/ArchiveConstants.cs @@ -52,6 +52,11 @@ namespace OpenSim.Framework.Serialization /// public const string INVENTORY_PATH = "inventory/"; + /// + /// Path for regions in a multi-region archive + /// + public const string REGIONS_PATH = "regions/"; + /// /// Path for the prims file /// diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs index ed339fdf6f..c3c612fb05 100644 --- a/OpenSim/Region/Application/OpenSim.cs +++ b/OpenSim/Region/Application/OpenSim.cs @@ -292,7 +292,7 @@ namespace OpenSim m_console.Commands.AddCommand("Archiving", false, "save oar", //"save oar [-v|--version=] [-p|--profile=] []", - "save oar [-h|--home=] [--noassets] [--publish] [--perm=] []", + "save oar [-h|--home=] [--noassets] [--publish] [--perm=] [--all] []", "Save a region's data to an OAR archive.", // "-v|--version= generates scene objects as per older versions of the serialization (e.g. -v=0)" + Environment.NewLine "-h|--home= adds the url of the profile service to the saved user information.\n" @@ -302,6 +302,7 @@ namespace OpenSim + " this is useful if you're making oars generally available that might be reloaded to the same grid from which you published\n" + "--perm= stops objects with insufficient permissions from being saved to the OAR.\n" + " can contain one or more of these characters: \"C\" = Copy, \"T\" = Transfer\n" + + "--all saves all the regions in the simulator, instead of just the current region.\n" + "The OAR path must be a filesystem path." + " If this is not given then the oar is saved to region.oar in the current directory.", SaveOar); diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs index 433166d848..a6923ef48d 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs @@ -43,6 +43,7 @@ using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; using OpenSim.Region.Framework.Scenes.Serialization; using OpenSim.Services.Interfaces; +using System.Threading; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -52,7 +53,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver public class ArchiveReadRequest { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// Contains data used while dearchiving a single scene. + /// + private class DearchiveContext + { + public Scene Scene { get; set; } + + public List SerialisedSceneObjects { get; set; } + + public List SerialisedParcels { get; set; } + + public List SceneObjects { get; set; } + + public DearchiveContext(Scene scene) + { + Scene = scene; + SerialisedSceneObjects = new List(); + SerialisedParcels = new List(); + SceneObjects = new List(); + } + } + /// /// The maximum major version of OAR that we can read. Minor versions shouldn't need a max number since version /// bumps here should be compatible. @@ -62,9 +86,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Has the control file been loaded for this archive? /// - public bool ControlFileLoaded { get; private set; } + public bool ControlFileLoaded { get; private set; } - protected Scene m_scene; + protected string m_loadPath; + protected Scene m_rootScene; protected Stream m_loadStream; protected Guid m_requestId; protected string m_errorMessage; @@ -91,7 +116,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver { if (m_UserMan == null) { - m_UserMan = m_scene.RequestModuleInterface(); + m_UserMan = m_rootScene.RequestModuleInterface(); } return m_UserMan; } @@ -104,10 +129,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver private IGroupsModule m_groupsModule; + private IAssetService m_assetService = null; + + public ArchiveReadRequest(Scene scene, string loadPath, bool merge, bool skipAssets, Guid requestId) { - m_scene = scene; + m_rootScene = scene; + m_loadPath = loadPath; try { m_loadStream = new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress); @@ -128,12 +157,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Zero can never be a valid user id m_validUserUuids[UUID.Zero] = false; - m_groupsModule = m_scene.RequestModuleInterface(); + m_groupsModule = m_rootScene.RequestModuleInterface(); + m_assetService = m_rootScene.AssetService; } public ArchiveReadRequest(Scene scene, Stream loadStream, bool merge, bool skipAssets, Guid requestId) { - m_scene = scene; + m_rootScene = scene; + m_loadPath = null; m_loadStream = loadStream; m_merge = merge; m_skipAssets = skipAssets; @@ -142,33 +173,34 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Zero can never be a valid user id m_validUserUuids[UUID.Zero] = false; - m_groupsModule = m_scene.RequestModuleInterface(); + m_groupsModule = m_rootScene.RequestModuleInterface(); + m_assetService = m_rootScene.AssetService; } /// /// Dearchive the region embodied in this request. /// public void DearchiveRegion() - { - // The same code can handle dearchiving 0.1 and 0.2 OpenSim Archive versions - DearchiveRegion0DotStar(); - } - - private void DearchiveRegion0DotStar() { int successfulAssetRestores = 0; int failedAssetRestores = 0; - List serialisedSceneObjects = new List(); - List serialisedParcels = new List(); - string filePath = "NONE"; - TarArchiveReader archive = new TarArchiveReader(m_loadStream); + DearchiveScenesInfo dearchivedScenes; + + // We dearchive all the scenes at once, because the files in the TAR archive might be mixed. + // Therefore, we have to keep track of the dearchive context of all the scenes. + Dictionary sceneContexts = new Dictionary(); + + string fullPath = "NONE"; + TarArchiveReader archive = null; byte[] data; TarArchiveReader.TarEntryType entryType; - + try { - while ((data = archive.ReadEntry(out filePath, out entryType)) != null) + FindAndLoadControlFile(out archive, out dearchivedScenes); + + while ((data = archive.ReadEntry(out fullPath, out entryType)) != null) { //m_log.DebugFormat( // "[ARCHIVER]: Successfully read {0} ({1} bytes)", filePath, data.Length); @@ -176,9 +208,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; + + // Find the scene that this file belongs to + + Scene scene; + string filePath; + if (!dearchivedScenes.GetRegionFromPath(fullPath, out scene, out filePath)) + continue; // this file belongs to a region that we're not loading + + DearchiveContext sceneContext = null; + if (scene != null) + { + if (!sceneContexts.TryGetValue(scene.RegionInfo.RegionID, out sceneContext)) + { + sceneContext = new DearchiveContext(scene); + sceneContexts.Add(scene.RegionInfo.RegionID, sceneContext); + } + } + + + // Process the file + if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) { - serialisedSceneObjects.Add(Encoding.UTF8.GetString(data)); + sceneContext.SerialisedSceneObjects.Add(Encoding.UTF8.GetString(data)); } else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH) && !m_skipAssets) { @@ -192,19 +245,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver } else if (!m_merge && filePath.StartsWith(ArchiveConstants.TERRAINS_PATH)) { - LoadTerrain(filePath, data); + LoadTerrain(scene, filePath, data); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.SETTINGS_PATH)) { - LoadRegionSettings(filePath, data); + LoadRegionSettings(scene, filePath, data, dearchivedScenes); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.LANDDATA_PATH)) { - serialisedParcels.Add(Encoding.UTF8.GetString(data)); + sceneContext.SerialisedParcels.Add(Encoding.UTF8.GetString(data)); } else if (filePath == ArchiveConstants.CONTROL_FILE_PATH) { - LoadControlFile(filePath, data); + // Ignore, because we already read the control file } } @@ -212,15 +265,16 @@ namespace OpenSim.Region.CoreModules.World.Archiver } catch (Exception e) { - m_log.ErrorFormat( - "[ARCHIVER]: Aborting load with error in archive file {0}. {1}", filePath, e); + m_log.Error( + String.Format("[ARCHIVER]: Aborting load with error in archive file {0} ", fullPath), e); m_errorMessage += e.ToString(); - m_scene.EventManager.TriggerOarFileLoaded(m_requestId, m_errorMessage); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, new List(), m_errorMessage); return; } finally { - archive.Close(); + if (archive != null) + archive.Close(); } if (!m_skipAssets) @@ -234,32 +288,143 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - if (!m_merge) + foreach (DearchiveContext sceneContext in sceneContexts.Values) { - m_log.Info("[ARCHIVER]: Clearing all existing scene objects"); - m_scene.DeleteAllSceneObjects(); + m_log.InfoFormat("[ARCHIVER:] Loading region {0}", sceneContext.Scene.RegionInfo.RegionName); + + if (!m_merge) + { + m_log.Info("[ARCHIVER]: Clearing all existing scene objects"); + sceneContext.Scene.DeleteAllSceneObjects(); + } + + try + { + LoadParcels(sceneContext.Scene, sceneContext.SerialisedParcels); + LoadObjects(sceneContext.Scene, sceneContext.SerialisedSceneObjects, sceneContext.SceneObjects); + + // Inform any interested parties that the region has changed. We waited until now so that all + // of the region's objects will be loaded when we send this notification. + IEstateModule estateModule = sceneContext.Scene.RequestModuleInterface(); + if (estateModule != null) + estateModule.TriggerRegionInfoChange(); + } + catch (Exception e) + { + m_log.Error("[ARCHIVER]: Error loading parcels or objects ", e); + m_errorMessage += e.ToString(); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, new List(), m_errorMessage); + return; + } } - LoadParcels(serialisedParcels); - LoadObjects(serialisedSceneObjects); + // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so + // that users can enter the scene. If we allow the scripts to start in the loop above + // then they significantly increase the time until the OAR finishes loading. + Util.FireAndForget(delegate(object o) + { + Thread.Sleep(15000); + m_log.Info("Starting scripts in scene objects"); + + foreach (DearchiveContext sceneContext in sceneContexts.Values) + { + foreach (SceneObjectGroup sceneObject in sceneContext.SceneObjects) + { + sceneObject.CreateScriptInstances(0, false, sceneContext.Scene.DefaultScriptEngine, 0); // StateSource.RegionStart + sceneObject.ResumeScripts(); + } + + sceneContext.SceneObjects.Clear(); + } + }); m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive"); - m_scene.EventManager.TriggerOarFileLoaded(m_requestId, m_errorMessage); + m_rootScene.EventManager.TriggerOarFileLoaded(m_requestId, dearchivedScenes.GetLoadedScenes(), m_errorMessage); + } + + /// + /// Searches through the files in the archive for the control file, and reads it. + /// We must read the control file first, in order to know which regions are available. + /// + /// + /// In most cases the control file *is* first, since that's how we create archives. However, + /// it's possible that someone rewrote the archive externally so we can't rely on this fact. + /// + /// + /// + private void FindAndLoadControlFile(out TarArchiveReader archive, out DearchiveScenesInfo dearchivedScenes) + { + archive = new TarArchiveReader(m_loadStream); + dearchivedScenes = new DearchiveScenesInfo(); + + string filePath; + byte[] data; + TarArchiveReader.TarEntryType entryType; + bool firstFile = true; + + while ((data = archive.ReadEntry(out filePath, out entryType)) != null) + { + if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) + continue; + + if (filePath == ArchiveConstants.CONTROL_FILE_PATH) + { + LoadControlFile(filePath, data, dearchivedScenes); + + // Find which scenes are available in the simulator + ArchiveScenesGroup simulatorScenes = new ArchiveScenesGroup(); + SceneManager.Instance.ForEachScene(delegate(Scene scene2) + { + simulatorScenes.AddScene(scene2); + }); + simulatorScenes.CalcSceneLocations(); + dearchivedScenes.SetSimulatorScenes(m_rootScene, simulatorScenes); + + // If the control file wasn't the first file then reset the read pointer + if (!firstFile) + { + m_log.Warn("Control file wasn't the first file in the archive"); + if (m_loadStream.CanSeek) + { + m_loadStream.Seek(0, SeekOrigin.Begin); + } + else if (m_loadPath != null) + { + archive.Close(); + archive = null; + m_loadStream.Close(); + m_loadStream = null; + m_loadStream = new GZipStream(ArchiveHelpers.GetStream(m_loadPath), CompressionMode.Decompress); + archive = new TarArchiveReader(m_loadStream); + } + else + { + // There isn't currently a scenario where this happens, but it's best to add a check just in case + throw new Exception("Error reading archive: control file wasn't the first file, and the input stream doesn't allow seeking"); + } + } + + return; + } + + firstFile = false; + } + + throw new Exception("Control file not found"); } /// /// Load serialized scene objects. /// - /// - protected void LoadObjects(List serialisedSceneObjects) + protected void LoadObjects(Scene scene, List serialisedSceneObjects, List sceneObjects) { // Reload serialized prims m_log.InfoFormat("[ARCHIVER]: Loading {0} scene objects. Please wait.", serialisedSceneObjects.Count); - UUID oldTelehubUUID = m_scene.RegionInfo.RegionSettings.TelehubObject; + UUID oldTelehubUUID = scene.RegionInfo.RegionSettings.TelehubObject; - IRegionSerialiserModule serialiser = m_scene.RequestModuleInterface(); + IRegionSerialiserModule serialiser = scene.RequestModuleInterface(); int sceneObjectsLoadedCount = 0; foreach (string serialisedSceneObject in serialisedSceneObjects) @@ -280,7 +445,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver SceneObjectGroup sceneObject = serialiser.DeserializeGroupFromXml2(serialisedSceneObject); - bool isTelehub = (sceneObject.UUID == oldTelehubUUID); + bool isTelehub = (sceneObject.UUID == oldTelehubUUID) && (oldTelehubUUID != UUID.Zero); // For now, give all incoming scene objects new uuids. This will allow scenes to be cloned // on the same region server and multiple examples a single object archive to be imported @@ -290,8 +455,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (isTelehub) { // Change the Telehub Object to the new UUID - m_scene.RegionInfo.RegionSettings.TelehubObject = sceneObject.UUID; - m_scene.RegionInfo.RegionSettings.Save(); + scene.RegionInfo.RegionSettings.TelehubObject = sceneObject.UUID; + scene.RegionInfo.RegionSettings.Save(); oldTelehubUUID = UUID.Zero; } @@ -301,17 +466,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver { if (part.CreatorData == null || part.CreatorData == string.Empty) { - if (!ResolveUserUuid(part.CreatorID)) - part.CreatorID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.CreatorID)) + part.CreatorID = scene.RegionInfo.EstateSettings.EstateOwner; } if (UserManager != null) UserManager.AddUser(part.CreatorID, part.CreatorData); - if (!ResolveUserUuid(part.OwnerID)) - part.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.OwnerID)) + part.OwnerID = scene.RegionInfo.EstateSettings.EstateOwner; - if (!ResolveUserUuid(part.LastOwnerID)) - part.LastOwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, part.LastOwnerID)) + part.LastOwnerID = scene.RegionInfo.EstateSettings.EstateOwner; if (!ResolveGroupUuid(part.GroupID)) part.GroupID = UUID.Zero; @@ -328,15 +493,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver TaskInventoryDictionary inv = part.TaskInventory; foreach (KeyValuePair kvp in inv) { - if (!ResolveUserUuid(kvp.Value.OwnerID)) + if (!ResolveUserUuid(scene, kvp.Value.OwnerID)) { - kvp.Value.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + kvp.Value.OwnerID = scene.RegionInfo.EstateSettings.EstateOwner; } if (kvp.Value.CreatorData == null || kvp.Value.CreatorData == string.Empty) { - if (!ResolveUserUuid(kvp.Value.CreatorID)) - kvp.Value.CreatorID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, kvp.Value.CreatorID)) + kvp.Value.CreatorID = scene.RegionInfo.EstateSettings.EstateOwner; } if (UserManager != null) @@ -348,10 +513,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - if (m_scene.AddRestoredSceneObject(sceneObject, true, false)) + if (scene.AddRestoredSceneObject(sceneObject, true, false)) { sceneObjectsLoadedCount++; - sceneObject.CreateScriptInstances(0, false, m_scene.DefaultScriptEngine, 0); + sceneObject.CreateScriptInstances(0, false, scene.DefaultScriptEngine, 0); sceneObject.ResumeScripts(); } } @@ -366,16 +531,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (oldTelehubUUID != UUID.Zero) { m_log.WarnFormat("Telehub object not found: {0}", oldTelehubUUID); - m_scene.RegionInfo.RegionSettings.TelehubObject = UUID.Zero; - m_scene.RegionInfo.RegionSettings.ClearSpawnPoints(); + scene.RegionInfo.RegionSettings.TelehubObject = UUID.Zero; + scene.RegionInfo.RegionSettings.ClearSpawnPoints(); } } /// /// Load serialized parcels. /// + /// /// - protected void LoadParcels(List serialisedParcels) + protected void LoadParcels(Scene scene, List serialisedParcels) { // Reload serialized parcels m_log.InfoFormat("[ARCHIVER]: Loading {0} parcels. Please wait.", serialisedParcels.Count); @@ -386,8 +552,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Validate User and Group UUID's - if (!ResolveUserUuid(parcel.OwnerID)) - parcel.OwnerID = m_scene.RegionInfo.EstateSettings.EstateOwner; + if (!ResolveUserUuid(scene, parcel.OwnerID)) + parcel.OwnerID = m_rootScene.RegionInfo.EstateSettings.EstateOwner; if (!ResolveGroupUuid(parcel.GroupID)) { @@ -398,7 +564,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver List accessList = new List(); foreach (LandAccessEntry entry in parcel.ParcelAccessList) { - if (ResolveUserUuid(entry.AgentID)) + if (ResolveUserUuid(scene, entry.AgentID)) accessList.Add(entry); // else, drop this access rule } @@ -414,23 +580,24 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (!m_merge) { bool setupDefaultParcel = (landData.Count == 0); - m_scene.LandChannel.Clear(setupDefaultParcel); + scene.LandChannel.Clear(setupDefaultParcel); } - m_scene.EventManager.TriggerIncomingLandDataFromStorage(landData); + scene.EventManager.TriggerIncomingLandDataFromStorage(landData); m_log.InfoFormat("[ARCHIVER]: Restored {0} parcels.", landData.Count); } /// /// Look up the given user id to check whether it's one that is valid for this grid. /// + /// /// /// - private bool ResolveUserUuid(UUID uuid) + private bool ResolveUserUuid(Scene scene, UUID uuid) { if (!m_validUserUuids.ContainsKey(uuid)) { - UserAccount account = m_scene.UserAccountService.GetUserAccount(m_scene.RegionInfo.ScopeID, uuid); + UserAccount account = scene.UserAccountService.GetUserAccount(scene.RegionInfo.ScopeID, uuid); m_validUserUuids.Add(uuid, account != null); } @@ -485,7 +652,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver string extension = filename.Substring(i); string uuid = filename.Remove(filename.Length - extension.Length); - if (m_scene.AssetService.GetMetadata(uuid) != null) + if (m_assetService.GetMetadata(uuid) != null) { // m_log.DebugFormat("[ARCHIVER]: found existing asset {0}",uuid); return true; @@ -505,7 +672,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We're relying on the asset service to do the sensible thing and not store the asset if it already // exists. - m_scene.AssetService.Store(asset); + m_assetService.Store(asset); /** * Create layers on decode for image assets. This is likely to significantly increase the time to load archives so @@ -533,12 +700,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Load region settings data /// + /// /// /// + /// /// /// true if settings were loaded successfully, false otherwise /// - private bool LoadRegionSettings(string settingsPath, byte[] data) + private bool LoadRegionSettings(Scene scene, string settingsPath, byte[] data, DearchiveScenesInfo dearchivedScenes) { RegionSettings loadedRegionSettings; @@ -554,7 +723,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver return false; } - RegionSettings currentRegionSettings = m_scene.RegionInfo.RegionSettings; + RegionSettings currentRegionSettings = scene.RegionInfo.RegionSettings; currentRegionSettings.AgentLimit = loadedRegionSettings.AgentLimit; currentRegionSettings.AllowDamage = loadedRegionSettings.AllowDamage; @@ -591,12 +760,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver foreach (SpawnPoint sp in loadedRegionSettings.SpawnPoints()) currentRegionSettings.AddSpawnPoint(sp); + currentRegionSettings.LoadedCreationDateTime = dearchivedScenes.LoadedCreationDateTime; + currentRegionSettings.LoadedCreationID = dearchivedScenes.GetOriginalRegionID(scene.RegionInfo.RegionID).ToString(); + currentRegionSettings.Save(); - m_scene.TriggerEstateSunUpdate(); + scene.TriggerEstateSunUpdate(); - IEstateModule estateModule = m_scene.RequestModuleInterface(); - + IEstateModule estateModule = scene.RequestModuleInterface(); if (estateModule != null) estateModule.sendRegionHandshakeToAll(); @@ -606,14 +777,15 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Load terrain data /// + /// /// /// /// /// true if terrain was resolved successfully, false otherwise. /// - private bool LoadTerrain(string terrainPath, byte[] data) + private bool LoadTerrain(Scene scene, string terrainPath, byte[] data) { - ITerrainModule terrainModule = m_scene.RequestModuleInterface(); + ITerrainModule terrainModule = scene.RequestModuleInterface(); MemoryStream ms = new MemoryStream(data); terrainModule.LoadFromStream(terrainPath, ms); @@ -629,17 +801,18 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// /// - public void LoadControlFile(string path, byte[] data) + /// + public DearchiveScenesInfo LoadControlFile(string path, byte[] data, DearchiveScenesInfo dearchivedScenes) { XmlNamespaceManager nsmgr = new XmlNamespaceManager(new NameTable()); XmlParserContext context = new XmlParserContext(null, nsmgr, null, XmlSpace.None); XmlTextReader xtr = new XmlTextReader(Encoding.ASCII.GetString(data), XmlNodeType.Document, context); - RegionSettings currentRegionSettings = m_scene.RegionInfo.RegionSettings; + // Loaded metadata will be empty if no information exists in the archive + dearchivedScenes.LoadedCreationDateTime = 0; + dearchivedScenes.DefaultOriginalID = ""; - // Loaded metadata will empty if no information exists in the archive - currentRegionSettings.LoadedCreationDateTime = 0; - currentRegionSettings.LoadedCreationID = ""; + bool multiRegion = false; while (xtr.Read()) { @@ -665,18 +838,44 @@ namespace OpenSim.Region.CoreModules.World.Archiver { int value; if (Int32.TryParse(xtr.ReadElementContentAsString(), out value)) - currentRegionSettings.LoadedCreationDateTime = value; + dearchivedScenes.LoadedCreationDateTime = value; } - else if (xtr.Name.ToString() == "id") + else if (xtr.Name.ToString() == "row") { - currentRegionSettings.LoadedCreationID = xtr.ReadElementContentAsString(); + multiRegion = true; + dearchivedScenes.StartRow(); + } + else if (xtr.Name.ToString() == "region") + { + dearchivedScenes.StartRegion(); + } + else if (xtr.Name.ToString() == "id") + { + string id = xtr.ReadElementContentAsString(); + dearchivedScenes.DefaultOriginalID = id; + if (multiRegion) + dearchivedScenes.SetRegionOriginalID(id); + } + else if (xtr.Name.ToString() == "dir") + { + dearchivedScenes.SetRegionDirectory(xtr.ReadElementContentAsString()); } } } - - currentRegionSettings.Save(); - + + dearchivedScenes.MultiRegionFormat = multiRegion; + if (!multiRegion) + { + // Add the single scene + dearchivedScenes.StartRow(); + dearchivedScenes.StartRegion(); + dearchivedScenes.SetRegionOriginalID(dearchivedScenes.DefaultOriginalID); + dearchivedScenes.SetRegionDirectory(""); + } + ControlFileLoaded = true; + + return dearchivedScenes; } } } \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs new file mode 100644 index 0000000000..a66ed88ad1 --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveScenesGroup.cs @@ -0,0 +1,176 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using OpenSim.Region.Framework.Scenes; +using OpenMetaverse; +using System.Drawing; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// A group of regions arranged in a rectangle, possibly with holes. + /// + /// + /// The regions usually (but not necessarily) belong to an archive file, in which case we + /// store additional information used to create the archive (e.g., each region's + /// directory within the archive). + /// + public class ArchiveScenesGroup + { + /// + /// All the regions. The outer dictionary contains rows (key: Y coordinate). + /// The inner dictionaries contain each row's regions (key: X coordinate). + /// + public SortedDictionary> Regions { get; set; } + + /// + /// The subdirectory where each region is stored in the archive. + /// + protected Dictionary m_regionDirs; + + /// + /// The grid coordinates of the regions' bounding box. + /// + public Rectangle Rect { get; set; } + + + public ArchiveScenesGroup() + { + Regions = new SortedDictionary>(); + m_regionDirs = new Dictionary(); + Rect = new Rectangle(0, 0, 0, 0); + } + + public void AddScene(Scene scene) + { + uint x = scene.RegionInfo.RegionLocX; + uint y = scene.RegionInfo.RegionLocY; + + SortedDictionary row; + if (!Regions.TryGetValue(y, out row)) + { + row = new SortedDictionary(); + Regions[y] = row; + } + + row[x] = scene; + } + + /// + /// Called after all the scenes have been added. Performs calculations that require + /// knowledge of all the scenes. + /// + public void CalcSceneLocations() + { + if (Regions.Count == 0) + return; + + // Find the bounding rectangle + + uint firstY = Regions.First().Key; + uint lastY = Regions.Last().Key; + + uint? firstX = null; + uint? lastX = null; + + foreach (SortedDictionary row in Regions.Values) + { + uint curFirstX = row.First().Key; + uint curLastX = row.Last().Key; + + firstX = (firstX == null) ? curFirstX : (firstX < curFirstX) ? firstX : curFirstX; + lastX = (lastX == null) ? curLastX : (lastX > curLastX) ? lastX : curLastX; + } + + Rect = new Rectangle((int)firstX, (int)firstY, (int)(lastY - firstY + 1), (int)(lastX - firstX + 1)); + + + // Calculate the subdirectory in which each region will be stored in the archive + + m_regionDirs.Clear(); + ForEachScene(delegate(Scene scene) + { + // We add the region's coordinates to ensure uniqueness even if multiple regions have the same name + string path = string.Format("{0}_{1}_{2}", + scene.RegionInfo.RegionLocX - Rect.X + 1, + scene.RegionInfo.RegionLocY - Rect.Y + 1, + scene.RegionInfo.RegionName.Replace(' ', '_')); + m_regionDirs[scene.RegionInfo.RegionID] = path; + }); + } + + /// + /// Returns the subdirectory where the region is stored. + /// + /// + /// + public string GetRegionDir(UUID regionID) + { + return m_regionDirs[regionID]; + } + + /// + /// Performs an action on all the scenes in this order: rows from South to North, + /// and within each row West to East. + /// + /// + public void ForEachScene(Action action) + { + foreach (SortedDictionary row in Regions.Values) + { + foreach (Scene scene in row.Values) + { + action(scene); + } + } + } + + /// + /// Returns the scene at position 'location'. + /// + /// A location in the grid + /// The scene at this location + /// Whether the scene was found + public bool TryGetScene(Point location, out Scene scene) + { + SortedDictionary row; + if (Regions.TryGetValue((uint)location.Y, out row)) + { + if (row.TryGetValue((uint)location.X, out scene)) + return true; + } + + scene = null; + return false; + } + + } +} diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs deleted file mode 100644 index 0780d86ed5..0000000000 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Xml; -using log4net; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Framework.Serialization; -using OpenSim.Framework.Serialization.External; -using OpenSim.Region.CoreModules.World.Terrain; -using OpenSim.Region.Framework.Interfaces; -using OpenSim.Region.Framework.Scenes; - -namespace OpenSim.Region.CoreModules.World.Archiver -{ - /// - /// Method called when all the necessary assets for an archive request have been received. - /// - public delegate void AssetsRequestCallback( - ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); - - /// - /// Execute the write of an archive once we have received all the necessary data - /// - public class ArchiveWriteRequestExecution - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - protected ITerrainModule m_terrainModule; - protected IRegionSerialiserModule m_serialiser; - protected List m_sceneObjects; - protected Scene m_scene; - protected TarArchiveWriter m_archiveWriter; - protected Guid m_requestId; - protected Dictionary m_options; - - public ArchiveWriteRequestExecution( - List sceneObjects, - ITerrainModule terrainModule, - IRegionSerialiserModule serialiser, - Scene scene, - TarArchiveWriter archiveWriter, - Guid requestId, - Dictionary options) - { - m_sceneObjects = sceneObjects; - m_terrainModule = terrainModule; - m_serialiser = serialiser; - m_scene = scene; - m_archiveWriter = archiveWriter; - m_requestId = requestId; - m_options = options; - } - - protected internal void ReceivedAllAssets( - ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) - { - try - { - Save(assetsFoundUuids, assetsNotFoundUuids); - } - finally - { - m_archiveWriter.Close(); - } - - m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_scene.RegionInfo.RegionName); - - m_scene.EventManager.TriggerOarFileSaved(m_requestId, String.Empty); - } - - protected internal void Save(ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) - { - foreach (UUID uuid in assetsNotFoundUuids) - { - m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); - } - -// m_log.InfoFormat( -// "[ARCHIVER]: Received {0} of {1} assets requested", -// assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); - - m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); - - // Write out region settings - string settingsPath - = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); - m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); - - m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); - - // Write out land data (aka parcel) settings - ListlandObjects = m_scene.LandChannel.AllParcels(); - foreach (ILandObject lo in landObjects) - { - LandData landData = lo.LandData; - string landDataPath = String.Format("{0}{1}.xml", ArchiveConstants.LANDDATA_PATH, - landData.GlobalID.ToString()); - m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); - } - - m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); - - // Write out terrain - string terrainPath - = String.Format("{0}{1}.r32", ArchiveConstants.TERRAINS_PATH, m_scene.RegionInfo.RegionName); - - MemoryStream ms = new MemoryStream(); - m_terrainModule.SaveToStream(terrainPath, ms); - m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); - ms.Close(); - - m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); - - // Write out scene object metadata - foreach (SceneObjectGroup sceneObject in m_sceneObjects) - { - //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); - - string serializedObject = m_serialiser.SerializeGroupToXml2(sceneObject, m_options); - m_archiveWriter.WriteFile(ArchiveHelpers.CreateObjectPath(sceneObject), serializedObject); - } - } - } -} \ No newline at end of file diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index 4edaaca594..2c34f4ba9f 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs @@ -43,6 +43,7 @@ using OpenSim.Region.Framework.Scenes; using Ionic.Zlib; using GZipStream = Ionic.Zlib.GZipStream; using CompressionMode = Ionic.Zlib.CompressionMode; +using OpenSim.Framework.Serialization.External; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -61,17 +62,29 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// The maximum major version of OAR that we can write. /// - public static int MAX_MAJOR_VERSION = 0; + public static int MAX_MAJOR_VERSION = 1; + + /// + /// Whether we're saving a multi-region archive. + /// + public bool MultiRegionFormat { get; set; } /// /// Determine whether this archive will save assets. Default is true. /// public bool SaveAssets { get; set; } - protected ArchiverModule m_module; - protected Scene m_scene; + /// + /// Determines which objects will be included in the archive, according to their permissions. + /// Default is null, meaning no permission checks. + /// + public string CheckPermissions { get; set; } + + protected Scene m_rootScene; protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; protected Guid m_requestId; + protected Dictionary m_options; /// /// Constructor @@ -82,7 +95,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// If there was a problem opening a stream for the file specified by the savePath /// - public ArchiveWriteRequestPreparation(ArchiverModule module, string savePath, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, string savePath, Guid requestId) : this(scene, requestId) { try { @@ -100,26 +113,23 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Constructor. /// - /// Calling module + /// The root scene to archive /// The stream to which to save data. /// The id associated with this request - public ArchiveWriteRequestPreparation(ArchiverModule module, Stream saveStream, Guid requestId) : this(module, requestId) + public ArchiveWriteRequestPreparation(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) { m_saveStream = saveStream; } - protected ArchiveWriteRequestPreparation(ArchiverModule module, Guid requestId) + protected ArchiveWriteRequestPreparation(Scene scene, Guid requestId) { - m_module = module; - - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this. - if (m_module != null) - m_scene = m_module.Scene; - + m_rootScene = scene; m_requestId = requestId; + m_archiveWriter = null; + MultiRegionFormat = false; SaveAssets = true; + CheckPermissions = null; } /// @@ -128,126 +138,157 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// if there was an io problem with creating the file public void ArchiveRegion(Dictionary options) { + m_options = options; + + if (options.ContainsKey("all") && (bool)options["all"]) + MultiRegionFormat = true; + if (options.ContainsKey("noassets") && (bool)options["noassets"]) SaveAssets = false; + Object temp; + if (options.TryGetValue("checkPermissions", out temp)) + CheckPermissions = (string)temp; + + + // Find the regions to archive + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + if (MultiRegionFormat) + { + m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + } + else + { + scenesGroup.AddScene(m_rootScene); + } + scenesGroup.CalcSceneLocations(); + + + m_archiveWriter = new TarArchiveWriter(m_saveStream); + try { + // Write out control file. It should be first so that it will be found ASAP when loading the file. + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup)); + m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); + + // Archive the regions + Dictionary assetUuids = new Dictionary(); - - EntityBase[] entities = m_scene.GetEntities(); - List sceneObjects = new List(); - string checkPermissions = null; - int numObjectsSkippedPermissions = 0; - Object temp; - if (options.TryGetValue("checkPermissions", out temp)) - checkPermissions = (string)temp; - - // Filter entities so that we only have scene objects. - // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods - // end up having to do this - foreach (EntityBase entity in entities) + scenesGroup.ForEachScene(delegate(Scene scene) { - if (entity is SceneObjectGroup) - { - SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : ""; + ArchiveOneRegion(scene, regionDir, assetUuids); + }); - if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) - { - if (!CanUserArchiveObject(m_scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, checkPermissions)) - { - // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. - ++numObjectsSkippedPermissions; - } - else - { - sceneObjects.Add(sceneObject); - } - } - } - } + // Archive the assets if (SaveAssets) { - UuidGatherer assetGatherer = new UuidGatherer(m_scene.AssetService); + m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count); - foreach (SceneObjectGroup sceneObject in sceneObjects) - { - assetGatherer.GatherAssetUuids(sceneObject, assetUuids); - } + // Asynchronously request all the assets required to perform this archive operation + AssetsRequest ar + = new AssetsRequest( + new AssetsArchiver(m_archiveWriter), assetUuids, + m_rootScene.AssetService, m_rootScene.UserAccountService, + m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", - sceneObjects.Count, assetUuids.Count); + Util.FireAndForget(o => ar.Execute()); + + // CloseArchive() will be called from ReceivedAllAssets() } else { m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified"); - } - - if (numObjectsSkippedPermissions > 0) - { - m_log.DebugFormat( - "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", - numObjectsSkippedPermissions); - } - - // Make sure that we also request terrain texture assets - RegionSettings regionSettings = m_scene.RegionInfo.RegionSettings; - - if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) - assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; - - if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) - assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; - - if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) - assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; - - if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) - assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; - - TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); - - // Asynchronously request all the assets required to perform this archive operation - ArchiveWriteRequestExecution awre - = new ArchiveWriteRequestExecution( - sceneObjects, - m_scene.RequestModuleInterface(), - m_scene.RequestModuleInterface(), - m_scene, - archiveWriter, - m_requestId, - options); - - m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - - // Write out control file. This has to be done first so that subsequent loaders will see this file first - // XXX: I know this is a weak way of doing it since external non-OAR aware tar executables will not do this - archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(options)); - m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); - - if (SaveAssets) - { - AssetsRequest ar - = new AssetsRequest( - new AssetsArchiver(archiveWriter), assetUuids, - m_scene.AssetService, m_scene.UserAccountService, - m_scene.RegionInfo.ScopeID, options, awre.ReceivedAllAssets); - - Util.FireAndForget(o => ar.Execute()); - } - else - { - awre.ReceivedAllAssets(new List(), new List()); + CloseArchive(string.Empty); } } - catch (Exception) + catch (Exception e) { - m_saveStream.Close(); + CloseArchive(e.Message); throw; - } + } + } + + + private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) + { + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); + + EntityBase[] entities = scene.GetEntities(); + List sceneObjects = new List(); + + int numObjectsSkippedPermissions = 0; + + // Filter entities so that we only have scene objects. + // FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods + // end up having to do this + IPermissionsModule permissionsModule = scene.RequestModuleInterface(); + foreach (EntityBase entity in entities) + { + if (entity is SceneObjectGroup) + { + SceneObjectGroup sceneObject = (SceneObjectGroup)entity; + + if (!sceneObject.IsDeleted && !sceneObject.IsAttachment) + { + if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, CheckPermissions, permissionsModule)) + { + // The user isn't allowed to copy/transfer this object, so it will not be included in the OAR. + ++numObjectsSkippedPermissions; + } + else + { + sceneObjects.Add(sceneObject); + } + } + } + } + + if (SaveAssets) + { + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + int prevAssets = assetUuids.Count; + + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + assetGatherer.GatherAssetUuids(sceneObject, assetUuids); + } + + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", + sceneObjects.Count, assetUuids.Count - prevAssets); + } + + if (numObjectsSkippedPermissions > 0) + { + m_log.DebugFormat( + "[ARCHIVER]: {0} scene objects skipped due to lack of permissions", + numObjectsSkippedPermissions); + } + + // Make sure that we also request terrain texture assets + RegionSettings regionSettings = scene.RegionInfo.RegionSettings; + + if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1) + assetUuids[regionSettings.TerrainTexture1] = AssetType.Texture; + + if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2) + assetUuids[regionSettings.TerrainTexture2] = AssetType.Texture; + + if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3) + assetUuids[regionSettings.TerrainTexture3] = AssetType.Texture; + + if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) + assetUuids[regionSettings.TerrainTexture4] = AssetType.Texture; + + Save(scene, sceneObjects, regionDir); } /// @@ -256,14 +297,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// The user /// The object group /// Which permissions to check: "C" = Copy, "T" = Transfer + /// The scene's permissions module /// Whether the user is allowed to export the object to an OAR - private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions) + private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string checkPermissions, IPermissionsModule permissionsModule) { if (checkPermissions == null) return true; - IPermissionsModule module = m_scene.RequestModuleInterface(); - if (module == null) + if (permissionsModule == null) return true; // this shouldn't happen // Check whether the user is permitted to export all of the parts in the SOG. If any @@ -275,7 +316,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver foreach (SceneObjectPart obj in objGroup.Parts) { uint perm; - PermissionClass permissionClass = module.GetPermissionClass(user, obj); + PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj); switch (permissionClass) { case PermissionClass.Owner: @@ -330,16 +371,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver } /// - /// Create the control file for the most up to date archive + /// Create the control file. /// /// - public string CreateControlFile(Dictionary options) + public string CreateControlFile(ArchiveScenesGroup scenesGroup) { - int majorVersion = MAX_MAJOR_VERSION, minorVersion = 8; + int majorVersion; + int minorVersion; + + if (MultiRegionFormat) + { + majorVersion = MAX_MAJOR_VERSION; + minorVersion = 0; + } + else + { + // To support older versions of OpenSim, we continue to create single-region OARs + // using the old file format. In the future this format will be discontinued. + majorVersion = 0; + minorVersion = 8; + } // -// if (options.ContainsKey("version")) +// if (m_options.ContainsKey("version")) // { -// string[] parts = options["version"].ToString().Split('.'); +// string[] parts = m_options["version"].ToString().Split('.'); // if (parts.Length >= 1) // { // majorVersion = Int32.Parse(parts[0]); @@ -368,10 +423,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver // } m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion); - //if (majorVersion == 1) - //{ - // m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim 0.7.0.2 and earlier. Please use the --version=0 option if you want to produce a compatible OAR"); - //} + if (majorVersion == 1) + { + m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR"); + } String s; @@ -389,37 +444,23 @@ namespace OpenSim.Region.CoreModules.World.Archiver DateTime now = DateTime.UtcNow; TimeSpan t = now - new DateTime(1970, 1, 1); xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString()); - xtw.WriteElementString("id", UUID.Random().ToString()); - xtw.WriteEndElement(); - - xtw.WriteStartElement("region_info"); - - bool isMegaregion; - Vector2 size; - IRegionCombinerModule rcMod = null; - - // FIXME: This is only here for regression test purposes since they do not supply a module. Need to fix - // this, possibly by doing control file creation somewhere else. - if (m_module != null) - rcMod = m_module.RegionCombinerModule; - - if (rcMod != null) - isMegaregion = rcMod.IsRootForMegaregion(m_scene.RegionInfo.RegionID); - else - isMegaregion = false; - - if (isMegaregion) - size = rcMod.GetSizeOfMegaregion(m_scene.RegionInfo.RegionID); - else - size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); - - xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); - xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); - + if (!MultiRegionFormat) + xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString()); xtw.WriteEndElement(); xtw.WriteElementString("assets_included", SaveAssets.ToString()); + if (MultiRegionFormat) + { + WriteRegionsManifest(scenesGroup, xtw); + } + else + { + xtw.WriteStartElement("region_info"); + WriteRegionInfo(m_rootScene, xtw); + xtw.WriteEndElement(); + } + xtw.WriteEndElement(); xtw.Flush(); @@ -428,11 +469,166 @@ namespace OpenSim.Region.CoreModules.World.Archiver s = sw.ToString(); } -// if (m_scene != null) -// Console.WriteLine( -// "[ARCHIVE WRITE REQUEST PREPARATION]: Control file for {0} is: {1}", m_scene.RegionInfo.RegionName, s); - return s; } + + /// + /// Writes the list of regions included in a multi-region OAR. + /// + private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw) + { + xtw.WriteStartElement("regions"); + + // Write the regions in order: rows from South to North, then regions from West to East. + // The list of regions can have "holes"; we write empty elements in their position. + + for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y) + { + SortedDictionary row; + if (scenesGroup.Regions.TryGetValue(y, out row)) + { + xtw.WriteStartElement("row"); + + for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x) + { + Scene scene; + if (row.TryGetValue(x, out scene)) + { + xtw.WriteStartElement("region"); + xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString()); + xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID)); + WriteRegionInfo(scene, xtw); + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing region + xtw.WriteElementString("region", ""); + } + } + + xtw.WriteEndElement(); + } + else + { + // Write a placeholder for a missing row + xtw.WriteElementString("row", ""); + } + } + + xtw.WriteEndElement(); // "regions" + } + + protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw) + { + bool isMegaregion; + Vector2 size; + + IRegionCombinerModule rcMod = scene.RequestModuleInterface(); + + if (rcMod != null) + isMegaregion = rcMod.IsRootForMegaregion(scene.RegionInfo.RegionID); + else + isMegaregion = false; + + if (isMegaregion) + size = rcMod.GetSizeOfMegaregion(scene.RegionInfo.RegionID); + else + size = new Vector2((float)Constants.RegionSize, (float)Constants.RegionSize); + + xtw.WriteElementString("is_megaregion", isMegaregion.ToString()); + xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y)); + } + + + protected void Save(Scene scene, List sceneObjects, string regionDir) + { + if (regionDir != string.Empty) + regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/"; + + m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive."); + + // Write out region settings + string settingsPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings)); + + m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive."); + + // Write out land data (aka parcel) settings + List landObjects = scene.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + LandData landData = lo.LandData; + string landDataPath = String.Format("{0}{1}{2}.xml", + regionDir, ArchiveConstants.LANDDATA_PATH, landData.GlobalID.ToString()); + m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options)); + } + + m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive."); + + // Write out terrain + string terrainPath = String.Format("{0}{1}{2}.r32", + regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName); + + MemoryStream ms = new MemoryStream(); + scene.RequestModuleInterface().SaveToStream(terrainPath, ms); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); + ms.Close(); + + m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive."); + + // Write out scene object metadata + IRegionSerialiserModule serializer = scene.RequestModuleInterface(); + foreach (SceneObjectGroup sceneObject in sceneObjects) + { + //m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType()); + + string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options); + string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject)); + m_archiveWriter.WriteFile(objectPath, serializedObject); + } + } + + protected void ReceivedAllAssets( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) + { + foreach (UUID uuid in assetsNotFoundUuids) + { + m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid); + } + + // m_log.InfoFormat( + // "[ARCHIVER]: Received {0} of {1} assets requested", + // assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); + + CloseArchive(String.Empty); + } + + + /// + /// Closes the archive and notifies that we're done. + /// + /// The error that occurred, or empty for success + protected void CloseArchive(string errorMessage) + { + try + { + if (m_archiveWriter != null) + m_archiveWriter.Close(); + m_saveStream.Close(); + } + catch (Exception e) + { + m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e); + if (errorMessage == string.Empty) + errorMessage = e.Message; + } + + m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName); + + m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage); + } + } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs index bf3b1240a4..26535a9c3d 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs @@ -146,6 +146,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver ops.Add("noassets", delegate(string v) { options["noassets"] = v != null; }); ops.Add("publish", v => options["wipe-owners"] = v != null); ops.Add("perm=", delegate(string v) { options["checkPermissions"] = v; }); + ops.Add("all", delegate(string v) { options["all"] = v != null; }); List mainParams = ops.Parse(cmdparams); @@ -169,7 +170,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Writing archive for region {0} to {1}", Scene.RegionInfo.RegionName, savePath); - new ArchiveWriteRequestPreparation(this, savePath, requestId).ArchiveRegion(options); + new ArchiveWriteRequestPreparation(Scene, savePath, requestId).ArchiveRegion(options); } public void ArchiveRegion(Stream saveStream) @@ -184,7 +185,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver public void ArchiveRegion(Stream saveStream, Guid requestId, Dictionary options) { - new ArchiveWriteRequestPreparation(this, saveStream, requestId).ArchiveRegion(options); + new ArchiveWriteRequestPreparation(Scene, saveStream, requestId).ArchiveRegion(options); } public void DearchiveRegion(string loadPath) diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index a073cb9fb8..57872790b2 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -46,6 +46,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// + /// Method called when all the necessary assets for an archive request have been received. + /// + public delegate void AssetsRequestCallback( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); + enum RequestState { Initial, diff --git a/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs b/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs new file mode 100644 index 0000000000..3dcc020188 --- /dev/null +++ b/OpenSim/Region/CoreModules/World/Archiver/DearchiveScenesGroup.cs @@ -0,0 +1,232 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using OpenSim.Region.Framework.Scenes; +using OpenMetaverse; +using System.Drawing; +using log4net; +using System.Reflection; +using OpenSim.Framework.Serialization; + +namespace OpenSim.Region.CoreModules.World.Archiver +{ + /// + /// The regions included in an OAR file. + /// + public class DearchiveScenesInfo + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// One region in the archive. + /// + public class RegionInfo + { + /// + /// The subdirectory in which the region is stored. + /// + public string Directory { get; set; } + + /// + /// The region's coordinates (relative to the South-West corner of the block). + /// + public Point Location { get; set; } + + /// + /// The UUID of the original scene from which this archived region was saved. + /// + public string OriginalID { get; set; } + + /// + /// The scene in the current simulator into which this region is loaded. + /// If null then the region doesn't have a corresponding scene, and it won't be loaded. + /// + public Scene Scene { get; set; } + } + + /// + /// Whether this archive uses the multi-region format. + /// + public Boolean MultiRegionFormat { get; set; } + + /// + /// Maps (Region directory -> region) + /// + protected Dictionary m_directory2region = new Dictionary(); + + /// + /// Maps (UUID of the scene in the simulator where the region will be loaded -> region) + /// + protected Dictionary m_newId2region = new Dictionary(); + + public int LoadedCreationDateTime { get; set; } + public string DefaultOriginalID { get; set; } + + // These variables are used while reading the archive control file + protected int? m_curY = null; + protected int? m_curX = null; + protected RegionInfo m_curRegion; + + + public DearchiveScenesInfo() + { + MultiRegionFormat = false; + } + + + // The following methods are used while reading the archive control file + + public void StartRow() + { + m_curY = (m_curY == null) ? 0 : m_curY + 1; + m_curX = null; + } + + public void StartRegion() + { + m_curX = (m_curX == null) ? 0 : m_curX + 1; + // Note: this doesn't mean we have a real region in this location; this could just be a "hole" + } + + public void SetRegionOriginalID(string id) + { + m_curRegion = new RegionInfo(); + m_curRegion.Location = new Point((int)m_curX, (int)m_curY); + m_curRegion.OriginalID = id; + // 'curRegion' will be saved in 'm_directory2region' when SetRegionDir() is called + } + + public void SetRegionDirectory(string directory) + { + m_curRegion.Directory = directory; + m_directory2region[directory] = m_curRegion; + } + + + /// + /// Sets all the scenes present in the simulator. + /// + /// + /// This method matches regions in the archive to scenes in the simulator according to + /// their relative position. We only load regions if there's an existing Scene in the + /// grid location where the region should be loaded. + /// + /// The scene where the Load OAR operation was run + /// All the scenes in the simulator + public void SetSimulatorScenes(Scene rootScene, ArchiveScenesGroup simulatorScenes) + { + foreach (RegionInfo archivedRegion in m_directory2region.Values) + { + Point location = new Point((int)rootScene.RegionInfo.RegionLocX, (int)rootScene.RegionInfo.RegionLocY); + location.Offset(archivedRegion.Location); + + Scene scene; + if (simulatorScenes.TryGetScene(location, out scene)) + { + archivedRegion.Scene = scene; + m_newId2region[scene.RegionInfo.RegionID] = archivedRegion; + } + else + { + m_log.WarnFormat("[ARCHIVER]: Not loading archived region {0} because there's no existing region at location {1},{2}", + archivedRegion.Directory, location.X, location.Y); + } + } + } + + /// + /// Returns the archived region according to the path of a file in the archive. + /// Also, converts the full path into a path that is relative to the region's directory. + /// + /// The path of a file in the archive + /// The corresponding Scene, or null if none + /// The path relative to the region's directory. (Or the original + /// path, if this file doesn't belong to a region.) + /// True: use this file; False: skip it + public bool GetRegionFromPath(string fullPath, out Scene scene, out string relativePath) + { + scene = null; + relativePath = fullPath; + + if (!MultiRegionFormat) + { + if (m_newId2region.Count > 0) + scene = m_newId2region.First().Value.Scene; + return true; + } + + if (!fullPath.StartsWith(ArchiveConstants.REGIONS_PATH)) + return true; // this file doesn't belong to a region + + string[] parts = fullPath.Split(new Char[] { '/' }, 3); + if (parts.Length != 3) + return false; + string regionDirectory = parts[1]; + relativePath = parts[2]; + + RegionInfo region; + if (m_directory2region.TryGetValue(regionDirectory, out region)) + { + scene = region.Scene; + return (scene != null); + } + else + { + return false; + } + } + + /// + /// Returns the original UUID of a region (from the simulator where the OAR was saved), + /// given the UUID of the scene it was loaded into in the current simulator. + /// + /// + /// + public string GetOriginalRegionID(UUID newID) + { + RegionInfo region; + if (m_newId2region.TryGetValue(newID, out region)) + return region.OriginalID; + else + return DefaultOriginalID; + } + + /// + /// Returns the scenes that have been (or will be) loaded. + /// + /// + public List GetLoadedScenes() + { + return m_newId2region.Keys.ToList(); + } + + } +} diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index 904110e457..cfdfd8c33e 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -47,6 +47,7 @@ using ArchiveConstants = OpenSim.Framework.Serialization.ArchiveConstants; using TarArchiveReader = OpenSim.Framework.Serialization.TarArchiveReader; using TarArchiveWriter = OpenSim.Framework.Serialization.TarArchiveWriter; using RegionSettings = OpenSim.Framework.RegionSettings; +using OpenSim.Region.Framework.Interfaces; namespace OpenSim.Region.CoreModules.World.Archiver.Tests { @@ -70,9 +71,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests m_scene = new SceneHelpers().SetupScene(); SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, serialiserModule, terrainModule); + + new SceneManager(); + SceneManager.Instance.Add(m_scene); } - - private void LoadCompleted(Guid requestId, string errorMessage) + + private void LoadCompleted(Guid requestId, List loadedScenes, string errorMessage) { lock (this) { @@ -186,7 +190,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); - arr.LoadControlFile(filePath, data); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); Assert.That(arr.ControlFileLoaded, Is.True); @@ -270,7 +274,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); - arr.LoadControlFile(filePath, data); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); Assert.That(arr.ControlFileLoaded, Is.True); @@ -307,7 +311,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectGroup sog1 = SceneHelpers.CreateSceneObject(1, ownerId, "obj1-", 0x11); SceneObjectPart sop2 @@ -362,11 +366,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests // Also check that direct entries which will also have a file entry containing that directory doesn't // upset load tar.WriteDir(ArchiveConstants.TERRAINS_PATH); - + tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); - + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectPart part1 = CreateSceneObjectPart1(); part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); @@ -519,6 +522,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests TestScene scene2 = new SceneHelpers().SetupScene(); SceneHelpers.SetupSceneModules(scene2, archiverModule, serialiserModule, terrainModule); + SceneManager.Instance.Add(scene2); + // Make sure there's a valid owner for the owner we saved (this should have been wiped if the code is // behaving correctly UserAccountHelpers.CreateUserWithInventory(scene2, objectOwner); @@ -554,7 +559,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteDir(ArchiveConstants.TERRAINS_PATH); tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(null, (Stream)null, Guid.Empty).CreateControlFile(new Dictionary())); + new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); RegionSettings rs = new RegionSettings(); rs.AgentLimit = 17; diff --git a/OpenSim/Region/Framework/Interfaces/IEstateModule.cs b/OpenSim/Region/Framework/Interfaces/IEstateModule.cs index 15cd23830d..1983984e1c 100644 --- a/OpenSim/Region/Framework/Interfaces/IEstateModule.cs +++ b/OpenSim/Region/Framework/Interfaces/IEstateModule.cs @@ -46,6 +46,11 @@ namespace OpenSim.Region.Framework.Interfaces /// void sendRegionHandshakeToAll(); + /// + /// Fires the OnRegionInfoChange event. + /// + void TriggerRegionInfoChange(); + void setEstateTerrainBaseTexture(int level, UUID texture); void setEstateTerrainTextureHeights(int corner, float lowValue, float highValue); } diff --git a/OpenSim/Region/Framework/Scenes/EventManager.cs b/OpenSim/Region/Framework/Scenes/EventManager.cs index 2f347859a3..e1c9c8e5bc 100644 --- a/OpenSim/Region/Framework/Scenes/EventManager.cs +++ b/OpenSim/Region/Framework/Scenes/EventManager.cs @@ -531,7 +531,7 @@ namespace OpenSim.Region.Framework.Scenes /// the scripts may not have started yet /// Message is non empty string if there were problems loading the oar file /// - public delegate void OarFileLoaded(Guid guid, string message); + public delegate void OarFileLoaded(Guid guid, List loadedScenes, string message); public event OarFileLoaded OnOarFileLoaded; /// @@ -2195,7 +2195,7 @@ namespace OpenSim.Region.Framework.Scenes return 6; } - public void TriggerOarFileLoaded(Guid requestId, string message) + public void TriggerOarFileLoaded(Guid requestId, List loadedScenes, string message) { OarFileLoaded handlerOarFileLoaded = OnOarFileLoaded; if (handlerOarFileLoaded != null) @@ -2204,7 +2204,7 @@ namespace OpenSim.Region.Framework.Scenes { try { - d(requestId, message); + d(requestId, loadedScenes, message); } catch (Exception e) { diff --git a/OpenSim/Region/OptionalModules/Scripting/RegionReadyModule/RegionReadyModule.cs b/OpenSim/Region/OptionalModules/Scripting/RegionReadyModule/RegionReadyModule.cs index fff3a32010..bad75f7819 100644 --- a/OpenSim/Region/OptionalModules/Scripting/RegionReadyModule/RegionReadyModule.cs +++ b/OpenSim/Region/OptionalModules/Scripting/RegionReadyModule/RegionReadyModule.cs @@ -181,7 +181,7 @@ namespace OpenSim.Region.OptionalModules.Scripting.RegionReady } } - void OnOarFileLoaded(Guid requestId, string message) + void OnOarFileLoaded(Guid requestId, List loadedScenes, string message) { m_oarFileLoading = true; From d7e6fe488d5258ec92230b6c652969f2d0420376 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Fri, 7 Sep 2012 23:30:54 +0100 Subject: [PATCH 4/6] Rename ArchiveWriteRequestPreparatio nto ArchiveWriteRequest since after the multi-OAR patch there is now only one class that handles this operation. Adapation of 0004-Renamed-ArchiveWriteRequestPreparation-to-ArchiveWri.patch in http://opensimulator.org/mantis/view.php?id=6105 since that did not directly apply --- ...eWriteRequestPreparation.cs => ArchiveWriteRequest.cs} | 8 ++++---- .../Region/CoreModules/World/Archiver/ArchiverModule.cs | 4 ++-- .../CoreModules/World/Archiver/Tests/ArchiverTests.cs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) rename OpenSim/Region/CoreModules/World/Archiver/{ArchiveWriteRequestPreparation.cs => ArchiveWriteRequest.cs} (98%) diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs similarity index 98% rename from OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs rename to OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs index 2c34f4ba9f..d751b1c990 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs @@ -50,7 +50,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Prepare to write out an archive. /// - public class ArchiveWriteRequestPreparation + public class ArchiveWriteRequest { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); @@ -95,7 +95,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// If there was a problem opening a stream for the file specified by the savePath /// - public ArchiveWriteRequestPreparation(Scene scene, string savePath, Guid requestId) : this(scene, requestId) + public ArchiveWriteRequest(Scene scene, string savePath, Guid requestId) : this(scene, requestId) { try { @@ -116,12 +116,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// The root scene to archive /// The stream to which to save data. /// The id associated with this request - public ArchiveWriteRequestPreparation(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) + public ArchiveWriteRequest(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId) { m_saveStream = saveStream; } - protected ArchiveWriteRequestPreparation(Scene scene, Guid requestId) + protected ArchiveWriteRequest(Scene scene, Guid requestId) { m_rootScene = scene; m_requestId = requestId; diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs index 26535a9c3d..2a87dc2a5f 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiverModule.cs @@ -170,7 +170,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Writing archive for region {0} to {1}", Scene.RegionInfo.RegionName, savePath); - new ArchiveWriteRequestPreparation(Scene, savePath, requestId).ArchiveRegion(options); + new ArchiveWriteRequest(Scene, savePath, requestId).ArchiveRegion(options); } public void ArchiveRegion(Stream saveStream) @@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver public void ArchiveRegion(Stream saveStream, Guid requestId, Dictionary options) { - new ArchiveWriteRequestPreparation(Scene, saveStream, requestId).ArchiveRegion(options); + new ArchiveWriteRequest(Scene, saveStream, requestId).ArchiveRegion(options); } public void DearchiveRegion(string loadPath) diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index cfdfd8c33e..abbaf419c2 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -311,7 +311,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectGroup sog1 = SceneHelpers.CreateSceneObject(1, ownerId, "obj1-", 0x11); SceneObjectPart sop2 @@ -369,7 +369,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); SceneObjectPart part1 = CreateSceneObjectPart1(); part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); @@ -559,7 +559,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests tar.WriteDir(ArchiveConstants.TERRAINS_PATH); tar.WriteFile( ArchiveConstants.CONTROL_FILE_PATH, - new ArchiveWriteRequestPreparation(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); + new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty).CreateControlFile(new ArchiveScenesGroup())); RegionSettings rs = new RegionSettings(); rs.AgentLimit = 17; From 5dd2569bf7d50217d1786c3268a537745b2e9fd3 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Mon, 27 Aug 2012 12:47:04 +0300 Subject: [PATCH 5/6] Added unit tests for multi-region OARs --- .../ClientStack/RegionApplicationBase.cs | 2 +- .../World/Archiver/Tests/ArchiverTests.cs | 384 +++++++++++++++--- .../Region/Framework/Scenes/SceneManager.cs | 6 +- 3 files changed, 332 insertions(+), 60 deletions(-) diff --git a/OpenSim/Region/ClientStack/RegionApplicationBase.cs b/OpenSim/Region/ClientStack/RegionApplicationBase.cs index 4672f8aa8a..853b72d9e7 100644 --- a/OpenSim/Region/ClientStack/RegionApplicationBase.cs +++ b/OpenSim/Region/ClientStack/RegionApplicationBase.cs @@ -76,7 +76,7 @@ namespace OpenSim.Region.ClientStack protected override void StartupSpecific() { - SceneManager = new SceneManager(); + SceneManager = SceneManager.Instance; m_clientStackManager = CreateClientStackManager(); Initialize(); diff --git a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs index abbaf419c2..0a309057e7 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs @@ -57,23 +57,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests private Guid m_lastRequestId; private string m_lastErrorMessage; + protected SceneHelpers m_sceneHelpers; protected TestScene m_scene; protected ArchiverModule m_archiverModule; + protected SerialiserModule m_serialiserModule; protected TaskInventoryItem m_soundItem; [SetUp] public void SetUp() { + new SceneManager(); + m_archiverModule = new ArchiverModule(); - SerialiserModule serialiserModule = new SerialiserModule(); + m_serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - m_scene = new SceneHelpers().SetupScene(); - SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, serialiserModule, terrainModule); - - new SceneManager(); - SceneManager.Instance.Add(m_scene); + m_sceneHelpers = new SceneHelpers(); + m_scene = m_sceneHelpers.SetupScene(); + SceneHelpers.SetupSceneModules(m_scene, m_archiverModule, m_serialiserModule, terrainModule); } private void LoadCompleted(Guid requestId, List loadedScenes, string errorMessage) @@ -132,26 +134,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests TestHelpers.InMethod(); // log4net.Config.XmlConfigurator.Configure(); - SceneObjectPart part1 = CreateSceneObjectPart1(); - SceneObjectGroup sog1 = new SceneObjectGroup(part1); - m_scene.AddNewSceneObject(sog1, false); - - SceneObjectPart part2 = CreateSceneObjectPart2(); - - AssetNotecard nc = new AssetNotecard(); - nc.BodyText = "Hello World!"; - nc.Encode(); - UUID ncAssetUuid = new UUID("00000000-0000-0000-1000-000000000000"); - UUID ncItemUuid = new UUID("00000000-0000-0000-1100-000000000000"); - AssetBase ncAsset - = AssetHelpers.CreateAsset(ncAssetUuid, AssetType.Notecard, nc.AssetData, UUID.Zero); - m_scene.AssetService.Store(ncAsset); - SceneObjectGroup sog2 = new SceneObjectGroup(part2); - TaskInventoryItem ncItem - = new TaskInventoryItem { Name = "ncItem", AssetID = ncAssetUuid, ItemID = ncItemUuid }; - part2.Inventory.AddInventoryItem(ncItem, true); - - m_scene.AddNewSceneObject(sog2, false); + SceneObjectGroup sog1; + SceneObjectGroup sog2; + UUID ncAssetUuid; + CreateTestObjects(m_scene, out sog1, out sog2, out ncAssetUuid); MemoryStream archiveWriteStream = new MemoryStream(); m_scene.EventManager.OnOarFileSaved += SaveCompleted; @@ -215,6 +201,30 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests // TODO: Test presence of more files and contents of files. } + private void CreateTestObjects(Scene scene, out SceneObjectGroup sog1, out SceneObjectGroup sog2, out UUID ncAssetUuid) + { + SceneObjectPart part1 = CreateSceneObjectPart1(); + sog1 = new SceneObjectGroup(part1); + scene.AddNewSceneObject(sog1, false); + + AssetNotecard nc = new AssetNotecard(); + nc.BodyText = "Hello World!"; + nc.Encode(); + ncAssetUuid = UUID.Random(); + UUID ncItemUuid = UUID.Random(); + AssetBase ncAsset + = AssetHelpers.CreateAsset(ncAssetUuid, AssetType.Notecard, nc.AssetData, UUID.Zero); + m_scene.AssetService.Store(ncAsset); + + TaskInventoryItem ncItem + = new TaskInventoryItem { Name = "ncItem", AssetID = ncAssetUuid, ItemID = ncItemUuid }; + SceneObjectPart part2 = CreateSceneObjectPart2(); + sog2 = new SceneObjectGroup(part2); + part2.Inventory.AddInventoryItem(ncItem, true); + + scene.AddNewSceneObject(sog2, false); + } + /// /// Test saving an OpenSim Region Archive with the no assets option /// @@ -392,31 +402,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(soundDataResourceName, Is.Not.Null); byte[] soundData; - Console.WriteLine("Loading " + soundDataResourceName); - using (Stream resource = assembly.GetManifestResourceStream(soundDataResourceName)) - { - using (BinaryReader br = new BinaryReader(resource)) - { - // FIXME: Use the inspector instead - soundData = br.ReadBytes(99999999); - UUID soundUuid = UUID.Parse("00000000-0000-0000-0000-000000000001"); - string soundAssetFileName - = ArchiveConstants.ASSETS_PATH + soundUuid - + ArchiveConstants.ASSET_TYPE_TO_EXTENSION[(sbyte)AssetType.SoundWAV]; - tar.WriteFile(soundAssetFileName, soundData); - - /* - AssetBase soundAsset = AssetHelpers.CreateAsset(soundUuid, soundData); - scene.AssetService.Store(soundAsset); - asset1FileName = ArchiveConstants.ASSETS_PATH + soundUuid + ".wav"; - */ - - TaskInventoryItem item1 - = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; - part1.Inventory.AddInventoryItem(item1, true); - } - } - + UUID soundUuid; + CreateSoundAsset(tar, assembly, soundDataResourceName, out soundData, out soundUuid); + + TaskInventoryItem item1 + = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; + part1.Inventory.AddInventoryItem(item1, true); m_scene.AddNewSceneObject(object1, false); string object1FileName = string.Format( @@ -438,6 +429,34 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(m_lastErrorMessage, Is.Null); + TestLoadedRegion(part1, soundItemName, soundData); + } + + private static void CreateSoundAsset(TarArchiveWriter tar, Assembly assembly, string soundDataResourceName, out byte[] soundData, out UUID soundUuid) + { + using (Stream resource = assembly.GetManifestResourceStream(soundDataResourceName)) + { + using (BinaryReader br = new BinaryReader(resource)) + { + // FIXME: Use the inspector instead + soundData = br.ReadBytes(99999999); + soundUuid = UUID.Parse("00000000-0000-0000-0000-000000000001"); + string soundAssetFileName + = ArchiveConstants.ASSETS_PATH + soundUuid + + ArchiveConstants.ASSET_TYPE_TO_EXTENSION[(sbyte)AssetType.SoundWAV]; + tar.WriteFile(soundAssetFileName, soundData); + + /* + AssetBase soundAsset = AssetHelpers.CreateAsset(soundUuid, soundData); + scene.AssetService.Store(soundAsset); + asset1FileName = ArchiveConstants.ASSETS_PATH + soundUuid + ".wav"; + */ + } + } + } + + private void TestLoadedRegion(SceneObjectPart part1, string soundItemName, byte[] soundData) + { SceneObjectPart object1PartLoaded = m_scene.GetSceneObjectPart(part1.Name); Assert.That(object1PartLoaded, Is.Not.Null, "object1 was not loaded"); @@ -457,9 +476,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(loadedSoundAsset.Data, Is.EqualTo(soundData), "saved and loaded sound data do not match"); Assert.Greater(m_scene.LandChannel.AllParcels().Count, 0, "incorrect number of parcels"); - - // Temporary - Console.WriteLine("Successfully completed {0}", MethodBase.GetCurrentMethod()); } /// @@ -519,11 +535,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests SerialiserModule serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - TestScene scene2 = new SceneHelpers().SetupScene(); + m_sceneHelpers = new SceneHelpers(); + TestScene scene2 = m_sceneHelpers.SetupScene(); SceneHelpers.SetupSceneModules(scene2, archiverModule, serialiserModule, terrainModule); - SceneManager.Instance.Add(scene2); - // Make sure there's a valid owner for the owner we saved (this should have been wiped if the code is // behaving correctly UserAccountHelpers.CreateUserWithInventory(scene2, objectOwner); @@ -669,7 +684,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests SerialiserModule serialiserModule = new SerialiserModule(); TerrainModule terrainModule = new TerrainModule(); - Scene scene = new SceneHelpers().SetupScene(); + Scene scene = m_sceneHelpers.SetupScene(); SceneHelpers.SetupSceneModules(scene, archiverModule, serialiserModule, terrainModule); m_scene.AddNewSceneObject(new SceneObjectGroup(part2), false); @@ -705,5 +720,258 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests Assert.That(object2PartMerged.GroupPosition, Is.EqualTo(part2.GroupPosition), "object2 group position not equal after merge"); } } + + /// + /// Test saving a multi-region OAR. + /// + [Test] + public void TestSaveMultiRegionOar() + { + TestHelpers.InMethod(); + + // Create test regions + + int WIDTH = 2; + int HEIGHT = 2; + + List scenes = new List(); + + // Maps (Directory in OAR file -> scene) + Dictionary regionPaths = new Dictionary(); + + // Maps (Scene -> expected object paths) + Dictionary> expectedPaths = new Dictionary>(); + + // List of expected assets + List expectedAssets = new List(); + + for (uint y = 0; y < HEIGHT; y++) + { + for (uint x = 0; x < WIDTH; x++) + { + Scene scene; + if (x == 0 && y == 0) + { + scene = m_scene; // this scene was already created in SetUp() + } + else + { + scene = m_sceneHelpers.SetupScene(string.Format("Unit test region {0}", (y * WIDTH) + x + 1), UUID.Random(), 1000 + x, 1000 + y); + SceneHelpers.SetupSceneModules(scene, new ArchiverModule(), m_serialiserModule, new TerrainModule()); + } + scenes.Add(scene); + + string dir = String.Format("{0}_{1}_{2}", x + 1, y + 1, scene.RegionInfo.RegionName.Replace(" ", "_")); + regionPaths[dir] = scene; + + SceneObjectGroup sog1; + SceneObjectGroup sog2; + UUID ncAssetUuid; + + CreateTestObjects(scene, out sog1, out sog2, out ncAssetUuid); + + expectedPaths[scene.RegionInfo.RegionID] = new List(); + expectedPaths[scene.RegionInfo.RegionID].Add(ArchiveHelpers.CreateObjectPath(sog1)); + expectedPaths[scene.RegionInfo.RegionID].Add(ArchiveHelpers.CreateObjectPath(sog2)); + + expectedAssets.Add(ncAssetUuid); + } + } + + + // Save OAR + + MemoryStream archiveWriteStream = new MemoryStream(); + m_scene.EventManager.OnOarFileSaved += SaveCompleted; + + Guid requestId = new Guid("00000000-0000-0000-0000-808080808080"); + + Dictionary options = new Dictionary(); + options.Add("all", true); + + lock (this) + { + m_archiverModule.ArchiveRegion(archiveWriteStream, requestId, options); + Monitor.Wait(this, 60000); + } + + + // Check that the OAR contains the expected data + + Assert.That(m_lastRequestId, Is.EqualTo(requestId)); + + byte[] archive = archiveWriteStream.ToArray(); + MemoryStream archiveReadStream = new MemoryStream(archive); + TarArchiveReader tar = new TarArchiveReader(archiveReadStream); + + Dictionary> foundPaths = new Dictionary>(); + List foundAssets = new List(); + + foreach (Scene scene in scenes) + { + foundPaths[scene.RegionInfo.RegionID] = new List(); + } + + string filePath; + TarArchiveReader.TarEntryType tarEntryType; + + byte[] data = tar.ReadEntry(out filePath, out tarEntryType); + Assert.That(filePath, Is.EqualTo(ArchiveConstants.CONTROL_FILE_PATH)); + + ArchiveReadRequest arr = new ArchiveReadRequest(m_scene, (Stream)null, false, false, Guid.Empty); + arr.LoadControlFile(filePath, data, new DearchiveScenesInfo()); + + Assert.That(arr.ControlFileLoaded, Is.True); + + while (tar.ReadEntry(out filePath, out tarEntryType) != null) + { + if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) + { + // Assets are shared, so this file doesn't belong to any specific region. + string fileName = filePath.Remove(0, ArchiveConstants.ASSETS_PATH.Length); + if (fileName.EndsWith("_notecard.txt")) + foundAssets.Add(UUID.Parse(fileName.Substring(0, fileName.Length - "_notecard.txt".Length))); + } + else + { + // This file belongs to one of the regions. Find out which one. + Assert.IsTrue(filePath.StartsWith(ArchiveConstants.REGIONS_PATH)); + string[] parts = filePath.Split(new Char[] { '/' }, 3); + Assert.AreEqual(3, parts.Length); + string regionDirectory = parts[1]; + string relativePath = parts[2]; + Scene scene = regionPaths[regionDirectory]; + + if (relativePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) + { + foundPaths[scene.RegionInfo.RegionID].Add(relativePath); + } + } + } + + Assert.AreEqual(scenes.Count, foundPaths.Count); + foreach (Scene scene in scenes) + { + Assert.That(foundPaths[scene.RegionInfo.RegionID], Is.EquivalentTo(expectedPaths[scene.RegionInfo.RegionID])); + } + + Assert.That(foundAssets, Is.EquivalentTo(expectedAssets)); + } + + /// + /// Test loading a multi-region OAR. + /// + [Test] + public void TestLoadMultiRegionOar() + { + TestHelpers.InMethod(); + + // Create an ArchiveScenesGroup with the regions in the OAR. This is needed to generate the control file. + + int WIDTH = 2; + int HEIGHT = 2; + + for (uint y = 0; y < HEIGHT; y++) + { + for (uint x = 0; x < WIDTH; x++) + { + Scene scene; + if (x == 0 && y == 0) + { + scene = m_scene; // this scene was already created in SetUp() + } + else + { + scene = m_sceneHelpers.SetupScene(string.Format("Unit test region {0}", (y * WIDTH) + x + 1), UUID.Random(), 1000 + x, 1000 + y); + SceneHelpers.SetupSceneModules(scene, new ArchiverModule(), m_serialiserModule, new TerrainModule()); + } + } + } + + ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup(); + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scenesGroup.AddScene(scene); + }); + scenesGroup.CalcSceneLocations(); + + // Generate the OAR file + + MemoryStream archiveWriteStream = new MemoryStream(); + TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream); + + ArchiveWriteRequest writeRequest = new ArchiveWriteRequest(m_scene, (Stream)null, Guid.Empty); + writeRequest.MultiRegionFormat = true; + tar.WriteFile( + ArchiveConstants.CONTROL_FILE_PATH, writeRequest.CreateControlFile(scenesGroup)); + + SceneObjectPart part1 = CreateSceneObjectPart1(); + part1.SitTargetOrientation = new Quaternion(0.2f, 0.3f, 0.4f, 0.5f); + part1.SitTargetPosition = new Vector3(1, 2, 3); + + SceneObjectGroup object1 = new SceneObjectGroup(part1); + + // Let's put some inventory items into our object + string soundItemName = "sound-item1"; + UUID soundItemUuid = UUID.Parse("00000000-0000-0000-0000-000000000002"); + Type type = GetType(); + Assembly assembly = type.Assembly; + string soundDataResourceName = null; + string[] names = assembly.GetManifestResourceNames(); + foreach (string name in names) + { + if (name.EndsWith(".Resources.test-sound.wav")) + soundDataResourceName = name; + } + Assert.That(soundDataResourceName, Is.Not.Null); + + byte[] soundData; + UUID soundUuid; + CreateSoundAsset(tar, assembly, soundDataResourceName, out soundData, out soundUuid); + + TaskInventoryItem item1 + = new TaskInventoryItem { AssetID = soundUuid, ItemID = soundItemUuid, Name = soundItemName }; + part1.Inventory.AddInventoryItem(item1, true); + m_scene.AddNewSceneObject(object1, false); + + string object1FileName = string.Format( + "{0}_{1:000}-{2:000}-{3:000}__{4}.xml", + part1.Name, + Math.Round(part1.GroupPosition.X), Math.Round(part1.GroupPosition.Y), Math.Round(part1.GroupPosition.Z), + part1.UUID); + string path = "regions/1_1_Unit_test_region/" + ArchiveConstants.OBJECTS_PATH + object1FileName; + tar.WriteFile(path, SceneObjectSerializer.ToXml2Format(object1)); + + tar.Close(); + + + // Delete the current objects, to test that they're loaded from the OAR and didn't + // just remain in the scene. + SceneManager.Instance.ForEachScene(delegate(Scene scene) + { + scene.DeleteAllSceneObjects(); + }); + + // Create a "hole", to test that that the corresponding region isn't loaded from the OAR + SceneManager.Instance.CloseScene(SceneManager.Instance.Scenes[1]); + + + // Check thay the OAR file contains the expected data + + MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray()); + + lock (this) + { + m_scene.EventManager.OnOarFileLoaded += LoadCompleted; + m_archiverModule.DearchiveRegion(archiveReadStream); + } + + Assert.That(m_lastErrorMessage, Is.Null); + + Assert.AreEqual(3, SceneManager.Instance.Scenes.Count); + + TestLoadedRegion(part1, soundItemName, soundData); + } + } } diff --git a/OpenSim/Region/Framework/Scenes/SceneManager.cs b/OpenSim/Region/Framework/Scenes/SceneManager.cs index c81b55de16..cb5b2ba9b2 100644 --- a/OpenSim/Region/Framework/Scenes/SceneManager.cs +++ b/OpenSim/Region/Framework/Scenes/SceneManager.cs @@ -92,7 +92,11 @@ namespace OpenSim.Region.Framework.Scenes private static SceneManager m_instance = null; public static SceneManager Instance { - get { return m_instance; } + get { + if (m_instance == null) + m_instance = new SceneManager(); + return m_instance; + } } private readonly List m_localScenes = new List(); From d5c8c6bc12d434bb6a4bbc6f326a4ae784c12861 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Sat, 15 Sep 2012 01:08:15 +0100 Subject: [PATCH 6/6] For FlotsamAssetCache, always update access times of cached scene assets before looking for files to expire. This is to resolve a problem where an asset marked as local but not temporary but still used in the scene would be removed. The timed expiry scan no longer tries to refetch assets from the scene that are not currently in the cache - this is not helpful since it just drags a lot of data into the cache that may never be referenced. This removes the DeepScanBeforePurge option since setting this to false will introduce the above problem. This previously had a default of true. --- .../CoreModules/Asset/FlotsamAssetCache.cs | 37 +++++++++---------- bin/config-include/FlotsamCache.ini.example | 7 ---- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 5d8a278f23..8aa173ab2b 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -107,8 +107,6 @@ namespace OpenSim.Region.CoreModules.Asset private IAssetService m_AssetService; private List m_Scenes = new List(); - private bool m_DeepScanBeforePurge; - public FlotsamAssetCache() { m_InvalidChars.AddRange(Path.GetInvalidPathChars()); @@ -170,8 +168,6 @@ namespace OpenSim.Region.CoreModules.Asset m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen); m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt); - - m_DeepScanBeforePurge = assetConfig.GetBoolean("DeepScanBeforePurge", m_DeepScanBeforePurge); } m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory); @@ -506,13 +502,10 @@ namespace OpenSim.Region.CoreModules.Asset // Purge all files last accessed prior to this point DateTime purgeLine = DateTime.Now - m_FileExpiration; - // An optional deep scan at this point will ensure assets present in scenes, - // or referenced by objects in the scene, but not recently accessed - // are not purged. - if (m_DeepScanBeforePurge) - { - CacheScenes(); - } + // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore, + // before cleaning up expired files we must scan the objects in the scene to make sure that we retain + // such local assets if they have not been recently accessed. + TouchAllSceneAssets(false); foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) { @@ -705,11 +698,14 @@ namespace OpenSim.Region.CoreModules.Asset /// /// Iterates through all Scenes, doing a deep scan through assets - /// to cache all assets present in the scene or referenced by assets - /// in the scene + /// to update the access time of all assets present in the scene or referenced by assets + /// in the scene. /// - /// - private int CacheScenes() + /// + /// If true, then assets scanned which are not found in cache are added to the cache. + /// + /// Number of distinct asset references found in the scene. + private int TouchAllSceneAssets(bool storeUncached) { UuidGatherer gatherer = new UuidGatherer(m_AssetService); @@ -732,7 +728,7 @@ namespace OpenSim.Region.CoreModules.Asset { File.SetLastAccessTime(filename, DateTime.Now); } - else + else if (storeUncached) { m_AssetService.Get(assetID.ToString()); } @@ -860,13 +856,14 @@ namespace OpenSim.Region.CoreModules.Asset break; - case "assets": - m_log.Info("[FLOTSAM ASSET CACHE]: Caching all assets, in all scenes."); + m_log.Info("[FLOTSAM ASSET CACHE]: Ensuring assets are cached for all scenes."); Util.FireAndForget(delegate { - int assetsCached = CacheScenes(); - m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Completed Scene Caching, {0} assets found.", assetsCached); + int assetReferenceTotal = TouchAllSceneAssets(true); + m_log.InfoFormat( + "[FLOTSAM ASSET CACHE]: Completed check with {0} assets.", + assetReferenceTotal); }); break; diff --git a/bin/config-include/FlotsamCache.ini.example b/bin/config-include/FlotsamCache.ini.example index b9c6d84808..ad74fc14e9 100644 --- a/bin/config-include/FlotsamCache.ini.example +++ b/bin/config-include/FlotsamCache.ini.example @@ -54,10 +54,3 @@ ; Warning level for cache directory size ;CacheWarnAt = 30000 - - ; Perform a deep scan of all assets within all regions, looking for all assets - ; present or referenced. Mark all assets found that are already present in the - ; cache, and request all assets that are found that are not already cached (this - ; will cause those assets to be cached) - ; - DeepScanBeforePurge = true