From 08606ae409b400b6f9b83006ed04826eede8a9c7 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Thu, 8 Jan 2015 20:21:40 +0000 Subject: [PATCH] Make the IteratingUuidGatherer the only UuidGatherer. This UUID gatherer provides a superset of the previous gatherer's functionality as it also allows the caller to control gathering iterations for load purposes. --- .../CoreModules/Asset/FlotsamAssetCache.cs | 14 +- .../Archiver/InventoryArchiveWriteRequest.cs | 14 +- .../EntityTransfer/HGEntityTransferModule.cs | 4 +- .../InventoryAccess/HGAssetMapper.cs | 21 +- .../World/Archiver/ArchiveWriteRequest.cs | 10 +- .../Scenes/Tests/UuidGathererTests.cs | 48 +- .../Region/Framework/Scenes/UuidGatherer.cs | 704 ++---------------- .../Avatar/Appearance/AppearanceInfoModule.cs | 10 +- 8 files changed, 112 insertions(+), 713 deletions(-) diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 59b30a95ec..7d9c9a950f 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -776,7 +776,6 @@ namespace OpenSim.Region.CoreModules.Asset { UuidGatherer gatherer = new UuidGatherer(m_AssetService); - Dictionary assetIdsToCheck = new Dictionary(); Dictionary assetsFound = new Dictionary(); foreach (Scene s in m_Scenes) @@ -784,10 +783,11 @@ namespace OpenSim.Region.CoreModules.Asset StampRegionStatusFile(s.RegionInfo.RegionID); s.ForEachSOG(delegate(SceneObjectGroup e) - { - gatherer.GatherAssetUuids(e, assetIdsToCheck); + { + gatherer.AddForInspection(e); + gatherer.GatherAll(); - foreach (UUID assetID in assetIdsToCheck.Keys) + foreach (UUID assetID in gatherer.GatheredUuids.Keys) { if (!assetsFound.ContainsKey(assetID)) { @@ -800,7 +800,7 @@ namespace OpenSim.Region.CoreModules.Asset else if (storeUncached) { AssetBase cachedAsset = m_AssetService.Get(assetID.ToString()); - if (cachedAsset == null && assetIdsToCheck[assetID] != (sbyte)AssetType.Unknown) + if (cachedAsset == null && gatherer.GatheredUuids[assetID] != (sbyte)AssetType.Unknown) assetsFound[assetID] = false; else assetsFound[assetID] = true; @@ -810,11 +810,11 @@ namespace OpenSim.Region.CoreModules.Asset { m_log.DebugFormat( "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets", - assetID, assetIdsToCheck[assetID], e.Name, e.AbsolutePosition, s.Name); + assetID, gatherer.GatheredUuids[assetID], e.Name, e.AbsolutePosition, s.Name); } } - assetIdsToCheck.Clear(); + gatherer.GatheredUuids.Clear(); }); } diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs index 6dab227278..42a4c7a47f 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs @@ -93,11 +93,6 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver /// protected UUID m_id; - /// - /// Used to collect the uuids of the assets that we need to save into the archive - /// - protected Dictionary m_assetUuids = new Dictionary(); - /// /// Used to collect the uuids of the users that we need to save into the archive /// @@ -225,7 +220,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver // Don't chase down link asset items as they actually point to their target item IDs rather than an asset if (SaveAssets && itemAssetType != AssetType.Link && itemAssetType != AssetType.LinkFolder) - m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (sbyte)inventoryItem.AssetType, m_assetUuids); + m_assetGatherer.AddForInspection(inventoryItem.AssetID); } /// @@ -422,12 +417,15 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (SaveAssets) { - m_log.DebugFormat("[INVENTORY ARCHIVER]: Saving {0} assets for items", m_assetUuids.Count); + m_assetGatherer.GatherAll(); + + m_log.DebugFormat( + "[INVENTORY ARCHIVER]: Saving {0} assets for items", m_assetGatherer.GatheredUuids.Count); AssetsRequest ar = new AssetsRequest( new AssetsArchiver(m_archiveWriter), - m_assetUuids, m_scene.AssetService, + m_assetGatherer.GatheredUuids, m_scene.AssetService, m_scene.UserAccountService, m_scene.RegionInfo.ScopeID, options, ReceivedAllAssets); diff --git a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs index 2ddb599af4..fceda802cf 100644 --- a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs +++ b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs @@ -605,8 +605,8 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer // so.Name, so.AttachedAvatar, url); IDictionary ids = new Dictionary(); - IteratingHGUuidGatherer uuidGatherer - = new IteratingHGUuidGatherer(Scene.AssetService, url, ids); + HGUuidGatherer uuidGatherer + = new HGUuidGatherer(Scene.AssetService, url, ids); uuidGatherer.AddForInspection(so); while (!uuidGatherer.Complete) diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs index 2ac15176e3..6343a81689 100644 --- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs +++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs @@ -412,12 +412,13 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess // The act of gathering UUIDs downloads some assets from the remote server // but not all... - Dictionary ids = new Dictionary(); HGUuidGatherer uuidGatherer = new HGUuidGatherer(m_scene.AssetService, userAssetURL); - uuidGatherer.GatherAssetUuids(assetID, meta.Type, ids); - m_log.DebugFormat("[HG ASSET MAPPER]: Preparing to get {0} assets", ids.Count); + uuidGatherer.AddForInspection(assetID); + uuidGatherer.GatherAll(); + + m_log.DebugFormat("[HG ASSET MAPPER]: Preparing to get {0} assets", uuidGatherer.GatheredUuids.Count); bool success = true; - foreach (UUID uuid in ids.Keys) + foreach (UUID uuid in uuidGatherer.GatheredUuids.Keys) if (FetchAsset(userAssetURL, uuid) == null) success = false; @@ -428,7 +429,6 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess m_log.DebugFormat("[HG ASSET MAPPER]: Successfully got item {0} from asset server {1}", assetID, userAssetURL); } - public void Post(UUID assetID, UUID ownerID, string userAssetURL) { m_log.DebugFormat("[HG ASSET MAPPER]: Starting to send asset {0} with children to asset server {1}", assetID, userAssetURL); @@ -442,9 +442,8 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess return; } - Dictionary ids = new Dictionary(); HGUuidGatherer uuidGatherer = new HGUuidGatherer(m_scene.AssetService, string.Empty); - uuidGatherer.GatherAssetUuids(asset.FullID, asset.Type, ids); + uuidGatherer.AddForInspection(asset.FullID); // Check which assets already exist in the destination server @@ -452,16 +451,16 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess if (!url.EndsWith("/") && !url.EndsWith("=")) url = url + "/"; - string[] remoteAssetIDs = new string[ids.Count]; + string[] remoteAssetIDs = new string[uuidGatherer.GatheredUuids.Count]; int i = 0; - foreach (UUID id in ids.Keys) + foreach (UUID id in uuidGatherer.GatheredUuids.Keys) remoteAssetIDs[i++] = url + id.ToString(); bool[] exist = m_scene.AssetService.AssetsExist(remoteAssetIDs); var existSet = new HashSet(); i = 0; - foreach (UUID id in ids.Keys) + foreach (UUID id in uuidGatherer.GatheredUuids.Keys) { if (exist[i]) existSet.Add(id.ToString()); @@ -472,7 +471,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess bool success = true; - foreach (UUID uuid in ids.Keys) + foreach (UUID uuid in uuidGatherer.GatheredUuids.Keys) { if (!existSet.Contains(uuid.ToString())) { diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs index b7d7c26cfc..d67bdcecc1 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs @@ -219,7 +219,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids) { - m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.RegionInfo.RegionName); + m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.Name); EntityBase[] entities = scene.GetEntities(); List sceneObjects = new List(); @@ -253,13 +253,13 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (SaveAssets) { - UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService); + UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService, assetUuids); int prevAssets = assetUuids.Count; foreach (SceneObjectGroup sceneObject in sceneObjects) - { - assetGatherer.GatherAssetUuids(sceneObject, assetUuids); - } + assetGatherer.AddForInspection(sceneObject); + + assetGatherer.GatherAll(); m_log.DebugFormat( "[ARCHIVER]: {0} scene objects to serialize requiring save of {1} assets", diff --git a/OpenSim/Region/Framework/Scenes/Tests/UuidGathererTests.cs b/OpenSim/Region/Framework/Scenes/Tests/UuidGathererTests.cs index a12b170723..1fe0e7956f 100644 --- a/OpenSim/Region/Framework/Scenes/Tests/UuidGathererTests.cs +++ b/OpenSim/Region/Framework/Scenes/Tests/UuidGathererTests.cs @@ -61,11 +61,11 @@ namespace OpenSim.Region.Framework.Scenes.Tests = AssetHelpers.CreateAsset(corruptAssetUuid, AssetType.Notecard, "CORRUPT ASSET", UUID.Zero); m_assetService.Store(corruptAsset); - IDictionary foundAssetUuids = new Dictionary(); - m_uuidGatherer.GatherAssetUuids(corruptAssetUuid, (sbyte)AssetType.Object, foundAssetUuids); + m_uuidGatherer.AddForInspection(corruptAssetUuid); + m_uuidGatherer.GatherAll(); // We count the uuid as gathered even if the asset itself is corrupt. - Assert.That(foundAssetUuids.Count, Is.EqualTo(1)); + Assert.That(m_uuidGatherer.GatheredUuids.Count, Is.EqualTo(1)); } /// @@ -77,38 +77,42 @@ namespace OpenSim.Region.Framework.Scenes.Tests TestHelpers.InMethod(); UUID missingAssetUuid = UUID.Parse("00000000-0000-0000-0000-000000000666"); - IDictionary foundAssetUuids = new Dictionary(); - - m_uuidGatherer.GatherAssetUuids(missingAssetUuid, (sbyte)AssetType.Object, foundAssetUuids); - // We count the uuid as gathered even if the asset itself is missing. - Assert.That(foundAssetUuids.Count, Is.EqualTo(1)); + m_uuidGatherer.AddForInspection(missingAssetUuid); + m_uuidGatherer.GatherAll(); + + Assert.That(m_uuidGatherer.GatheredUuids.Count, Is.EqualTo(0)); } [Test] public void TestNotecardAsset() { TestHelpers.InMethod(); -// log4net.Config.XmlConfigurator.Configure(); + TestHelpers.EnableLogging(); UUID ownerId = TestHelpers.ParseTail(0x10); - UUID soAssetId = TestHelpers.ParseTail(0x20); + UUID embeddedId = TestHelpers.ParseTail(0x20); + UUID missingEmbeddedId = TestHelpers.ParseTail(0x21); UUID ncAssetId = TestHelpers.ParseTail(0x30); - SceneObjectGroup so = SceneHelpers.CreateSceneObject(1, ownerId); - AssetBase soAsset = AssetHelpers.CreateAsset(soAssetId, so); - m_assetService.Store(soAsset); - - AssetBase ncAsset = AssetHelpers.CreateNotecardAsset(ncAssetId, soAssetId.ToString()); + AssetBase ncAsset + = AssetHelpers.CreateNotecardAsset( + ncAssetId, string.Format("Hello{0}World{1}", embeddedId, missingEmbeddedId)); m_assetService.Store(ncAsset); - IDictionary foundAssetUuids = new Dictionary(); - m_uuidGatherer.GatherAssetUuids(ncAssetId, (sbyte)AssetType.Notecard, foundAssetUuids); + AssetBase embeddedAsset = AssetHelpers.CreateNotecardAsset(embeddedId, "We'll meet again."); + m_assetService.Store(embeddedAsset); - // We count the uuid as gathered even if the asset itself is corrupt. - Assert.That(foundAssetUuids.Count, Is.EqualTo(2)); - Assert.That(foundAssetUuids.ContainsKey(ncAssetId)); - Assert.That(foundAssetUuids.ContainsKey(soAssetId)); + m_uuidGatherer.AddForInspection(ncAssetId); + m_uuidGatherer.GatherAll(); + +// foreach (UUID key in m_uuidGatherer.GatheredUuids.Keys) +// System.Console.WriteLine("key : {0}", key); + + Assert.That(m_uuidGatherer.GatheredUuids.Count, Is.EqualTo(3)); + Assert.That(m_uuidGatherer.GatheredUuids.ContainsKey(ncAssetId)); + Assert.That(m_uuidGatherer.GatheredUuids.ContainsKey(embeddedId)); + Assert.That(m_uuidGatherer.GatheredUuids.ContainsKey(missingEmbeddedId)); } } -} +} \ No newline at end of file diff --git a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs index 2c5353fc6b..28653c68ca 100644 --- a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs +++ b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs @@ -55,617 +55,17 @@ namespace OpenSim.Region.Framework.Scenes { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - protected IAssetService m_assetService; - - // /// - // /// Used as a temporary store of an asset which represents an object. This can be a null if no appropriate - // /// asset was found by the asset service. - // /// - // private AssetBase m_requestedObjectAsset; - // - // /// - // /// Signal whether we are currently waiting for the asset service to deliver an asset. - // /// - // private bool m_waitingForObjectAsset; - - public UuidGatherer(IAssetService assetService) - { - m_assetService = assetService; - } - - /// - /// Gather all the asset uuids associated with the asset referenced by a given uuid - /// - /// - /// This includes both those directly associated with - /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained - /// within this object). - /// This method assumes that the asset type associated with this asset in persistent storage is correct (which - /// should always be the case). So with this method we always need to retrieve asset data even if the asset - /// is of a type which is known not to reference any other assets - /// - /// The uuid of the asset for which to gather referenced assets - /// The assets gathered - public void GatherAssetUuids(UUID assetUuid, IDictionary assetUuids) - { - // avoid infinite loops - if (assetUuids.ContainsKey(assetUuid)) - return; - - try - { - AssetBase assetBase = GetAsset(assetUuid); - - if (null != assetBase) - { - sbyte assetType = assetBase.Type; - assetUuids[assetUuid] = assetType; - - if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) - { - GetWearableAssetUuids(assetBase, assetUuids); - } - else if ((sbyte)AssetType.Gesture == assetType) - { - GetGestureAssetUuids(assetBase, assetUuids); - } - else if ((sbyte)AssetType.Notecard == assetType) - { - GetTextEmbeddedAssetUuids(assetBase, assetUuids); - } - else if ((sbyte)AssetType.LSLText == assetType) - { - GetTextEmbeddedAssetUuids(assetBase, assetUuids); - } - else if ((sbyte)OpenSimAssetType.Material == assetType) - { - GetMaterialAssetUuids(assetBase, assetUuids); - } - else if ((sbyte)AssetType.Object == assetType) - { - GetSceneObjectAssetUuids(assetBase, assetUuids); - } - } - } - catch (Exception) - { - m_log.ErrorFormat("[UUID GATHERER]: Failed to gather uuids for asset id {0}", assetUuid); - throw; - } - } - - /// - /// Gather all the asset uuids associated with the asset referenced by a given uuid - /// - /// - /// This includes both those directly associated with - /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained - /// within this object). - /// - /// The uuid of the asset for which to gather referenced assets - /// The type of the asset for the uuid given - /// The assets gathered - public void GatherAssetUuids(UUID assetUuid, sbyte assetType, IDictionary assetUuids) - { - // avoid infinite loops - if (assetUuids.ContainsKey(assetUuid)) - return; - - try - { - assetUuids[assetUuid] = assetType; - - if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) - { - GetWearableAssetUuids(assetUuid, assetUuids); - } - else if ((sbyte)AssetType.Gesture == assetType) - { - GetGestureAssetUuids(assetUuid, assetUuids); - } - else if ((sbyte)AssetType.Notecard == assetType) - { - GetTextEmbeddedAssetUuids(assetUuid, assetUuids); - } - else if ((sbyte)AssetType.LSLText == assetType) - { - GetTextEmbeddedAssetUuids(assetUuid, assetUuids); - } - else if ((sbyte)OpenSimAssetType.Material == assetType) - { - GetMaterialAssetUuids(assetUuid, assetUuids); - } - else if ((sbyte)AssetType.Object == assetType) - { - GetSceneObjectAssetUuids(assetUuid, assetUuids); - } - } - catch (Exception) - { - m_log.ErrorFormat( - "[UUID GATHERER]: Failed to gather uuids for asset id {0}, type {1}", - assetUuid, assetType); - throw; - } - } - - /// - /// Gather all the asset uuids associated with a given object. - /// - /// - /// This includes both those directly associated with - /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained - /// within this object). - /// - /// The scene object for which to gather assets - /// - /// A dictionary which is populated with the asset UUIDs gathered and the type of that asset. - /// For assets where the type is not clear (e.g. UUIDs extracted from LSL and notecards), the type is Unknown. - /// - public void GatherAssetUuids(SceneObjectGroup sceneObject, IDictionary assetUuids) - { - // m_log.DebugFormat( - // "[ASSET GATHERER]: Getting assets for object {0}, {1}", sceneObject.Name, sceneObject.UUID); - - SceneObjectPart[] parts = sceneObject.Parts; - for (int i = 0; i < parts.Length; i++) - { - SceneObjectPart part = parts[i]; - - // m_log.DebugFormat( - // "[ARCHIVER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID); - - try - { - Primitive.TextureEntry textureEntry = part.Shape.Textures; - if (textureEntry != null) - { - // Get the prim's default texture. This will be used for faces which don't have their own texture - if (textureEntry.DefaultTexture != null) - GatherTextureEntryAssets(textureEntry.DefaultTexture, assetUuids); - - if (textureEntry.FaceTextures != null) - { - // Loop through the rest of the texture faces (a non-null face means the face is different from DefaultTexture) - foreach (Primitive.TextureEntryFace texture in textureEntry.FaceTextures) - { - if (texture != null) - GatherTextureEntryAssets(texture, assetUuids); - } - } - } - - // If the prim is a sculpt then preserve this information too - if (part.Shape.SculptTexture != UUID.Zero) - assetUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; - - if (part.Shape.ProjectionTextureUUID != UUID.Zero) - assetUuids[part.Shape.ProjectionTextureUUID] = (sbyte)AssetType.Texture; - - if (part.CollisionSound != UUID.Zero) - assetUuids[part.CollisionSound] = (sbyte)AssetType.Sound; - - if (part.ParticleSystem.Length > 0) - { - try - { - Primitive.ParticleSystem ps = new Primitive.ParticleSystem(part.ParticleSystem, 0); - if (ps.Texture != UUID.Zero) - assetUuids[ps.Texture] = (sbyte)AssetType.Texture; - } - catch (Exception) - { - m_log.WarnFormat( - "[UUID GATHERER]: Could not check particle system for part {0} {1} in object {2} {3} since it is corrupt. Continuing.", - part.Name, part.UUID, sceneObject.Name, sceneObject.UUID); - } - } - - TaskInventoryDictionary taskDictionary = (TaskInventoryDictionary)part.TaskInventory.Clone(); - - // Now analyze this prim's inventory items to preserve all the uuids that they reference - foreach (TaskInventoryItem tii in taskDictionary.Values) - { - // m_log.DebugFormat( - // "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", - // tii.Name, tii.Type, part.Name, part.UUID); - - if (!assetUuids.ContainsKey(tii.AssetID)) - GatherAssetUuids(tii.AssetID, (sbyte)tii.Type, assetUuids); - } - - // FIXME: We need to make gathering modular but we cannot yet, since gatherers are not guaranteed - // to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and - // inventory transfer. There needs to be a way for a module to register a method without assuming a - // Scene.EventManager is present. - // part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids); - - - // still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs - GatherMaterialsUuids(part, assetUuids); - } - catch (Exception e) - { - m_log.ErrorFormat("[UUID GATHERER]: Failed to get part - {0}", e); - m_log.DebugFormat( - "[UUID GATHERER]: Texture entry length for prim was {0} (min is 46)", - part.Shape.TextureEntry.Length); - } - } - } - - /// - /// Gather all the asset uuids found in one face of a Texture Entry. - /// - private void GatherTextureEntryAssets(Primitive.TextureEntryFace texture, IDictionary assetUuids) - { - assetUuids[texture.TextureID] = (sbyte)AssetType.Texture; - - if (texture.MaterialID != UUID.Zero) - { - GatherAssetUuids(texture.MaterialID, (sbyte)OpenSimAssetType.Material, assetUuids); - } - } - - /// - /// Gather all of the texture asset UUIDs used to reference "Materials" such as normal and specular maps - /// stored in legacy format in part.DynAttrs - /// - /// - /// - //public void GatherMaterialsUuids(SceneObjectPart part, IDictionary assetUuids) - public void GatherMaterialsUuids(SceneObjectPart part, IDictionary assetUuids) - { - // scan thru the dynAttrs map of this part for any textures used as materials - OSD osdMaterials = null; - - lock (part.DynAttrs) - { - if (part.DynAttrs.ContainsStore("OpenSim", "Materials")) - { - OSDMap materialsStore = part.DynAttrs.GetStore("OpenSim", "Materials"); - - if (materialsStore == null) - return; - - materialsStore.TryGetValue("Materials", out osdMaterials); - } - - if (osdMaterials != null) - { - //m_log.Info("[UUID Gatherer]: found Materials: " + OSDParser.SerializeJsonString(osd)); - - if (osdMaterials is OSDArray) - { - OSDArray matsArr = osdMaterials as OSDArray; - foreach (OSDMap matMap in matsArr) - { - try - { - if (matMap.ContainsKey("Material")) - { - OSDMap mat = matMap["Material"] as OSDMap; - if (mat.ContainsKey("NormMap")) - { - UUID normalMapId = mat["NormMap"].AsUUID(); - if (normalMapId != UUID.Zero) - { - assetUuids[normalMapId] = (sbyte)AssetType.Texture; - //m_log.Info("[UUID Gatherer]: found normal map ID: " + normalMapId.ToString()); - } - } - if (mat.ContainsKey("SpecMap")) - { - UUID specularMapId = mat["SpecMap"].AsUUID(); - if (specularMapId != UUID.Zero) - { - assetUuids[specularMapId] = (sbyte)AssetType.Texture; - //m_log.Info("[UUID Gatherer]: found specular map ID: " + specularMapId.ToString()); - } - } - } - - } - catch (Exception e) - { - m_log.Warn("[UUID Gatherer]: exception getting materials: " + e.Message); - } - } - } - } - } - } - - /// - /// Get an asset synchronously, potentially using an asynchronous callback. If the - /// asynchronous callback is used, we will wait for it to complete. - /// - /// - /// - protected virtual AssetBase GetAsset(UUID uuid) - { - return m_assetService.Get(uuid.ToString()); - - // XXX: Switching to do this synchronously where the call was async before but we always waited for it - // to complete anyway! - // m_waitingForObjectAsset = true; - // m_assetCache.Get(uuid.ToString(), this, AssetReceived); - // - // // The asset cache callback can either - // // - // // 1. Complete on the same thread (if the asset is already in the cache) or - // // 2. Come in via a different thread (if we need to go fetch it). - // // - // // The code below handles both these alternatives. - // lock (this) - // { - // if (m_waitingForObjectAsset) - // { - // Monitor.Wait(this); - // m_waitingForObjectAsset = false; - // } - // } - // - // return m_requestedObjectAsset; - } - - /// - /// Record the asset uuids embedded within the given text (e.g. a script). - /// - /// - /// Dictionary in which to record the references - private void GetTextEmbeddedAssetUuids(UUID textAssetUuid, IDictionary assetUuids) - { - // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); - - AssetBase textAsset = GetAsset(textAssetUuid); - - if (null != textAsset) - GetTextEmbeddedAssetUuids(textAsset, assetUuids); - } - - /// - /// Record the asset uuids embedded within the given text (e.g. a script). - /// - /// - /// Dictionary in which to record the references - private void GetTextEmbeddedAssetUuids(AssetBase textAsset, IDictionary assetUuids) - { - // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); - - string script = Utils.BytesToString(textAsset.Data); - // m_log.DebugFormat("[ARCHIVER]: Script {0}", script); - MatchCollection uuidMatches = Util.PermissiveUUIDPattern.Matches(script); - // m_log.DebugFormat("[ARCHIVER]: Found {0} matches in text", uuidMatches.Count); - - foreach (Match uuidMatch in uuidMatches) - { - UUID uuid = new UUID(uuidMatch.Value); - // m_log.DebugFormat("[ARCHIVER]: Recording {0} in text", uuid); - - // Embedded asset references (if not false positives) could be for many types of asset, so we will - // label these as unknown. - assetUuids[uuid] = (sbyte)AssetType.Unknown; - } - } - - /// - /// Record the uuids referenced by the given wearable asset - /// - /// - /// Dictionary in which to record the references - private void GetWearableAssetUuids(UUID wearableAssetUuid, IDictionary assetUuids) - { - AssetBase assetBase = GetAsset(wearableAssetUuid); - - if (null != assetBase) - GetWearableAssetUuids(assetBase, assetUuids); - } - - /// - /// Record the uuids referenced by the given wearable asset - /// - /// - /// Dictionary in which to record the references - private void GetWearableAssetUuids(AssetBase assetBase, IDictionary assetUuids) - { - //m_log.Debug(new System.Text.ASCIIEncoding().GetString(bodypartAsset.Data)); - AssetWearable wearableAsset = new AssetBodypart(assetBase.FullID, assetBase.Data); - wearableAsset.Decode(); - - //m_log.DebugFormat( - // "[ARCHIVER]: Wearable asset {0} references {1} assets", wearableAssetUuid, wearableAsset.Textures.Count); - - foreach (UUID uuid in wearableAsset.Textures.Values) - { - assetUuids[uuid] = (sbyte)AssetType.Texture; - } - } - - /// - /// Get all the asset uuids associated with a given object. This includes both those directly associated with - /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained - /// within this object). - /// - /// - /// - private void GetSceneObjectAssetUuids(UUID sceneObjectUuid, IDictionary assetUuids) - { - AssetBase sceneObjectAsset = GetAsset(sceneObjectUuid); - - if (null != sceneObjectAsset) - GetSceneObjectAssetUuids(sceneObjectAsset, assetUuids); - } - - /// - /// Get all the asset uuids associated with a given object. This includes both those directly associated with - /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained - /// within this object). - /// - /// - /// - private void GetSceneObjectAssetUuids(AssetBase sceneObjectAsset, IDictionary assetUuids) - { - string xml = Utils.BytesToString(sceneObjectAsset.Data); - - CoalescedSceneObjects coa; - if (CoalescedSceneObjectsSerializer.TryFromXml(xml, out coa)) - { - foreach (SceneObjectGroup sog in coa.Objects) - GatherAssetUuids(sog, assetUuids); - } - else - { - SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(xml); - - if (null != sog) - GatherAssetUuids(sog, assetUuids); - } - } - - /// - /// Get the asset uuid associated with a gesture - /// - /// - /// - private void GetGestureAssetUuids(UUID gestureUuid, IDictionary assetUuids) - { - AssetBase gestureAsset = GetAsset(gestureUuid); - if (null == gestureAsset) - return; - - GetGestureAssetUuids(gestureAsset, assetUuids); - } - - /// - /// Get the asset uuid associated with a gesture - /// - /// - /// - private void GetGestureAssetUuids(AssetBase gestureAsset, IDictionary assetUuids) - { - using (MemoryStream ms = new MemoryStream(gestureAsset.Data)) - using (StreamReader sr = new StreamReader(ms)) - { - sr.ReadLine(); // Unknown (Version?) - sr.ReadLine(); // Unknown - sr.ReadLine(); // Unknown - sr.ReadLine(); // Name - sr.ReadLine(); // Comment ? - int count = Convert.ToInt32(sr.ReadLine()); // Item count - - for (int i = 0 ; i < count ; i++) - { - string type = sr.ReadLine(); - if (type == null) - break; - string name = sr.ReadLine(); - if (name == null) - break; - string id = sr.ReadLine(); - if (id == null) - break; - string unknown = sr.ReadLine(); - if (unknown == null) - break; - - // If it can be parsed as a UUID, it is an asset ID - UUID uuid; - if (UUID.TryParse(id, out uuid)) - assetUuids[uuid] = (sbyte)AssetType.Animation; // the asset is either an Animation or a Sound, but this distinction isn't important - } - } - } - - /// - /// Get the asset uuid's referenced in a material. - /// - private void GetMaterialAssetUuids(UUID materialUuid, IDictionary assetUuids) - { - AssetBase assetBase = GetAsset(materialUuid); - if (null == assetBase) - return; - - GetMaterialAssetUuids(assetBase, assetUuids); - } - - /// - /// Get the asset uuid's referenced in a material. - /// - private void GetMaterialAssetUuids(AssetBase materialAsset, IDictionary assetUuids) - { - OSDMap mat = (OSDMap)OSDParser.DeserializeLLSDXml(materialAsset.Data); - - UUID normMap = mat["NormMap"].AsUUID(); - if (normMap != UUID.Zero) - assetUuids[normMap] = (sbyte)AssetType.Texture; - - UUID specMap = mat["SpecMap"].AsUUID(); - if (specMap != UUID.Zero) - assetUuids[specMap] = (sbyte)AssetType.Texture; - } - } - - public class HGUuidGatherer : UuidGatherer - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - protected string m_assetServerURL; - - public HGUuidGatherer(IAssetService assetService, string assetServerURL) - : base(assetService) - { - m_assetServerURL = assetServerURL; - if (!m_assetServerURL.EndsWith("/") && !m_assetServerURL.EndsWith("=")) - m_assetServerURL = m_assetServerURL + "/"; - } - - protected override AssetBase GetAsset(UUID uuid) - { - if (string.Empty == m_assetServerURL) - return base.GetAsset(uuid); - else - return FetchAsset(uuid); - } - - public AssetBase FetchAsset(UUID assetID) - { - // Test if it's already here - AssetBase asset = m_assetService.Get(assetID.ToString()); - if (asset == null) - { - // It's not, so fetch it from abroad - asset = m_assetService.Get(m_assetServerURL + assetID.ToString()); - if (asset != null) - m_log.DebugFormat("[HGUUIDGatherer]: Copied asset {0} from {1} to local asset server", assetID, m_assetServerURL); - else - m_log.DebugFormat("[HGUUIDGatherer]: Failed to fetch asset {0} from {1}", assetID, m_assetServerURL); - } - //else - // m_log.DebugFormat("[HGUUIDGatherer]: Asset {0} from {1} was already here", assetID, m_assetServerURL); - - return asset; - } - } - - /// - /// Gather uuids for a given entity. - /// - /// - /// This does a deep inspection of the entity to retrieve all the assets it uses (whether as textures, as scripts - /// contained in inventory, as scripts contained in objects contained in another object's inventory, etc. Assets - /// are only retrieved when they are necessary to carry out the inspection (i.e. a serialized object needs to be - /// retrieved to work out which assets it references). - /// - public class IteratingUuidGatherer - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - /// /// Is gathering complete? /// public bool Complete { get { return m_assetUuidsToInspect.Count <= 0; } } + /// + /// The dictionary of UUIDs gathered so far. If Complete == true then this is all the reachable UUIDs. + /// + /// The gathered uuids. + public IDictionary GatheredUuids { get; private set; } + /// /// Gets the next UUID to inspect. /// @@ -683,10 +83,17 @@ namespace OpenSim.Region.Framework.Scenes protected IAssetService m_assetService; - protected IDictionary m_gatheredAssetUuids; - protected Queue m_assetUuidsToInspect; + /// + /// Initializes a new instance of the class. + /// + /// In this case the collection of gathered assets will start out blank. + /// + /// Asset service. + /// + public UuidGatherer(IAssetService assetService) : this(assetService, new Dictionary()) {} + /// /// Initializes a new instance of the class. /// @@ -697,10 +104,10 @@ namespace OpenSim.Region.Framework.Scenes /// Gathered UUIDs will be collected in this dictinaory. /// It can be pre-populated if you want to stop the gatherer from analyzing assets that have already been fetched and inspected. /// - public IteratingUuidGatherer(IAssetService assetService, IDictionary collector) + public UuidGatherer(IAssetService assetService, IDictionary collector) { m_assetService = assetService; - m_gatheredAssetUuids = collector; + GatheredUuids = collector; // FIXME: Not efficient for searching, can improve. m_assetUuidsToInspect = new Queue(); @@ -765,13 +172,13 @@ namespace OpenSim.Region.Framework.Scenes // If the prim is a sculpt then preserve this information too if (part.Shape.SculptTexture != UUID.Zero) - m_gatheredAssetUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; + GatheredUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; if (part.Shape.ProjectionTextureUUID != UUID.Zero) - m_gatheredAssetUuids[part.Shape.ProjectionTextureUUID] = (sbyte)AssetType.Texture; + GatheredUuids[part.Shape.ProjectionTextureUUID] = (sbyte)AssetType.Texture; if (part.CollisionSound != UUID.Zero) - m_gatheredAssetUuids[part.CollisionSound] = (sbyte)AssetType.Sound; + GatheredUuids[part.CollisionSound] = (sbyte)AssetType.Sound; if (part.ParticleSystem.Length > 0) { @@ -779,7 +186,7 @@ namespace OpenSim.Region.Framework.Scenes { Primitive.ParticleSystem ps = new Primitive.ParticleSystem(part.ParticleSystem, 0); if (ps.Texture != UUID.Zero) - m_gatheredAssetUuids[ps.Texture] = (sbyte)AssetType.Texture; + GatheredUuids[ps.Texture] = (sbyte)AssetType.Texture; } catch (Exception) { @@ -798,7 +205,7 @@ namespace OpenSim.Region.Framework.Scenes // "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", // tii.Name, tii.Type, part.Name, part.UUID); - if (!m_gatheredAssetUuids.ContainsKey(tii.AssetID)) + if (!GatheredUuids.ContainsKey(tii.AssetID)) AddForInspection(tii.AssetID, (sbyte)tii.Type); } @@ -865,7 +272,7 @@ namespace OpenSim.Region.Framework.Scenes private void GetAssetUuids(UUID assetUuid) { // avoid infinite loops - if (m_gatheredAssetUuids.ContainsKey(assetUuid)) + if (GatheredUuids.ContainsKey(assetUuid)) return; try @@ -875,7 +282,7 @@ namespace OpenSim.Region.Framework.Scenes if (null != assetBase) { sbyte assetType = assetBase.Type; - m_gatheredAssetUuids[assetUuid] = assetType; + GatheredUuids[assetUuid] = assetType; if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) { @@ -915,29 +322,15 @@ namespace OpenSim.Region.Framework.Scenes // Here, we want to collect uuids which require further asset fetches but mark the others as gathered try { - m_gatheredAssetUuids[assetUuid] = assetType; + GatheredUuids[assetUuid] = assetType; - if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) - { - AddForInspection(assetUuid); - } - else if ((sbyte)AssetType.Gesture == assetType) - { - AddForInspection(assetUuid); - } - else if ((sbyte)AssetType.Notecard == assetType) - { - AddForInspection(assetUuid); - } - else if ((sbyte)AssetType.LSLText == assetType) - { - AddForInspection(assetUuid); - } - else if ((sbyte)OpenSimAssetType.Material == assetType) - { - AddForInspection(assetUuid); - } - else if ((sbyte)AssetType.Object == assetType) + if ((sbyte)AssetType.Bodypart == assetType + || (sbyte)AssetType.Clothing == assetType + || (sbyte)AssetType.Gesture == assetType + || (sbyte)AssetType.Notecard == assetType + || (sbyte)AssetType.LSLText == assetType + || (sbyte)OpenSimAssetType.Material == assetType + || (sbyte)AssetType.Object == assetType) { AddForInspection(assetUuid); } @@ -956,7 +349,7 @@ namespace OpenSim.Region.Framework.Scenes /// private void RecordTextureEntryAssetUuids(Primitive.TextureEntryFace texture) { - m_gatheredAssetUuids[texture.TextureID] = (sbyte)AssetType.Texture; + GatheredUuids[texture.TextureID] = (sbyte)AssetType.Texture; if (texture.MaterialID != UUID.Zero) AddForInspection(texture.MaterialID); @@ -1003,7 +396,7 @@ namespace OpenSim.Region.Framework.Scenes UUID normalMapId = mat["NormMap"].AsUUID(); if (normalMapId != UUID.Zero) { - m_gatheredAssetUuids[normalMapId] = (sbyte)AssetType.Texture; + GatheredUuids[normalMapId] = (sbyte)AssetType.Texture; //m_log.Info("[UUID Gatherer]: found normal map ID: " + normalMapId.ToString()); } } @@ -1012,7 +405,7 @@ namespace OpenSim.Region.Framework.Scenes UUID specularMapId = mat["SpecMap"].AsUUID(); if (specularMapId != UUID.Zero) { - m_gatheredAssetUuids[specularMapId] = (sbyte)AssetType.Texture; + GatheredUuids[specularMapId] = (sbyte)AssetType.Texture; //m_log.Info("[UUID Gatherer]: found specular map ID: " + specularMapId.ToString()); } } @@ -1048,17 +441,17 @@ namespace OpenSim.Region.Framework.Scenes { // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); - string script = Utils.BytesToString(textAsset.Data); - // m_log.DebugFormat("[ARCHIVER]: Script {0}", script); - MatchCollection uuidMatches = Util.PermissiveUUIDPattern.Matches(script); - // m_log.DebugFormat("[ARCHIVER]: Found {0} matches in text", uuidMatches.Count); + string text = Utils.BytesToString(textAsset.Data); +// m_log.DebugFormat("[UUID GATHERER]: Text {0}", text); + MatchCollection uuidMatches = Util.PermissiveUUIDPattern.Matches(text); +// m_log.DebugFormat("[UUID GATHERER]: Found {0} matches in text", uuidMatches.Count); foreach (Match uuidMatch in uuidMatches) { UUID uuid = new UUID(uuidMatch.Value); - // m_log.DebugFormat("[ARCHIVER]: Recording {0} in text", uuid); +// m_log.DebugFormat("[UUID GATHERER]: Recording {0} in text", uuid); - AddForInspection(uuid); + GatheredUuids[uuid] = (sbyte)AssetType.Unknown; } } @@ -1076,7 +469,7 @@ namespace OpenSim.Region.Framework.Scenes // "[ARCHIVER]: Wearable asset {0} references {1} assets", wearableAssetUuid, wearableAsset.Textures.Count); foreach (UUID uuid in wearableAsset.Textures.Values) - m_gatheredAssetUuids[uuid] = (sbyte)AssetType.Texture; + GatheredUuids[uuid] = (sbyte)AssetType.Texture; } /// @@ -1138,7 +531,7 @@ namespace OpenSim.Region.Framework.Scenes // If it can be parsed as a UUID, it is an asset ID UUID uuid; if (UUID.TryParse(id, out uuid)) - m_gatheredAssetUuids[uuid] = (sbyte)AssetType.Animation; // the asset is either an Animation or a Sound, but this distinction isn't important + GatheredUuids[uuid] = (sbyte)AssetType.Animation; // the asset is either an Animation or a Sound, but this distinction isn't important } } } @@ -1152,21 +545,24 @@ namespace OpenSim.Region.Framework.Scenes UUID normMap = mat["NormMap"].AsUUID(); if (normMap != UUID.Zero) - m_gatheredAssetUuids[normMap] = (sbyte)AssetType.Texture; + GatheredUuids[normMap] = (sbyte)AssetType.Texture; UUID specMap = mat["SpecMap"].AsUUID(); if (specMap != UUID.Zero) - m_gatheredAssetUuids[specMap] = (sbyte)AssetType.Texture; + GatheredUuids[specMap] = (sbyte)AssetType.Texture; } } - public class IteratingHGUuidGatherer : IteratingUuidGatherer + public class HGUuidGatherer : UuidGatherer { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); protected string m_assetServerURL; - public IteratingHGUuidGatherer(IAssetService assetService, string assetServerURL, IDictionary collector) + public HGUuidGatherer(IAssetService assetService, string assetServerURL) + : this(assetService, assetServerURL, new Dictionary()) {} + + public HGUuidGatherer(IAssetService assetService, string assetServerURL, IDictionary collector) : base(assetService, collector) { m_assetServerURL = assetServerURL; diff --git a/OpenSim/Region/OptionalModules/Avatar/Appearance/AppearanceInfoModule.cs b/OpenSim/Region/OptionalModules/Avatar/Appearance/AppearanceInfoModule.cs index f67f6131d4..2f9bb1e24a 100644 --- a/OpenSim/Region/OptionalModules/Avatar/Appearance/AppearanceInfoModule.cs +++ b/OpenSim/Region/OptionalModules/Avatar/Appearance/AppearanceInfoModule.cs @@ -426,10 +426,10 @@ namespace OpenSim.Region.OptionalModules.Avatar.Appearance cdl.AddRow("Assets", ""); sb.Append(cdl.ToString()); - Dictionary assetUuids = new Dictionary(); - uuidGatherer.GatherAssetUuids(wi.AssetID, assetUuids); + uuidGatherer.AddForInspection(wi.AssetID); + uuidGatherer.GatherAll(); string[] assetStrings - = Array.ConvertAll(assetUuids.Keys.ToArray(), u => u.ToString()); + = Array.ConvertAll(uuidGatherer.GatheredUuids.Keys.ToArray(), u => u.ToString()); bool[] existChecks = scene.AssetService.AssetsExist(assetStrings); @@ -440,7 +440,9 @@ namespace OpenSim.Region.OptionalModules.Avatar.Appearance cdt.AddColumn("Found", 5); for (int k = 0; k < existChecks.Length; k++) - cdt.AddRow((AssetType)assetUuids[new UUID(assetStrings[k])], assetStrings[k], existChecks[k] ? "yes" : "no"); + cdt.AddRow( + (AssetType)uuidGatherer.GatheredUuids[new UUID(assetStrings[k])], + assetStrings[k], existChecks[k] ? "yes" : "no"); sb.Append(cdt.ToString()); sb.Append("\n");