diff --git a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs index 522de794e9..be409bb624 100644 --- a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs +++ b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs @@ -571,9 +571,22 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer "[HG ENTITY TRANSFER MODULE]: Incoming attachment {0} for HG user {1} with asset server {2}", so.Name, so.AttachedAvatar, url); - Dictionary ids = new Dictionary(); - HGUuidGatherer uuidGatherer = new HGUuidGatherer(Scene.AssetService, url); - uuidGatherer.GatherAssetUuids(so, ids); + IteratingHGUuidGatherer uuidGatherer = new IteratingHGUuidGatherer(Scene.AssetService, url); + uuidGatherer.RecordAssetUuids(so); + + // XXX: We will shortly use this iterating mechanism to check if a fetch is taking too long + // but just for now we will simply fetch everything. If this was permanent could use + // GatherAll() + while (uuidGatherer.GatherNext()) + m_log.DebugFormat( + "[HG ENTITY TRANSFER]: Gathered attachment {0} for HG user {1} with asset server {2}", + so.Name, so.OwnerID, url); + + IDictionary ids = uuidGatherer.GetGatheredUuids(); + + m_log.DebugFormat( + "[HG ENTITY TRANSFER]: Fetching {0} assets for attachment {1} for HG user {2} with asset server {3}", + ids.Count, so.Name, so.OwnerID, url); foreach (KeyValuePair kvp in ids) uuidGatherer.FetchAsset(kvp.Key); diff --git a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs index 20ff5b5585..9c4e4c03b6 100644 --- a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs +++ b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs @@ -57,17 +57,17 @@ namespace OpenSim.Region.Framework.Scenes protected IAssetService m_assetService; -// /// -// /// Used as a temporary store of an asset which represents an object. This can be a null if no appropriate -// /// asset was found by the asset service. -// /// -// private AssetBase m_requestedObjectAsset; -// -// /// -// /// Signal whether we are currently waiting for the asset service to deliver an asset. -// /// -// private bool m_waitingForObjectAsset; - + // /// + // /// Used as a temporary store of an asset which represents an object. This can be a null if no appropriate + // /// asset was found by the asset service. + // /// + // private AssetBase m_requestedObjectAsset; + // + // /// + // /// Signal whether we are currently waiting for the asset service to deliver an asset. + // /// + // private bool m_waitingForObjectAsset; + public UuidGatherer(IAssetService assetService) { m_assetService = assetService; @@ -133,7 +133,7 @@ namespace OpenSim.Region.Framework.Scenes throw; } } - + /// /// Gather all the asset uuids associated with the asset referenced by a given uuid /// @@ -154,7 +154,7 @@ namespace OpenSim.Region.Framework.Scenes try { assetUuids[assetUuid] = assetType; - + if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) { GetWearableAssetUuids(assetUuid, assetUuids); @@ -204,16 +204,16 @@ namespace OpenSim.Region.Framework.Scenes /// public void GatherAssetUuids(SceneObjectGroup sceneObject, IDictionary assetUuids) { -// m_log.DebugFormat( -// "[ASSET GATHERER]: Getting assets for object {0}, {1}", sceneObject.Name, sceneObject.UUID); + // m_log.DebugFormat( + // "[ASSET GATHERER]: Getting assets for object {0}, {1}", sceneObject.Name, sceneObject.UUID); SceneObjectPart[] parts = sceneObject.Parts; for (int i = 0; i < parts.Length; i++) { SceneObjectPart part = parts[i]; -// m_log.DebugFormat( -// "[ARCHIVER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID); + // m_log.DebugFormat( + // "[ARCHIVER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID); try { @@ -234,7 +234,7 @@ namespace OpenSim.Region.Framework.Scenes } } } - + // If the prim is a sculpt then preserve this information too if (part.Shape.SculptTexture != UUID.Zero) assetUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; @@ -262,13 +262,13 @@ namespace OpenSim.Region.Framework.Scenes } TaskInventoryDictionary taskDictionary = (TaskInventoryDictionary)part.TaskInventory.Clone(); - + // Now analyze this prim's inventory items to preserve all the uuids that they reference foreach (TaskInventoryItem tii in taskDictionary.Values) { -// m_log.DebugFormat( -// "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", -// tii.Name, tii.Type, part.Name, part.UUID); + // m_log.DebugFormat( + // "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", + // tii.Name, tii.Type, part.Name, part.UUID); if (!assetUuids.ContainsKey(tii.AssetID)) GatherAssetUuids(tii.AssetID, (sbyte)tii.Type, assetUuids); @@ -278,7 +278,7 @@ namespace OpenSim.Region.Framework.Scenes // to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and // inventory transfer. There needs to be a way for a module to register a method without assuming a // Scene.EventManager is present. -// part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids); + // part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids); // still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs @@ -375,7 +375,7 @@ namespace OpenSim.Region.Framework.Scenes } } } - + /// /// Get an asset synchronously, potentially using an asynchronous callback. If the /// asynchronous callback is used, we will wait for it to complete. @@ -388,25 +388,25 @@ namespace OpenSim.Region.Framework.Scenes // XXX: Switching to do this synchronously where the call was async before but we always waited for it // to complete anyway! -// m_waitingForObjectAsset = true; -// m_assetCache.Get(uuid.ToString(), this, AssetReceived); -// -// // The asset cache callback can either -// // -// // 1. Complete on the same thread (if the asset is already in the cache) or -// // 2. Come in via a different thread (if we need to go fetch it). -// // -// // The code below handles both these alternatives. -// lock (this) -// { -// if (m_waitingForObjectAsset) -// { -// Monitor.Wait(this); -// m_waitingForObjectAsset = false; -// } -// } -// -// return m_requestedObjectAsset; + // m_waitingForObjectAsset = true; + // m_assetCache.Get(uuid.ToString(), this, AssetReceived); + // + // // The asset cache callback can either + // // + // // 1. Complete on the same thread (if the asset is already in the cache) or + // // 2. Come in via a different thread (if we need to go fetch it). + // // + // // The code below handles both these alternatives. + // lock (this) + // { + // if (m_waitingForObjectAsset) + // { + // Monitor.Wait(this); + // m_waitingForObjectAsset = false; + // } + // } + // + // return m_requestedObjectAsset; } /// @@ -416,7 +416,7 @@ namespace OpenSim.Region.Framework.Scenes /// Dictionary in which to record the references private void GetTextEmbeddedAssetUuids(UUID textAssetUuid, IDictionary assetUuids) { -// m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); + // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); AssetBase textAsset = GetAsset(textAssetUuid); @@ -630,7 +630,526 @@ namespace OpenSim.Region.Framework.Scenes public AssetBase FetchAsset(UUID assetID) { + // Test if it's already here + AssetBase asset = m_assetService.Get(assetID.ToString()); + if (asset == null) + { + // It's not, so fetch it from abroad + asset = m_assetService.Get(m_assetServerURL + assetID.ToString()); + if (asset != null) + m_log.DebugFormat("[HGUUIDGatherer]: Copied asset {0} from {1} to local asset server", assetID, m_assetServerURL); + else + m_log.DebugFormat("[HGUUIDGatherer]: Failed to fetch asset {0} from {1}", assetID, m_assetServerURL); + } + //else + // m_log.DebugFormat("[HGUUIDGatherer]: Asset {0} from {1} was already here", assetID, m_assetServerURL); + return asset; + } + } + + public class IteratingUuidGatherer + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// Is gathering complete? + /// + public bool GatheringComplete { get { return m_assetUuidsToInspect.Count <= 0; } } + + protected IAssetService m_assetService; + + protected IDictionary m_gatheredAssetUuids; + + protected Queue m_assetUuidsToInspect; + + public IteratingUuidGatherer(IAssetService assetService) + { + m_assetService = assetService; + m_gatheredAssetUuids = new Dictionary(); + + // FIXME: Not efficient for searching, can improve. + m_assetUuidsToInspect = new Queue(); + } + + public IDictionary GetGatheredUuids() + { + return new Dictionary(m_gatheredAssetUuids); + } + + public bool AddAssetUuidToInspect(UUID uuid) + { + if (m_assetUuidsToInspect.Contains(uuid)) + return false; + + m_assetUuidsToInspect.Enqueue(uuid); + + return true; + } + + /// + /// Gathers the next set of assets returned by the next uuid to get from the asset service. + /// + /// false if gathering is already complete, true otherwise + public bool GatherNext() + { + if (GatheringComplete) + return false; + + GetAssetUuids(m_assetUuidsToInspect.Dequeue()); + + return true; + } + + /// + /// Gathers all remaining asset UUIDS no matter how many calls are required to the asset service. + /// + /// false if gathering is already complete, true otherwise + public bool GatherAll() + { + if (GatheringComplete) + return false; + + while (GatherNext()); + + return true; + } + + /// + /// Gather all the asset uuids associated with the asset referenced by a given uuid + /// + /// + /// This includes both those directly associated with + /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained + /// within this object). + /// This method assumes that the asset type associated with this asset in persistent storage is correct (which + /// should always be the case). So with this method we always need to retrieve asset data even if the asset + /// is of a type which is known not to reference any other assets + /// + /// The uuid of the asset for which to gather referenced assets + private void GetAssetUuids(UUID assetUuid) + { + // avoid infinite loops + if (m_gatheredAssetUuids.ContainsKey(assetUuid)) + return; + + try + { + AssetBase assetBase = GetAsset(assetUuid); + + if (null != assetBase) + { + sbyte assetType = assetBase.Type; + m_gatheredAssetUuids[assetUuid] = assetType; + + if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) + { + RecordWearableAssetUuids(assetBase); + } + else if ((sbyte)AssetType.Gesture == assetType) + { + RecordGestureAssetUuids(assetBase); + } + else if ((sbyte)AssetType.Notecard == assetType) + { + RecordTextEmbeddedAssetUuids(assetBase); + } + else if ((sbyte)AssetType.LSLText == assetType) + { + RecordTextEmbeddedAssetUuids(assetBase); + } + else if ((sbyte)OpenSimAssetType.Material == assetType) + { + RecordMaterialAssetUuids(assetBase); + } + else if ((sbyte)AssetType.Object == assetType) + { + RecordSceneObjectAssetUuids(assetBase); + } + } + } + catch (Exception) + { + m_log.ErrorFormat("[UUID GATHERER]: Failed to gather uuids for asset id {0}", assetUuid); + throw; + } + } + + private void RecordAssetUuids(UUID assetUuid, sbyte assetType) + { + // Here, we want to collect uuids which require further asset fetches but mark the others as gathered + try + { + m_gatheredAssetUuids[assetUuid] = assetType; + + if ((sbyte)AssetType.Bodypart == assetType || (sbyte)AssetType.Clothing == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + else if ((sbyte)AssetType.Gesture == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + else if ((sbyte)AssetType.Notecard == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + else if ((sbyte)AssetType.LSLText == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + else if ((sbyte)OpenSimAssetType.Material == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + else if ((sbyte)AssetType.Object == assetType) + { + AddAssetUuidToInspect(assetUuid); + } + } + catch (Exception) + { + m_log.ErrorFormat( + "[ITERATABLE UUID GATHERER]: Failed to gather uuids for asset id {0}, type {1}", + assetUuid, assetType); + throw; + } + } + + /// + /// Gather all the asset uuids associated with a given object. + /// + /// + /// This includes both those directly associated with + /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained + /// within this object). + /// + /// The scene object for which to gather assets + public void RecordAssetUuids(SceneObjectGroup sceneObject) + { + // m_log.DebugFormat( + // "[ASSET GATHERER]: Getting assets for object {0}, {1}", sceneObject.Name, sceneObject.UUID); + + SceneObjectPart[] parts = sceneObject.Parts; + for (int i = 0; i < parts.Length; i++) + { + SceneObjectPart part = parts[i]; + + // m_log.DebugFormat( + // "[ARCHIVER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID); + + try + { + Primitive.TextureEntry textureEntry = part.Shape.Textures; + if (textureEntry != null) + { + // Get the prim's default texture. This will be used for faces which don't have their own texture + if (textureEntry.DefaultTexture != null) + RecordTextureEntryAssetUuids(textureEntry.DefaultTexture); + + if (textureEntry.FaceTextures != null) + { + // Loop through the rest of the texture faces (a non-null face means the face is different from DefaultTexture) + foreach (Primitive.TextureEntryFace texture in textureEntry.FaceTextures) + { + if (texture != null) + RecordTextureEntryAssetUuids(texture); + } + } + } + + // If the prim is a sculpt then preserve this information too + if (part.Shape.SculptTexture != UUID.Zero) + m_gatheredAssetUuids[part.Shape.SculptTexture] = (sbyte)AssetType.Texture; + + if (part.Shape.ProjectionTextureUUID != UUID.Zero) + m_gatheredAssetUuids[part.Shape.ProjectionTextureUUID] = (sbyte)AssetType.Texture; + + if (part.CollisionSound != UUID.Zero) + m_gatheredAssetUuids[part.CollisionSound] = (sbyte)AssetType.Sound; + + if (part.ParticleSystem.Length > 0) + { + try + { + Primitive.ParticleSystem ps = new Primitive.ParticleSystem(part.ParticleSystem, 0); + if (ps.Texture != UUID.Zero) + m_gatheredAssetUuids[ps.Texture] = (sbyte)AssetType.Texture; + } + catch (Exception) + { + m_log.WarnFormat( + "[UUID GATHERER]: Could not check particle system for part {0} {1} in object {2} {3} since it is corrupt. Continuing.", + part.Name, part.UUID, sceneObject.Name, sceneObject.UUID); + } + } + + TaskInventoryDictionary taskDictionary = (TaskInventoryDictionary)part.TaskInventory.Clone(); + + // Now analyze this prim's inventory items to preserve all the uuids that they reference + foreach (TaskInventoryItem tii in taskDictionary.Values) + { + // m_log.DebugFormat( + // "[ARCHIVER]: Analysing item {0} asset type {1} in {2} {3}", + // tii.Name, tii.Type, part.Name, part.UUID); + + if (!m_gatheredAssetUuids.ContainsKey(tii.AssetID)) + RecordAssetUuids(tii.AssetID, (sbyte)tii.Type); + } + + // FIXME: We need to make gathering modular but we cannot yet, since gatherers are not guaranteed + // to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and + // inventory transfer. There needs to be a way for a module to register a method without assuming a + // Scene.EventManager is present. + // part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids); + + + // still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs + RecordMaterialsUuids(part); + } + catch (Exception e) + { + m_log.ErrorFormat("[UUID GATHERER]: Failed to get part - {0}", e); + m_log.DebugFormat( + "[UUID GATHERER]: Texture entry length for prim was {0} (min is 46)", + part.Shape.TextureEntry.Length); + } + } + } + + /// + /// Collect all the asset uuids found in one face of a Texture Entry. + /// + private void RecordTextureEntryAssetUuids(Primitive.TextureEntryFace texture) + { + m_gatheredAssetUuids[texture.TextureID] = (sbyte)AssetType.Texture; + + if (texture.MaterialID != UUID.Zero) + AddAssetUuidToInspect(texture.MaterialID); + } + + /// + /// Gather all of the texture asset UUIDs used to reference "Materials" such as normal and specular maps + /// stored in legacy format in part.DynAttrs + /// + /// + public void RecordMaterialsUuids(SceneObjectPart part) + { + // scan thru the dynAttrs map of this part for any textures used as materials + OSD osdMaterials = null; + + lock (part.DynAttrs) + { + if (part.DynAttrs.ContainsStore("OpenSim", "Materials")) + { + OSDMap materialsStore = part.DynAttrs.GetStore("OpenSim", "Materials"); + + if (materialsStore == null) + return; + + materialsStore.TryGetValue("Materials", out osdMaterials); + } + + if (osdMaterials != null) + { + //m_log.Info("[UUID Gatherer]: found Materials: " + OSDParser.SerializeJsonString(osd)); + + if (osdMaterials is OSDArray) + { + OSDArray matsArr = osdMaterials as OSDArray; + foreach (OSDMap matMap in matsArr) + { + try + { + if (matMap.ContainsKey("Material")) + { + OSDMap mat = matMap["Material"] as OSDMap; + if (mat.ContainsKey("NormMap")) + { + UUID normalMapId = mat["NormMap"].AsUUID(); + if (normalMapId != UUID.Zero) + { + m_gatheredAssetUuids[normalMapId] = (sbyte)AssetType.Texture; + //m_log.Info("[UUID Gatherer]: found normal map ID: " + normalMapId.ToString()); + } + } + if (mat.ContainsKey("SpecMap")) + { + UUID specularMapId = mat["SpecMap"].AsUUID(); + if (specularMapId != UUID.Zero) + { + m_gatheredAssetUuids[specularMapId] = (sbyte)AssetType.Texture; + //m_log.Info("[UUID Gatherer]: found specular map ID: " + specularMapId.ToString()); + } + } + } + + } + catch (Exception e) + { + m_log.Warn("[UUID Gatherer]: exception getting materials: " + e.Message); + } + } + } + } + } + } + + /// + /// Get an asset synchronously, potentially using an asynchronous callback. If the + /// asynchronous callback is used, we will wait for it to complete. + /// + /// + /// + protected virtual AssetBase GetAsset(UUID uuid) + { + return m_assetService.Get(uuid.ToString()); + } + + /// + /// Record the asset uuids embedded within the given text (e.g. a script). + /// + /// + private void RecordTextEmbeddedAssetUuids(AssetBase textAsset) + { + // m_log.DebugFormat("[ASSET GATHERER]: Getting assets for uuid references in asset {0}", embeddingAssetId); + + string script = Utils.BytesToString(textAsset.Data); + // m_log.DebugFormat("[ARCHIVER]: Script {0}", script); + MatchCollection uuidMatches = Util.PermissiveUUIDPattern.Matches(script); + // m_log.DebugFormat("[ARCHIVER]: Found {0} matches in text", uuidMatches.Count); + + foreach (Match uuidMatch in uuidMatches) + { + UUID uuid = new UUID(uuidMatch.Value); + // m_log.DebugFormat("[ARCHIVER]: Recording {0} in text", uuid); + + AddAssetUuidToInspect(uuid); + } + } + + /// + /// Record the uuids referenced by the given wearable asset + /// + /// + private void RecordWearableAssetUuids(AssetBase assetBase) + { + //m_log.Debug(new System.Text.ASCIIEncoding().GetString(bodypartAsset.Data)); + AssetWearable wearableAsset = new AssetBodypart(assetBase.FullID, assetBase.Data); + wearableAsset.Decode(); + + //m_log.DebugFormat( + // "[ARCHIVER]: Wearable asset {0} references {1} assets", wearableAssetUuid, wearableAsset.Textures.Count); + + foreach (UUID uuid in wearableAsset.Textures.Values) + m_gatheredAssetUuids[uuid] = (sbyte)AssetType.Texture; + } + + /// + /// Get all the asset uuids associated with a given object. This includes both those directly associated with + /// it (e.g. face textures) and recursively, those of items within it's inventory (e.g. objects contained + /// within this object). + /// + /// + private void RecordSceneObjectAssetUuids(AssetBase sceneObjectAsset) + { + string xml = Utils.BytesToString(sceneObjectAsset.Data); + + CoalescedSceneObjects coa; + if (CoalescedSceneObjectsSerializer.TryFromXml(xml, out coa)) + { + foreach (SceneObjectGroup sog in coa.Objects) + RecordAssetUuids(sog); + } + else + { + SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(xml); + + if (null != sog) + RecordAssetUuids(sog); + } + } + + /// + /// Get the asset uuid associated with a gesture + /// + /// + private void RecordGestureAssetUuids(AssetBase gestureAsset) + { + using (MemoryStream ms = new MemoryStream(gestureAsset.Data)) + using (StreamReader sr = new StreamReader(ms)) + { + sr.ReadLine(); // Unknown (Version?) + sr.ReadLine(); // Unknown + sr.ReadLine(); // Unknown + sr.ReadLine(); // Name + sr.ReadLine(); // Comment ? + int count = Convert.ToInt32(sr.ReadLine()); // Item count + + for (int i = 0 ; i < count ; i++) + { + string type = sr.ReadLine(); + if (type == null) + break; + string name = sr.ReadLine(); + if (name == null) + break; + string id = sr.ReadLine(); + if (id == null) + break; + string unknown = sr.ReadLine(); + if (unknown == null) + break; + + // If it can be parsed as a UUID, it is an asset ID + UUID uuid; + if (UUID.TryParse(id, out uuid)) + m_gatheredAssetUuids[uuid] = (sbyte)AssetType.Animation; // the asset is either an Animation or a Sound, but this distinction isn't important + } + } + } + + /// + /// Get the asset uuid's referenced in a material. + /// + private void RecordMaterialAssetUuids(AssetBase materialAsset) + { + OSDMap mat = (OSDMap)OSDParser.DeserializeLLSDXml(materialAsset.Data); + + UUID normMap = mat["NormMap"].AsUUID(); + if (normMap != UUID.Zero) + m_gatheredAssetUuids[normMap] = (sbyte)AssetType.Texture; + + UUID specMap = mat["SpecMap"].AsUUID(); + if (specMap != UUID.Zero) + m_gatheredAssetUuids[specMap] = (sbyte)AssetType.Texture; + } + } + + public class IteratingHGUuidGatherer : IteratingUuidGatherer + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + protected string m_assetServerURL; + + public IteratingHGUuidGatherer(IAssetService assetService, string assetServerURL) + : base(assetService) + { + m_assetServerURL = assetServerURL; + if (!m_assetServerURL.EndsWith("/") && !m_assetServerURL.EndsWith("=")) + m_assetServerURL = m_assetServerURL + "/"; + } + + protected override AssetBase GetAsset(UUID uuid) + { + if (string.Empty == m_assetServerURL) + return base.GetAsset(uuid); + else + return FetchAsset(uuid); + } + + public AssetBase FetchAsset(UUID assetID) + { // Test if it's already here AssetBase asset = m_assetService.Get(assetID.ToString()); if (asset == null)