add more cooldown to flotsam cache files expire

OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs

@@ -76,7 +76,7 @@ namespace OpenSim.Region.CoreModules.Asset
         private ulong m_MemoryHits;
         private ulong m_weakRefHits;
 
-        private HashSet<string> m_CurrentlyWriting = new HashSet<string>();
+        private static HashSet<string> m_CurrentlyWriting = new HashSet<string>();
 
         private bool m_FileCacheEnabled = true;
 
@@ -170,8 +170,7 @@ namespace OpenSim.Region.CoreModules.Asset
                         m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay);
 
                         m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration));
-                        m_FileExpirationCleanupTimer
+                        m_FileExpirationCleanupTimer = TimeSpan.FromHours(
-                            = TimeSpan.FromHours(
                                 assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours));
 
                         m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers);
@@ -749,7 +748,7 @@ namespace OpenSim.Region.CoreModules.Asset
         /// </summary>
         /// <param name="filename"></param>
         /// <param name="asset"></param>
-        private void WriteFileCache(string filename, AssetBase asset, bool replace)
+        private static void WriteFileCache(string filename, AssetBase asset, bool replace)
         {
              // Make sure the target cache directory exists
             string directory = Path.GetDirectoryName(filename);
@@ -791,9 +790,6 @@ namespace OpenSim.Region.CoreModules.Asset
                     if(replace)
                         File.Delete(filename);
                     File.Move(tempname, filename);
-
-                    if (m_LogLevel >= 2)
-                        m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID);
                 }
                 catch (IOException)
                 {
@@ -870,47 +866,59 @@ namespace OpenSim.Region.CoreModules.Asset
         /// to update the access time of all assets present in the scene or referenced by assets
         /// in the scene.
         /// </summary>
-        /// <param name="storeUncached">
+        /// <param name="tryGetUncached">
         /// If true, then assets scanned which are not found in cache are added to the cache.
         /// </param>
         /// <returns>Number of distinct asset references found in the scene.</returns>
-        private int TouchAllSceneAssets(bool storeUncached)
+        private int TouchAllSceneAssets(bool tryGetUncached)
         {
             UuidGatherer gatherer = new UuidGatherer(m_AssetService);
 
-            Dictionary<UUID, bool> assetsFound = new Dictionary<UUID, bool>();
+            int cooldown = 0;
-
             foreach (Scene s in m_Scenes)
             {
                 StampRegionStatusFile(s.RegionInfo.RegionID);
+                gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture1, (sbyte)AssetType.Texture);
+                gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture2, (sbyte)AssetType.Texture);
+                gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture3, (sbyte)AssetType.Texture);
+                gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture4, (sbyte)AssetType.Texture);
+                gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainImageID, (sbyte)AssetType.Texture);
 
                 s.ForEachSOG(delegate(SceneObjectGroup e)
                 {
-                    if(!m_timerRunning && !storeUncached)
+                    if(!m_timerRunning && !tryGetUncached)
                         return;
 
                     gatherer.AddForInspection(e);
                     gatherer.GatherAll();
 
-                    if(!m_timerRunning && !storeUncached)
+                    if (++cooldown > 100)
-                        return;
+                    {
-
-                    if(!storeUncached)
                         Thread.Sleep(50);
+                        cooldown = 0;
+                    }
                 });
-                if(!m_timerRunning && !storeUncached)
+                if(!m_timerRunning && !tryGetUncached)
                     break;
-                gatherer.GatherAll();
             }
 
             gatherer.GatherAll();
 
+            cooldown = 0;
             // windows does not update access time :(
             foreach(UUID id in gatherer.GatheredUuids.Keys)
             {
                 string idstr = id.ToString();
-                if(!UpdateFileLastAccessTime(GetFileName(idstr)))
+                if(!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached)
+                {
+                    cooldown += 50;
                     m_AssetService.Get(idstr);
+                }
+                if (++cooldown > 1000)
+                {
+                    Thread.Sleep(50);
+                    cooldown = 0;
+                }
             }
 
             int count = gatherer.GatheredUuids.Count;
@@ -918,7 +926,6 @@ namespace OpenSim.Region.CoreModules.Asset
             gatherer.FailedUUIDs.Clear();
             gatherer.UncertainAssetsUUIDs.Clear();
 
-            GC.Collect();
             return count;
         }
 
@@ -1164,7 +1171,11 @@ namespace OpenSim.Region.CoreModules.Asset
                             con.Output("{0} is not a valid date & time", cmd);
                             break;
                         }
-
+                        if (expirationDate >= DateTime.Now)
+                        {
+                            con.Output("{0} date & time must be in past", cmd);
+                            break;
+                        }
                         if (m_FileCacheEnabled)
                             CleanExpiredFiles(m_CacheDirectory, expirationDate);
                         else

OpenSim/Region/Framework/Scenes/Serialization/CoalescedSceneObjectsSerializer.cs

@@ -122,7 +122,7 @@ namespace OpenSim.Region.Framework.Scenes.Serialization
 
         public static bool TryFromXml(string xml, out CoalescedSceneObjects coa)
         {
-//            m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
+            //            m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
 
             coa = null;
 
@@ -137,9 +137,9 @@ namespace OpenSim.Region.Framework.Scenes.Serialization
 
                         if (reader.Name != "CoalescedObject")
                         {
-    //                        m_log.DebugFormat(
+                            //                        m_log.DebugFormat(
-    //                            "[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() root element was {0} so returning false",
+                            //                            "[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() root element was {0} so returning false",
-    //                            reader.Name);
+                            //                            reader.Name);
 
                             return false;
                         }
@@ -178,12 +178,73 @@ namespace OpenSim.Region.Framework.Scenes.Serialization
             }
             catch (Exception e)
             {
-                m_log.Error("[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of xml failed ",  e);
+                m_log.Error("[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of xml failed ", e);
                 Util.LogFailedXML("[COALESCED SCENE OBJECTS SERIALIZER]:", xml);
                 return false;
             }
 
             return true;
         }
+
+        public static bool TryFromXmlData(byte[] data, out CoalescedSceneObjects coa)
+        {
+            // m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
+
+            coa = null;
+            try
+            {
+                // Quickly check if this is a coalesced object, without fully parsing the XML
+                using (MemoryStream ms = new MemoryStream(data))
+                {
+                    using (XmlTextReader reader = new XmlTextReader(ms))
+                    {
+                        reader.MoveToContent(); // skip possible xml declaration
+
+                        if (reader.Name != "CoalescedObject")
+                        {
+                            return false;
+                        }
+                    }
+
+                    ms.Seek(0, SeekOrigin.Begin);
+                    XmlDocument doc = new XmlDocument();
+                    doc.Load(ms);
+                    XmlElement e = (XmlElement)doc.SelectSingleNode("/CoalescedObject");
+                    if (e == null)
+                        return false;
+
+                    coa = new CoalescedSceneObjects(UUID.Zero);
+
+                    XmlNodeList groups = e.SelectNodes("SceneObjectGroup");
+                    int i = 0;
+
+                    foreach (XmlNode n in groups)
+                    {
+                        SceneObjectGroup so = SceneObjectSerializer.FromOriginalXmlFormat(n.OuterXml);
+                        if (so != null)
+                        {
+                            coa.Add(so);
+                        }
+                        else
+                        {
+                            // XXX: Possibly we should fail outright here rather than continuing if a particular component of the
+                            // coalesced object fails to load.
+                            m_log.WarnFormat(
+                                "[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of xml for component {0} failed.  Continuing.",
+                                i);
+                        }
+
+                        i++;
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                m_log.Error("[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of binary xml failed ", e);
+                return false;
+            }
+
+            return true;
+        }
     }
 }

OpenSim/Region/Framework/Scenes/UuidGatherer.cs

@@ -293,6 +293,33 @@ namespace OpenSim.Region.Framework.Scenes
             possibleNotAssetCount = 0;
         }
 
+        public bool AddGathered(UUID uuid, sbyte type)
+        {
+            if (uuid == UUID.Zero)
+                return false;
+
+            if (ToSkip.Contains(uuid))
+                return false;
+
+            if (FailedUUIDs.Contains(uuid))
+            {
+                if (UncertainAssetsUUIDs.Contains(uuid))
+                    possibleNotAssetCount++;
+                else
+                    ErrorCount++;
+                return false;
+            }
+            if (GatheredUuids.ContainsKey(uuid))
+                return false;
+            if (m_assetUuidsToInspect.Contains(uuid))
+                return false;
+
+            //            m_log.DebugFormat("[UUID GATHERER]: Adding asset {0} for inspection", uuid);
+
+            GatheredUuids[uuid] = type; 
+            return true;
+        }
+
         /// <summary>
         /// Adds the asset uuid for inspection during the gathering process.
         /// </summary>
@@ -346,8 +373,8 @@ namespace OpenSim.Region.Framework.Scenes
             {
                 SceneObjectPart part = parts[i];
 
-                //                m_log.DebugFormat(
+                // m_log.DebugFormat(
-                //                    "[UUID GATHERER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID);
+                // "[UUID GATHERER]: Getting part {0}, {1} for object {2}", part.Name, part.UUID, sceneObject.UUID);
 
                 try
                 {
@@ -406,13 +433,6 @@ namespace OpenSim.Region.Framework.Scenes
                         AddForInspection(tii.AssetID, (sbyte)tii.Type);
                     }
 
-                    // FIXME: We need to make gathering modular but we cannot yet, since gatherers are not guaranteed
-                    // to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and
-                    // inventory transfer.  There needs to be a way for a module to register a method without assuming a
-                    // Scene.EventManager is present.
-                    //                    part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids);
-
-                    // still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs
                     RecordMaterialsUuids(part);
                 }
                 catch (Exception e)
@@ -778,10 +798,7 @@ namespace OpenSim.Region.Framework.Scenes
         /// <param name="sceneObjectAsset"></param>
         private void RecordSceneObjectAssetUuids(AssetBase sceneObjectAsset)
         {
-            string xml = Utils.BytesToString(sceneObjectAsset.Data);
+            if (CoalescedSceneObjectsSerializer.TryFromXmlData(sceneObjectAsset.Data, out CoalescedSceneObjects coa))
-
-            CoalescedSceneObjects coa;
-            if (CoalescedSceneObjectsSerializer.TryFromXml(xml, out coa))
             {
                 foreach (SceneObjectGroup sog in coa.Objects)
                 {
@@ -791,7 +808,7 @@ namespace OpenSim.Region.Framework.Scenes
             }
             else
             {
-                SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(xml);
+                SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(Utils.BytesToString(sceneObjectAsset.Data));
                 if (null != sog)
                 {
                     sog.TemporaryInstance = true;