flotsam: deeper changes on files expire

OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs

@@ -33,6 +33,7 @@ using System.Reflection;
 using System.Runtime.Serialization.Formatters.Binary;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using System.Timers;
 using log4net;
 using Nini.Config;
@@ -257,6 +258,7 @@ namespace OpenSim.Region.CoreModules.Asset
                 {
                     if(m_Scenes.Count <= 0)
                     {
+                        m_cleanupRunning = false;
                         if (m_timerRunning)
                         {
                             m_timerRunning = false;
@@ -676,48 +678,49 @@ namespace OpenSim.Region.CoreModules.Asset
                 weakAssetReferences = new Dictionary<string, WeakReference>();
         }
 
-        private void CleanupExpiredFiles(object source, ElapsedEventArgs e)
+        private async void CleanupExpiredFiles(object source, ElapsedEventArgs e)
         {
             lock (timerLock)
             {
-                if(!m_timerRunning || m_cleanupRunning)
+                if (!m_timerRunning || m_cleanupRunning || !Directory.Exists(m_CacheDirectory))
                     return;
                 m_cleanupRunning = true;
             }
 
+            // Purge all files last accessed prior to this point
+            await DoCleanExpiredFiles(DateTime.Now - m_FileExpiration).ConfigureAwait(false);
+        }
+
+        private async Task DoCleanExpiredFiles(DateTime purgeLine)
+        {
             long heap = 0;
             //if (m_LogLevel >= 2)
             {
-                m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Start automatic Check for expired files older then {0}.", m_FileExpiration);
+                m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Start Check for expired files older then {0}.", m_FileExpiration);
                 heap = GC.GetTotalMemory(false);
             }
 
-            // Purge all files last accessed prior to this point
-            DateTime purgeLine = DateTime.Now - m_FileExpiration;
-
             // An asset cache may contain local non-temporary assets that are not in the asset service.  Therefore,
             // before cleaning up expired files we must scan the objects in the scene to make sure that we retain
             // such local assets if they have not been recently accessed.
-            TouchAllSceneAssets(false);
+            Dictionary<UUID,sbyte> gids = await gatherSceneAssets().ConfigureAwait(false);
-            int cooldown = 0;
-            m_log.Info("[FLOTSAM ASSET CACHE] asset files expire");
-            if (Directory.Exists(m_CacheDirectory))
-            {
-                foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
-                    CleanExpiredFiles(dir, purgeLine, ref cooldown);
-            }
 
-            lock(timerLock)
+            int cooldown = 0;
+            m_log.Info("[FLOTSAM ASSET CACHE] do asset files expire");
+            foreach (string subdir in Directory.GetDirectories(m_CacheDirectory))
+                CleanExpiredFiles(subdir, gids, purgeLine, ref cooldown);
+
+            lock (timerLock)
             {
-                if(m_timerRunning)
+                if (m_timerRunning)
                     m_CacheCleanTimer.Start();
                 m_cleanupRunning = false;
             }
             //if (m_LogLevel >= 2)
             {
                 heap = GC.GetTotalMemory(false) - heap;
-                double fheap = Math.Round((double)(heap / (1024 * 1024)),3);
+                double fheap = Math.Round((double)(heap / (1024 * 1024)), 3);
-                m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Finished automatic Check for expired files heap delta: {0}MB.", fheap);
+                m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Finished Check for expired files, heap delta: {0}MB.", fheap);
             }
         }
 
@@ -787,6 +790,75 @@ namespace OpenSim.Region.CoreModules.Asset
             }
         }
 
+        private void CleanExpiredFiles(string dir, Dictionary<UUID, sbyte> gids, DateTime purgeLine, ref int cooldown)
+        {
+            try
+            {
+                if (!Directory.Exists(dir))
+                    return;
+
+                int dirSize = 0;
+                foreach (string file in Directory.GetFiles(dir))
+                {
+
+                    if (!m_cleanupRunning)
+                        return;
+
+                    ++dirSize;
+                    string id = Path.GetFileName(file);
+                    if (String.IsNullOrEmpty(id))
+                        continue; //??
+
+                    if(UUID.TryParse(id, out UUID uid) && gids.ContainsKey(uid))
+                        continue;
+
+                    if (File.GetLastAccessTime(file) < purgeLine)
+                    {
+                        File.Delete(file);
+                        cooldown += 2;
+                        lock (weakAssetReferencesLock)
+                            weakAssetReferences.Remove(id);
+                    }
+                    if (++cooldown >= 30)
+                    {
+                        Thread.Sleep(50);
+                        cooldown = 0;
+                    }
+                }
+
+                // Recurse into lower tiers
+                foreach (string subdir in Directory.GetDirectories(dir))
+                {
+                    if (!m_cleanupRunning)
+                        return;
+
+                    ++dirSize;
+                    CleanExpiredFiles(subdir, gids, purgeLine, ref cooldown);
+                }
+
+                // Check if a tier directory is empty, if so, delete it
+                if (m_cleanupRunning && dirSize == 0)
+                {
+                    Directory.Delete(dir);
+                }
+                else if (dirSize >= m_CacheWarnAt)
+                {
+                    m_log.WarnFormat(
+                        "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}.  Suggest increasing tiers, tier length, or reducing cache expiration",
+                        dir, dirSize);
+                }
+            }
+            catch (DirectoryNotFoundException)
+            {
+                // If we get here, another node on the same box has
+                // already removed the directory. Continue with next.
+            }
+            catch (Exception e)
+            {
+                m_log.WarnFormat("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception {1}", dir, e.Message);
+            }
+        }
+
         /// <summary>
         /// Determines the filename for an AssetID stored in the file cache
         /// </summary>
@@ -950,11 +1022,39 @@ namespace OpenSim.Region.CoreModules.Asset
         /// If true, then assets scanned which are not found in cache are added to the cache.
         /// </param>
         /// <returns>Number of distinct asset references found in the scene.</returns>
-        private int TouchAllSceneAssets(bool tryGetUncached)
+        private async Task<int> TouchAllSceneAssets(bool tryGetUncached)
         {
             m_log.Info("[FLOTSAM ASSET CACHE] start touch files of assets in use");
 
-            UuidGatherer gatherer = new UuidGatherer(m_AssetService);
+            Dictionary<UUID,sbyte> gatheredids = await gatherSceneAssets();
+
+            int cooldown = 0;
+            foreach(UUID id in gatheredids.Keys)
+            {
+                if (!m_cleanupRunning)
+                    break;
+
+                string idstr = id.ToString();
+                if (!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached)
+                {
+                    cooldown += 5;
+                    m_AssetService.Get(idstr);
+                }
+                if (++cooldown > 50)
+                {
+                    Thread.Sleep(50);
+                    cooldown = 0;
+                }
+            }
+            return gatheredids.Count;
+        }
+
+        private async Task<Dictionary<UUID, sbyte>> gatherSceneAssets()
+        {
+            m_log.Info("[FLOTSAM ASSET CACHE] gather assets in use");
+
+            Dictionary<UUID, sbyte> gatheredids = new Dictionary<UUID, sbyte>();
+            UuidGatherer gatherer = new UuidGatherer(m_AssetService, gatheredids);
 
             int cooldown = 0;
             foreach (Scene s in m_Scenes)
@@ -966,60 +1066,62 @@ namespace OpenSim.Region.CoreModules.Asset
                 gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture4, (sbyte)AssetType.Texture);
                 gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainImageID, (sbyte)AssetType.Texture);
 
+                if (s.RegionEnvironment != null)
+                    s.RegionEnvironment.GatherAssets(gatheredids);
+
+                if (s.LandChannel != null)
+                {
+                    List<ILandObject> landObjects = s.LandChannel.AllParcels();
+                    foreach (ILandObject lo in landObjects)
+                    {
+                        if (lo.LandData != null && lo.LandData.Environment != null)
+                            lo.LandData.Environment.GatherAssets(gatheredids);
+                    }
+                }
+
                 EntityBase[] entities = s.Entities.GetEntities();
                 for (int i = 0; i < entities.Length; ++i)
                 {
-                    if (!m_timerRunning && !tryGetUncached)
+                    if (!m_cleanupRunning)
                         break;
 
                     EntityBase entity = entities[i];
-                    if(entity is SceneObjectGroup)
+                    if (entity is SceneObjectGroup)
                     {
                         SceneObjectGroup e = entity as SceneObjectGroup;
-                        if(e.IsDeleted)
+                        if (e.IsDeleted)
                             continue;
 
                         gatherer.AddForInspection(e);
-                        while(gatherer.GatherNext())
+                        while (gatherer.GatherNext())
                         {
                             if (++cooldown > 50)
                             {
-                                Thread.Sleep(50);
+                                await Task.Delay(60).ConfigureAwait(false);
+                                cooldown = 0;
+                            }
+                        }
+                        if (++cooldown > 25)
+                        {
+                            await Task.Delay(60).ConfigureAwait(false);
                             cooldown = 0;
                         }
                     }
                 }
-                }
                 entities = null;
-                if (!m_timerRunning && !tryGetUncached)
+                if (!m_cleanupRunning)
                     break;
             }
 
             gatherer.GatherAll();
 
-            cooldown = 0;
-            foreach(UUID id in gatherer.GatheredUuids.Keys)
-            {
-                string idstr = id.ToString();
-                if(!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached)
-                {
-                    cooldown += 5;
-                    m_AssetService.Get(idstr);
-                }
-                if (++cooldown > 50)
-                {
-                    Thread.Sleep(50);
-                    cooldown = 0;
-                }
-            }
-
-            int count = gatherer.GatheredUuids.Count;
-            gatherer.GatheredUuids.Clear();
             gatherer.FailedUUIDs.Clear();
             gatherer.UncertainAssetsUUIDs.Clear();
             gatherer = null;
 
-            return count;
+
+            m_log.InfoFormat("[FLOTSAM ASSET CACHE]     found {0} assets in use", gatheredids.Count);
+            return gatheredids;
         }
 
         /// <summary>
@@ -1199,9 +1301,9 @@ namespace OpenSim.Region.CoreModules.Asset
                         break;
 
                     case "assets":
-                        lock(timerLock)
+                        lock (timerLock)
                         {
-                            if(m_cleanupRunning)
+                            if (m_cleanupRunning)
                             {
                                 con.Output("Flotsam assets check already running");
                                 return;
@@ -1211,7 +1313,7 @@ namespace OpenSim.Region.CoreModules.Asset
 
                         con.Output("Flotsam Ensuring assets are cached for all scenes.");
 
-                        WorkManager.RunInThreadPool(delegate
+                        WorkManager.RunInThreadPool(async delegate
                         {
                             bool wasRunning= false;
                             lock(timerLock)
@@ -1221,10 +1323,13 @@ namespace OpenSim.Region.CoreModules.Asset
                                     m_CacheCleanTimer.Stop();
                                     m_timerRunning = false;
                                     wasRunning = true;
-                                    Thread.Sleep(100);
                                 }
                             }
-                            int assetReferenceTotal = TouchAllSceneAssets(true);
+
+                            if (wasRunning)
+                                await Task.Delay(100).ConfigureAwait(false);
+
+                            int assetReferenceTotal = await TouchAllSceneAssets(true).ConfigureAwait(false);
 
                             lock(timerLock)
                             {
@@ -1241,6 +1346,16 @@ namespace OpenSim.Region.CoreModules.Asset
                         break;
 
                     case "expire":
+                        lock (timerLock)
+                        {
+                            if (m_cleanupRunning)
+                            {
+                                con.Output("Flotsam assets check already running");
+                                return;
+                            }
+                            m_cleanupRunning = true;
+                        }
+
                         if (cmdparams.Length < 3)
                         {
                             con.Output("Invalid parameters for Expire, please specify a valid date & time");
@@ -1271,9 +1386,34 @@ namespace OpenSim.Region.CoreModules.Asset
                         }
                         if (m_FileCacheEnabled)
                         {
-                            TouchAllSceneAssets(false);
+                            WorkManager.RunInThreadPool(async delegate
-                            int cooldown = 0;
+                            {
-                            CleanExpiredFiles(m_CacheDirectory, expirationDate, ref cooldown);
+                                bool wasRunning = false;
+                                lock (timerLock)
+                                {
+                                    if (m_timerRunning)
+                                    {
+                                        m_CacheCleanTimer.Stop();
+                                        m_timerRunning = false;
+                                        wasRunning = true;
+                                    }
+                                }
+
+                                if(wasRunning)
+                                    await Task.Delay(100).ConfigureAwait(false);
+
+                                await DoCleanExpiredFiles(expirationDate).ConfigureAwait(false);
+
+                                lock (timerLock)
+                                {
+                                    if (wasRunning)
+                                    {
+                                        m_CacheCleanTimer.Start();
+                                        m_timerRunning = true;
+                                    }
+                                    m_cleanupRunning = false;
+                                }
+                            }, null, "TouchAllSceneAssets", false);
                         }
                         else
                             con.Output("File cache not active, not clearing.");

bin/OpenSimDefaults.ini

@@ -1802,7 +1802,7 @@
     ; AppDomainLoading = false
 
     ; Set this to true to load attachment scripts in separated domain, if AppDomainLoading is false
-    ; same issues as AppDomainLoading, but my be useful on regions with a lot of avatars traffic, if they carry scripts like AOs
+    ; same issues as AppDomainLoading, but mya be useful on regions with a lot of avatars traffic, if they carry scripts like AOs
     ; AttachmentsDomainLoading = false
 
     ; Controls whether previously compiled scripts DLLs are deleted on sim restart.
@@ -2066,7 +2066,7 @@
     ;XmlRpcServiceWriteKey   = 1234
 
     ; Disables HTTP Keep-Alive for XmlRpcGroupsServicesConnector HTTP Requests,
-    ; only set to false it if you absolute sure regions and groups server support it.
+    ; only set to false it if you absolutely sure regions and groups server supports it.
     ; XmlRpcDisableKeepAlive = true
 
     ; Minimum user level required to create groups