diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index 3c632759c0..f2a5726529 100755 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -33,6 +33,7 @@ using System.Reflection; using System.Runtime.Serialization.Formatters.Binary; using System.Text; using System.Threading; +using System.Threading.Tasks; using System.Timers; using log4net; using Nini.Config; @@ -257,6 +258,7 @@ namespace OpenSim.Region.CoreModules.Asset { if(m_Scenes.Count <= 0) { + m_cleanupRunning = false; if (m_timerRunning) { m_timerRunning = false; @@ -676,48 +678,49 @@ namespace OpenSim.Region.CoreModules.Asset weakAssetReferences = new Dictionary(); } - private void CleanupExpiredFiles(object source, ElapsedEventArgs e) + private async void CleanupExpiredFiles(object source, ElapsedEventArgs e) { lock (timerLock) { - if(!m_timerRunning || m_cleanupRunning) + if (!m_timerRunning || m_cleanupRunning || !Directory.Exists(m_CacheDirectory)) return; m_cleanupRunning = true; } + // Purge all files last accessed prior to this point + await DoCleanExpiredFiles(DateTime.Now - m_FileExpiration).ConfigureAwait(false); + } + + private async Task DoCleanExpiredFiles(DateTime purgeLine) + { long heap = 0; //if (m_LogLevel >= 2) { - m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Start automatic Check for expired files older then {0}.", m_FileExpiration); + m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Start Check for expired files older then {0}.", m_FileExpiration); heap = GC.GetTotalMemory(false); } - // Purge all files last accessed prior to this point - DateTime purgeLine = DateTime.Now - m_FileExpiration; - // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore, // before cleaning up expired files we must scan the objects in the scene to make sure that we retain // such local assets if they have not been recently accessed. - TouchAllSceneAssets(false); - int cooldown = 0; - m_log.Info("[FLOTSAM ASSET CACHE] asset files expire"); - if (Directory.Exists(m_CacheDirectory)) - { - foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) - CleanExpiredFiles(dir, purgeLine, ref cooldown); - } + Dictionary gids = await gatherSceneAssets().ConfigureAwait(false); - lock(timerLock) + int cooldown = 0; + m_log.Info("[FLOTSAM ASSET CACHE] do asset files expire"); + foreach (string subdir in Directory.GetDirectories(m_CacheDirectory)) + CleanExpiredFiles(subdir, gids, purgeLine, ref cooldown); + + lock (timerLock) { - if(m_timerRunning) + if (m_timerRunning) m_CacheCleanTimer.Start(); m_cleanupRunning = false; } //if (m_LogLevel >= 2) { heap = GC.GetTotalMemory(false) - heap; - double fheap = Math.Round((double)(heap / (1024 * 1024)),3); - m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Finished automatic Check for expired files heap delta: {0}MB.", fheap); + double fheap = Math.Round((double)(heap / (1024 * 1024)), 3); + m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Finished Check for expired files, heap delta: {0}MB.", fheap); } } @@ -787,6 +790,75 @@ namespace OpenSim.Region.CoreModules.Asset } } + private void CleanExpiredFiles(string dir, Dictionary gids, DateTime purgeLine, ref int cooldown) + { + try + { + if (!Directory.Exists(dir)) + return; + + int dirSize = 0; + foreach (string file in Directory.GetFiles(dir)) + { + + if (!m_cleanupRunning) + return; + + ++dirSize; + string id = Path.GetFileName(file); + if (String.IsNullOrEmpty(id)) + continue; //?? + + if(UUID.TryParse(id, out UUID uid) && gids.ContainsKey(uid)) + continue; + + if (File.GetLastAccessTime(file) < purgeLine) + { + File.Delete(file); + cooldown += 2; + lock (weakAssetReferencesLock) + weakAssetReferences.Remove(id); + } + if (++cooldown >= 30) + { + Thread.Sleep(50); + cooldown = 0; + } + } + + // Recurse into lower tiers + foreach (string subdir in Directory.GetDirectories(dir)) + { + if (!m_cleanupRunning) + return; + + ++dirSize; + CleanExpiredFiles(subdir, gids, purgeLine, ref cooldown); + } + + // Check if a tier directory is empty, if so, delete it + if (m_cleanupRunning && dirSize == 0) + { + Directory.Delete(dir); + } + else if (dirSize >= m_CacheWarnAt) + { + m_log.WarnFormat( + "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", + dir, dirSize); + } + } + catch (DirectoryNotFoundException) + { + // If we get here, another node on the same box has + // already removed the directory. Continue with next. + } + catch (Exception e) + { + m_log.WarnFormat("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception {1}", dir, e.Message); + } + } + /// /// Determines the filename for an AssetID stored in the file cache /// @@ -950,11 +1022,39 @@ namespace OpenSim.Region.CoreModules.Asset /// If true, then assets scanned which are not found in cache are added to the cache. /// /// Number of distinct asset references found in the scene. - private int TouchAllSceneAssets(bool tryGetUncached) + private async Task TouchAllSceneAssets(bool tryGetUncached) { m_log.Info("[FLOTSAM ASSET CACHE] start touch files of assets in use"); - UuidGatherer gatherer = new UuidGatherer(m_AssetService); + Dictionary gatheredids = await gatherSceneAssets(); + + int cooldown = 0; + foreach(UUID id in gatheredids.Keys) + { + if (!m_cleanupRunning) + break; + + string idstr = id.ToString(); + if (!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached) + { + cooldown += 5; + m_AssetService.Get(idstr); + } + if (++cooldown > 50) + { + Thread.Sleep(50); + cooldown = 0; + } + } + return gatheredids.Count; + } + + private async Task> gatherSceneAssets() + { + m_log.Info("[FLOTSAM ASSET CACHE] gather assets in use"); + + Dictionary gatheredids = new Dictionary(); + UuidGatherer gatherer = new UuidGatherer(m_AssetService, gatheredids); int cooldown = 0; foreach (Scene s in m_Scenes) @@ -966,60 +1066,62 @@ namespace OpenSim.Region.CoreModules.Asset gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture4, (sbyte)AssetType.Texture); gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainImageID, (sbyte)AssetType.Texture); + if (s.RegionEnvironment != null) + s.RegionEnvironment.GatherAssets(gatheredids); + + if (s.LandChannel != null) + { + List landObjects = s.LandChannel.AllParcels(); + foreach (ILandObject lo in landObjects) + { + if (lo.LandData != null && lo.LandData.Environment != null) + lo.LandData.Environment.GatherAssets(gatheredids); + } + } + EntityBase[] entities = s.Entities.GetEntities(); for (int i = 0; i < entities.Length; ++i) { - if (!m_timerRunning && !tryGetUncached) + if (!m_cleanupRunning) break; EntityBase entity = entities[i]; - if(entity is SceneObjectGroup) + if (entity is SceneObjectGroup) { SceneObjectGroup e = entity as SceneObjectGroup; - if(e.IsDeleted) + if (e.IsDeleted) continue; gatherer.AddForInspection(e); - while(gatherer.GatherNext()) + while (gatherer.GatherNext()) { if (++cooldown > 50) { - Thread.Sleep(50); + await Task.Delay(60).ConfigureAwait(false); cooldown = 0; } } + if (++cooldown > 25) + { + await Task.Delay(60).ConfigureAwait(false); + cooldown = 0; + } } } entities = null; - if (!m_timerRunning && !tryGetUncached) + if (!m_cleanupRunning) break; } gatherer.GatherAll(); - cooldown = 0; - foreach(UUID id in gatherer.GatheredUuids.Keys) - { - string idstr = id.ToString(); - if(!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached) - { - cooldown += 5; - m_AssetService.Get(idstr); - } - if (++cooldown > 50) - { - Thread.Sleep(50); - cooldown = 0; - } - } - - int count = gatherer.GatheredUuids.Count; - gatherer.GatheredUuids.Clear(); gatherer.FailedUUIDs.Clear(); gatherer.UncertainAssetsUUIDs.Clear(); gatherer = null; - return count; + + m_log.InfoFormat("[FLOTSAM ASSET CACHE] found {0} assets in use", gatheredids.Count); + return gatheredids; } /// @@ -1199,9 +1301,9 @@ namespace OpenSim.Region.CoreModules.Asset break; case "assets": - lock(timerLock) + lock (timerLock) { - if(m_cleanupRunning) + if (m_cleanupRunning) { con.Output("Flotsam assets check already running"); return; @@ -1211,7 +1313,7 @@ namespace OpenSim.Region.CoreModules.Asset con.Output("Flotsam Ensuring assets are cached for all scenes."); - WorkManager.RunInThreadPool(delegate + WorkManager.RunInThreadPool(async delegate { bool wasRunning= false; lock(timerLock) @@ -1221,10 +1323,13 @@ namespace OpenSim.Region.CoreModules.Asset m_CacheCleanTimer.Stop(); m_timerRunning = false; wasRunning = true; - Thread.Sleep(100); } } - int assetReferenceTotal = TouchAllSceneAssets(true); + + if (wasRunning) + await Task.Delay(100).ConfigureAwait(false); + + int assetReferenceTotal = await TouchAllSceneAssets(true).ConfigureAwait(false); lock(timerLock) { @@ -1241,6 +1346,16 @@ namespace OpenSim.Region.CoreModules.Asset break; case "expire": + lock (timerLock) + { + if (m_cleanupRunning) + { + con.Output("Flotsam assets check already running"); + return; + } + m_cleanupRunning = true; + } + if (cmdparams.Length < 3) { con.Output("Invalid parameters for Expire, please specify a valid date & time"); @@ -1271,9 +1386,34 @@ namespace OpenSim.Region.CoreModules.Asset } if (m_FileCacheEnabled) { - TouchAllSceneAssets(false); - int cooldown = 0; - CleanExpiredFiles(m_CacheDirectory, expirationDate, ref cooldown); + WorkManager.RunInThreadPool(async delegate + { + bool wasRunning = false; + lock (timerLock) + { + if (m_timerRunning) + { + m_CacheCleanTimer.Stop(); + m_timerRunning = false; + wasRunning = true; + } + } + + if(wasRunning) + await Task.Delay(100).ConfigureAwait(false); + + await DoCleanExpiredFiles(expirationDate).ConfigureAwait(false); + + lock (timerLock) + { + if (wasRunning) + { + m_CacheCleanTimer.Start(); + m_timerRunning = true; + } + m_cleanupRunning = false; + } + }, null, "TouchAllSceneAssets", false); } else con.Output("File cache not active, not clearing."); diff --git a/bin/OpenSimDefaults.ini b/bin/OpenSimDefaults.ini index 51a8bb9dc3..7f88d021c9 100644 --- a/bin/OpenSimDefaults.ini +++ b/bin/OpenSimDefaults.ini @@ -1802,7 +1802,7 @@ ; AppDomainLoading = false ; Set this to true to load attachment scripts in separated domain, if AppDomainLoading is false - ; same issues as AppDomainLoading, but my be useful on regions with a lot of avatars traffic, if they carry scripts like AOs + ; same issues as AppDomainLoading, but mya be useful on regions with a lot of avatars traffic, if they carry scripts like AOs ; AttachmentsDomainLoading = false ; Controls whether previously compiled scripts DLLs are deleted on sim restart. @@ -2066,7 +2066,7 @@ ;XmlRpcServiceWriteKey = 1234 ; Disables HTTP Keep-Alive for XmlRpcGroupsServicesConnector HTTP Requests, - ; only set to false it if you absolute sure regions and groups server support it. + ; only set to false it if you absolutely sure regions and groups server supports it. ; XmlRpcDisableKeepAlive = true ; Minimum user level required to create groups