OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add more cooldown to flotsam cache files expire

master
UbitUmarov 2020-05-18 00:20:28 +01:00
parent fe72b54652
commit 4fa8e20aab
3 changed files with 129 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
View File

@ -76,7 +76,7 @@ namespace OpenSim.Region.CoreModules.Asset
private ulong m_MemoryHits;
private ulong m_weakRefHits;
private HashSet<string> m_CurrentlyWriting = new HashSet<string>();
private static HashSet<string> m_CurrentlyWriting = new HashSet<string>();
private bool m_FileCacheEnabled = true;
@ -170,8 +170,7 @@ namespace OpenSim.Region.CoreModules.Asset
m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay);
m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration));
m_FileExpirationCleanupTimer
= TimeSpan.FromHours(
m_FileExpirationCleanupTimer = TimeSpan.FromHours(
assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours));
m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers);
@ -749,7 +748,7 @@ namespace OpenSim.Region.CoreModules.Asset
/// </summary>
/// <param name="filename"></param>
/// <param name="asset"></param>
private void WriteFileCache(string filename, AssetBase asset, bool replace)
private static void WriteFileCache(string filename, AssetBase asset, bool replace)
{
// Make sure the target cache directory exists
string directory = Path.GetDirectoryName(filename);
@ -791,9 +790,6 @@ namespace OpenSim.Region.CoreModules.Asset
if(replace)
File.Delete(filename);
File.Move(tempname, filename);
if (m_LogLevel >= 2)
m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID);
}
catch (IOException)
{
@ -870,55 +866,66 @@ namespace OpenSim.Region.CoreModules.Asset
/// to update the access time of all assets present in the scene or referenced by assets
/// in the scene.
/// </summary>
/// <param name="storeUncached">
/// <param name="tryGetUncached">
/// If true, then assets scanned which are not found in cache are added to the cache.
/// </param>
/// <returns>Number of distinct asset references found in the scene.</returns>
private int TouchAllSceneAssets(bool storeUncached)
private int TouchAllSceneAssets(bool tryGetUncached)
{
UuidGatherer gatherer = new UuidGatherer(m_AssetService);
Dictionary<UUID, bool> assetsFound = new Dictionary<UUID, bool>();
int cooldown = 0;
foreach (Scene s in m_Scenes)
{
StampRegionStatusFile(s.RegionInfo.RegionID);
gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture1, (sbyte)AssetType.Texture);
gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture2, (sbyte)AssetType.Texture);
gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture3, (sbyte)AssetType.Texture);
gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainTexture4, (sbyte)AssetType.Texture);
gatherer.AddGathered(s.RegionInfo.RegionSettings.TerrainImageID, (sbyte)AssetType.Texture);
s.ForEachSOG(delegate(SceneObjectGroup e)
{
if(!m_timerRunning && !storeUncached)
if(!m_timerRunning && !tryGetUncached)
return;
gatherer.AddForInspection(e);
gatherer.GatherAll();
if(!m_timerRunning && !storeUncached)
return;
if(!storeUncached)
if (++cooldown > 100)
{
Thread.Sleep(50);
cooldown = 0;
}
});
if(!m_timerRunning && !storeUncached)
if(!m_timerRunning && !tryGetUncached)
break;
gatherer.GatherAll();
}
gatherer.GatherAll();
cooldown = 0;
// windows does not update access time :(
foreach(UUID id in gatherer.GatheredUuids.Keys)
{
string idstr = id.ToString();
if(!UpdateFileLastAccessTime(GetFileName(idstr)))
if(!UpdateFileLastAccessTime(GetFileName(idstr)) && tryGetUncached)
{
cooldown += 50;
m_AssetService.Get(idstr);
}
if (++cooldown > 1000)
{
Thread.Sleep(50);
cooldown = 0;
}
}
int count = gatherer.GatheredUuids.Count;
gatherer.GatheredUuids.Clear();
gatherer.FailedUUIDs.Clear();
gatherer.UncertainAssetsUUIDs.Clear();
GC.Collect();
return count;
}
@ -1164,7 +1171,11 @@ namespace OpenSim.Region.CoreModules.Asset
con.Output("{0} is not a valid date & time", cmd);
break;
}
if (expirationDate >= DateTime.Now)
{
con.Output("{0} date & time must be in past", cmd);
break;
}
if (m_FileCacheEnabled)
CleanExpiredFiles(m_CacheDirectory, expirationDate);
else

63
OpenSim/Region/Framework/Scenes/Serialization/CoalescedSceneObjectsSerializer.cs
View File

@ -122,7 +122,7 @@ namespace OpenSim.Region.Framework.Scenes.Serialization
public static bool TryFromXml(string xml, out CoalescedSceneObjects coa)
{
// m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
// m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
coa = null;
@ -185,5 +185,66 @@ namespace OpenSim.Region.Framework.Scenes.Serialization
return true;
}
public static bool TryFromXmlData(byte[] data, out CoalescedSceneObjects coa)
{
// m_log.DebugFormat("[COALESCED SCENE OBJECTS SERIALIZER]: TryFromXml() deserializing {0}", xml);
coa = null;
try
{
// Quickly check if this is a coalesced object, without fully parsing the XML
using (MemoryStream ms = new MemoryStream(data))
{
using (XmlTextReader reader = new XmlTextReader(ms))
{
reader.MoveToContent(); // skip possible xml declaration
if (reader.Name != "CoalescedObject")
{
return false;
}
}
ms.Seek(0, SeekOrigin.Begin);
XmlDocument doc = new XmlDocument();
doc.Load(ms);
XmlElement e = (XmlElement)doc.SelectSingleNode("/CoalescedObject");
if (e == null)
return false;
coa = new CoalescedSceneObjects(UUID.Zero);
XmlNodeList groups = e.SelectNodes("SceneObjectGroup");
int i = 0;
foreach (XmlNode n in groups)
{
SceneObjectGroup so = SceneObjectSerializer.FromOriginalXmlFormat(n.OuterXml);
if (so != null)
{
coa.Add(so);
}
else
{
// XXX: Possibly we should fail outright here rather than continuing if a particular component of the
// coalesced object fails to load.
m_log.WarnFormat(
"[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of xml for component {0} failed. Continuing.",
i);
}
i++;
}
}
}
catch (Exception e)
{
m_log.Error("[COALESCED SCENE OBJECTS SERIALIZER]: Deserialization of binary xml failed ", e);
return false;
}
return true;
}
}
}

41
OpenSim/Region/Framework/Scenes/UuidGatherer.cs
View File

@ -293,6 +293,33 @@ namespace OpenSim.Region.Framework.Scenes
possibleNotAssetCount = 0;
}
public bool AddGathered(UUID uuid, sbyte type)
{
if (uuid == UUID.Zero)
return false;
if (ToSkip.Contains(uuid))
return false;
if (FailedUUIDs.Contains(uuid))
{
if (UncertainAssetsUUIDs.Contains(uuid))
possibleNotAssetCount++;
else
ErrorCount++;
return false;
}
if (GatheredUuids.ContainsKey(uuid))
return false;
if (m_assetUuidsToInspect.Contains(uuid))
return false;
// m_log.DebugFormat("[UUID GATHERER]: Adding asset {0} for inspection", uuid);
GatheredUuids[uuid] = type;
return true;
}
/// <summary>
/// Adds the asset uuid for inspection during the gathering process.
/// </summary>
@ -406,13 +433,6 @@ namespace OpenSim.Region.Framework.Scenes
AddForInspection(tii.AssetID, (sbyte)tii.Type);
}
// FIXME: We need to make gathering modular but we cannot yet, since gatherers are not guaranteed
// to be called with scene objects that are in a scene (e.g. in the case of hg asset mapping and
// inventory transfer. There needs to be a way for a module to register a method without assuming a
// Scene.EventManager is present.
// part.ParentGroup.Scene.EventManager.TriggerGatherUuids(part, assetUuids);
// still needed to retrieve textures used as materials for any parts containing legacy materials stored in DynAttrs
RecordMaterialsUuids(part);
}
catch (Exception e)
@ -778,10 +798,7 @@ namespace OpenSim.Region.Framework.Scenes
/// <param name="sceneObjectAsset"></param>
private void RecordSceneObjectAssetUuids(AssetBase sceneObjectAsset)
{
string xml = Utils.BytesToString(sceneObjectAsset.Data);
CoalescedSceneObjects coa;
if (CoalescedSceneObjectsSerializer.TryFromXml(xml, out coa))
if (CoalescedSceneObjectsSerializer.TryFromXmlData(sceneObjectAsset.Data, out CoalescedSceneObjects coa))
{
foreach (SceneObjectGroup sog in coa.Objects)
{
@ -791,7 +808,7 @@ namespace OpenSim.Region.Framework.Scenes
}
else
{
SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(xml);
SceneObjectGroup sog = SceneObjectSerializer.FromOriginalXmlFormat(Utils.BytesToString(sceneObjectAsset.Data));
if (null != sog)
{
sog.TemporaryInstance = true;