From c43e466301afd6dc83f473ef98a14fa8cd775181 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Sun, 9 Nov 2008 15:00:26 +0000 Subject: [PATCH] * Enabled GTCache for AssetCache * Items will now be locally cached for only 24 hours from last access. (Rather than until restart) * Caveat: Implementing the new caching mechanism means statistics gathering on AssetCache is no longer functional. (Justin - you might want to take a look and see if you can somehow get that back and running if you still need it) --- .../Communications/Cache/AssetCache.cs | 268 +++++------------- .../Region/ClientStack/ClientStackManager.cs | 32 ++- .../ClientStack/LindenUDP/LLClientView.cs | 2 +- prebuild.xml | 2 +- 4 files changed, 100 insertions(+), 204 deletions(-) diff --git a/OpenSim/Framework/Communications/Cache/AssetCache.cs b/OpenSim/Framework/Communications/Cache/AssetCache.cs index dfdb0e7822..0581cc69db 100644 --- a/OpenSim/Framework/Communications/Cache/AssetCache.cs +++ b/OpenSim/Framework/Communications/Cache/AssetCache.cs @@ -33,6 +33,7 @@ using OpenMetaverse; using OpenMetaverse.Packets; using log4net; using OpenSim.Framework.Statistics; +using GlynnTucker.Cache; namespace OpenSim.Framework.Communications.Cache { @@ -52,18 +53,20 @@ namespace OpenSim.Framework.Communications.Cache /// public class AssetCache : IAssetReceiver { + protected ICache m_memcache = new SimpleMemoryCache(); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); /// /// The cache of assets. This does not include textures. /// - private Dictionary Assets; + //private Dictionary Assets; /// /// The cache of textures. /// - private Dictionary Textures; + //private Dictionary Textures; /// /// Assets requests which are waiting for asset server data. This includes texture requests @@ -95,52 +98,11 @@ namespace OpenSim.Framework.Communications.Cache /// public void ShowState() { - m_log.InfoFormat("Assets:{0} Textures:{1} RequestLists:{2}", - Assets.Count, - Textures.Count, + m_log.InfoFormat("Memcache:{1} RequestLists:{2}", + m_memcache.Count, // AssetRequests.Count, // RequestedAssets.Count, RequestLists.Count); - - int temporaryImages = 0; - int temporaryAssets = 0; - - long imageBytes = 0; - long assetBytes = 0; - - foreach (TextureImage texture in Textures.Values) - { - if (texture != null) - { - if (texture.Temporary) - { - temporaryImages++; - } - - imageBytes += texture.Data.GetLongLength(0); - } - } - - foreach (AssetInfo asset in Assets.Values) - { - if (asset != null) - { - if (asset.Temporary) - { - temporaryAssets++; - } - - assetBytes += asset.Data.GetLongLength(0); - } - } - - m_log.InfoFormat("Temporary Images: {0} Temporary Assets: {1}", - temporaryImages, - temporaryAssets); - - m_log.InfoFormat("Image data: {0}kb Asset data: {1}kb", - imageBytes / 1024, - assetBytes / 1024); } /// @@ -161,8 +123,6 @@ namespace OpenSim.Framework.Communications.Cache /// private void Initialize() { - Assets = new Dictionary(); - Textures = new Dictionary(); AssetRequests = new List(); RequestedAssets = new Dictionary(); @@ -181,7 +141,7 @@ namespace OpenSim.Framework.Communications.Cache m_assetServer = assetServer; m_assetServer.SetReceiver(this); - Thread assetCacheThread = new Thread(new ThreadStart(RunAssetManager)); + Thread assetCacheThread = new Thread(RunAssetManager); assetCacheThread.Name = "AssetCacheThread"; assetCacheThread.IsBackground = true; assetCacheThread.Start(); @@ -203,7 +163,7 @@ namespace OpenSim.Framework.Communications.Cache } catch (Exception e) { - m_log.Error("[ASSET CACHE]: " + e.ToString()); + m_log.Error("[ASSET CACHE]: " + e); } } } @@ -216,14 +176,11 @@ namespace OpenSim.Framework.Communications.Cache /// true if the asset was in the cache, false if it was not public bool TryGetCachedAsset(UUID assetId, out AssetBase asset) { - if (Textures.ContainsKey(assetId)) + Object tmp; + if(m_memcache.TryGet(assetId, out tmp)) { - asset = Textures[assetId]; - return true; - } - else if (Assets.ContainsKey(assetId)) - { - asset = Assets[assetId]; + asset = (AssetBase)tmp; + //m_log.Info("Retrieved from cache " + assetId); return true; } @@ -312,25 +269,22 @@ namespace OpenSim.Framework.Communications.Cache { return asset; } - else + m_assetServer.RequestAsset(assetID, isTexture); + + do { - m_assetServer.RequestAsset(assetID, isTexture); + Thread.Sleep(pollPeriod); - do + if (TryGetCachedAsset(assetID, out asset)) { - Thread.Sleep(pollPeriod); + return asset; + } + } while (--maxPolls > 0); - if (TryGetCachedAsset(assetID, out asset)) - { - return asset; - } - } while (--maxPolls > 0); + m_log.WarnFormat("[ASSET CACHE]: {0} {1} was not received before the retrieval timeout was reached", + isTexture ? "texture" : "asset", assetID.ToString()); - m_log.WarnFormat("[ASSET CACHE]: {0} {1} was not received before the retrieval timeout was reached", - isTexture ? "texture" : "asset", assetID.ToString()); - - return null; - } + return null; } /// @@ -339,68 +293,34 @@ namespace OpenSim.Framework.Communications.Cache /// public void AddAsset(AssetBase asset) { -// m_log.DebugFormat( -// "[ASSET CACHE]: Uploaded asset {0}, temporary {1}, store local {2}", -// asset.ID, asset.Temporary, asset.Local); - - if (asset.Type == (int)AssetType.Texture) + if (!m_memcache.Contains(asset.FullID)) { - if (!Textures.ContainsKey(asset.FullID)) + m_log.Info("[CACHE] Caching " + asset.FullID + " for 24 hours from last access"); + // Use 24 hour rolling asset cache. + m_memcache.AddOrUpdate(asset.FullID, asset, TimeSpan.FromHours(24)); + + // According to http://wiki.secondlife.com/wiki/AssetUploadRequest, Local signifies that the + // information is stored locally. It could disappear, in which case we could send the + // ImageNotInDatabase packet to tell the client this. + // + // However, this doesn't quite appear to work with local textures that are part of an avatar's + // appearance texture set. Whilst sending an ImageNotInDatabase does trigger an automatic rebake + // and reupload by the client, if those assets aren't pushed to the asset server anyway, then + // on crossing onto another region server, other avatars can no longer get the required textures. + // There doesn't appear to be any signal from the sim to the newly region border crossed client + // asking it to reupload its local texture assets to that region server. + // + // One can think of other cunning ways around this. For instance, on a region crossing or teleport, + // the original sim could squirt local assets to the new sim. Or the new sim could have pointers + // to the original sim to fetch the 'local' assets (this is getting more complicated). + // + // But for now, we're going to take the easy way out and store local assets globally. + // + // TODO: Also, Temporary is now deprecated. We should start ignoring it and not passing it out from LLClientView. + if (!asset.Temporary || asset.Local) { - TextureImage textur = new TextureImage(asset); - Textures.Add(textur.FullID, textur); - - if (StatsManager.SimExtraStats != null) - StatsManager.SimExtraStats.AddTexture(textur); - - // According to http://wiki.secondlife.com/wiki/AssetUploadRequest, Local signifies that the - // information is stored locally. It could disappear, in which case we could send the - // ImageNotInDatabase packet to tell the client this. - // - // However, this doesn't quite appear to work with local textures that are part of an avatar's - // appearance texture set. Whilst sending an ImageNotInDatabase does trigger an automatic rebake - // and reupload by the client, if those assets aren't pushed to the asset server anyway, then - // on crossing onto another region server, other avatars can no longer get the required textures. - // There doesn't appear to be any signal from the sim to the newly region border crossed client - // asking it to reupload its local texture assets to that region server. - // - // One can think of other cunning ways around this. For instance, on a region crossing or teleport, - // the original sim could squirt local assets to the new sim. Or the new sim could have pointers - // to the original sim to fetch the 'local' assets (this is getting more complicated). - // - // But for now, we're going to take the easy way out and store local assets globally. - // - // TODO: Also, Temporary is now deprecated. We should start ignoring it and not passing it out from LLClientView. - if (!asset.Temporary || asset.Local) - { - m_assetServer.StoreAsset(asset); - } + m_assetServer.StoreAsset(asset); } -// else -// { -// m_log.DebugFormat("[ASSET CACHE]: Textures already contains {0}", asset.ID); -// } - } - else - { - if (!Assets.ContainsKey(asset.FullID)) - { - AssetInfo assetInf = new AssetInfo(asset); - Assets.Add(assetInf.FullID, assetInf); - - if (StatsManager.SimExtraStats != null) - StatsManager.SimExtraStats.AddAsset(assetInf); - - // See comment above. - if (!asset.Temporary || asset.Local) - { - m_assetServer.StoreAsset(asset); - } - } -// else -// { -// m_log.DebugFormat("[ASSET CACHE]: Assets already contains {0}", asset.ID); -// } } } @@ -417,71 +337,46 @@ namespace OpenSim.Framework.Communications.Cache // in the 2 caches differently. Also, locks are probably // needed in all of this, or move to synchronized non // generic forms for Dictionaries. - if (Textures.ContainsKey(uuid)) + if(m_memcache.Contains(uuid)) { - Textures.Remove(uuid); - } - else if (Assets.ContainsKey(uuid)) - { - Assets.Remove(uuid); + m_memcache.Remove(uuid); } } // See IAssetReceiver public void AssetReceived(AssetBase asset, bool IsTexture) { -// m_log.DebugFormat("[ASSET CACHE]: Received asset {0}", asset.ID); - - //check if it is a texture or not - //then add to the correct cache list - //then check for waiting requests for this asset/texture (in the Requested lists) - //and move those requests into the Requests list. - if (IsTexture) - { - TextureImage image = new TextureImage(asset); - if (!Textures.ContainsKey(image.FullID)) - { - Textures.Add(image.FullID, image); - if (StatsManager.SimExtraStats != null) - { - StatsManager.SimExtraStats.AddTexture(image); - } + AssetInfo assetInf = new AssetInfo(asset); + if (!m_memcache.Contains(assetInf.FullID)) + { + m_memcache.AddOrUpdate(assetInf.FullID, assetInf, TimeSpan.FromHours(24)); + + if (StatsManager.SimExtraStats != null) + { + StatsManager.SimExtraStats.AddAsset(assetInf); } - } - else - { - AssetInfo assetInf = new AssetInfo(asset); - if (!Assets.ContainsKey(assetInf.FullID)) + + if (RequestedAssets.ContainsKey(assetInf.FullID)) { - Assets.Add(assetInf.FullID, assetInf); + AssetRequest req = RequestedAssets[assetInf.FullID]; + req.AssetInf = assetInf; + req.NumPackets = CalculateNumPackets(assetInf.Data); - if (StatsManager.SimExtraStats != null) - { - StatsManager.SimExtraStats.AddAsset(assetInf); - } - - if (RequestedAssets.ContainsKey(assetInf.FullID)) - { - AssetRequest req = RequestedAssets[assetInf.FullID]; - req.AssetInf = assetInf; - req.NumPackets = CalculateNumPackets(assetInf.Data); - - RequestedAssets.Remove(assetInf.FullID); - // If it's a direct request for a script, drop it - // because it's a hacked client - if (req.AssetRequestSource != 2 || assetInf.Type != 10) - AssetRequests.Add(req); - } + RequestedAssets.Remove(assetInf.FullID); + // If it's a direct request for a script, drop it + // because it's a hacked client + if (req.AssetRequestSource != 2 || assetInf.Type != 10) + AssetRequests.Add(req); } } // Notify requesters for this asset - AssetRequestsList reqList = null; - + AssetRequestsList reqList; + lock (RequestLists) - { - if (RequestLists.TryGetValue(asset.FullID, out reqList)) + { + if (RequestLists.TryGetValue(asset.FullID, out reqList)) RequestLists.Remove(asset.FullID); } @@ -489,7 +384,7 @@ namespace OpenSim.Framework.Communications.Cache { if (StatsManager.SimExtraStats != null) StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested); - + foreach (NewAssetRequest req in reqList.Requests) { // Xantor 20080526 are we really calling all the callbacks if multiple queued for 1 request? -- Yes, checked @@ -504,17 +399,8 @@ namespace OpenSim.Framework.Communications.Cache { // m_log.WarnFormat("[ASSET CACHE]: AssetNotFound for {0}", assetID); - if (IsTexture) - { - Textures[assetID] = null; - } - else - { - Assets[assetID] = null; - } - // Notify requesters for this asset - AssetRequestsList reqList = null; + AssetRequestsList reqList; lock (RequestLists) { if (RequestLists.TryGetValue(assetID, out reqList)) @@ -578,7 +464,7 @@ namespace OpenSim.Framework.Communications.Cache //check to see if asset is in local cache, if not we need to request it from asset server. //Console.WriteLine("asset request " + requestID); - if (!Assets.ContainsKey(requestID)) + if (!m_memcache.Contains(requestID)) { //not found asset // so request from asset server @@ -598,7 +484,7 @@ namespace OpenSim.Framework.Communications.Cache } // It has an entry in our cache - AssetInfo asset = Assets[requestID]; + AssetInfo asset = (AssetInfo)m_memcache[requestID]; // FIXME: We never tell the client about assets which do not exist when requested by this transfer mechanism, which can't be right. if (null == asset) diff --git a/OpenSim/Region/ClientStack/ClientStackManager.cs b/OpenSim/Region/ClientStack/ClientStackManager.cs index 5723739a82..098466da67 100644 --- a/OpenSim/Region/ClientStack/ClientStackManager.cs +++ b/OpenSim/Region/ClientStack/ClientStackManager.cs @@ -48,22 +48,32 @@ namespace OpenSim.Region.Environment { m_log.Info("[CLIENTSTACK]: Attempting to load " + dllName); - plugin = null; - pluginAssembly = Assembly.LoadFrom(dllName); - - foreach (Type pluginType in pluginAssembly.GetTypes()) + try { - if (pluginType.IsPublic) - { - Type typeInterface = pluginType.GetInterface("IClientNetworkServer", true); + plugin = null; + pluginAssembly = Assembly.LoadFrom(dllName); - if (typeInterface != null) + foreach (Type pluginType in pluginAssembly.GetTypes()) + { + if (pluginType.IsPublic) { - m_log.Info("[CLIENTSTACK]: Added IClientNetworkServer Interface"); - plugin = pluginType; - return; + Type typeInterface = pluginType.GetInterface("IClientNetworkServer", true); + + if (typeInterface != null) + { + m_log.Info("[CLIENTSTACK]: Added IClientNetworkServer Interface"); + plugin = pluginType; + return; + } } } + } catch (ReflectionTypeLoadException e) + { + foreach(Exception e2 in e.LoaderExceptions) + { + m_log.Error(e2.ToString()); + } + throw e; } } diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs index c953228bf8..6ad4c2dc06 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs @@ -664,7 +664,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_log.DebugFormat( "[CLIENT]: Entered main packet processing loop for {0} {1}", FirstName, LastName); - while (true) + while (IsActive) { LLQueItem nextPacket = m_PacketHandler.PacketQueue.Dequeue(); diff --git a/prebuild.xml b/prebuild.xml index c5f9a01088..2fbb337eee 100644 --- a/prebuild.xml +++ b/prebuild.xml @@ -743,6 +743,7 @@ + @@ -968,7 +969,6 @@ -