From f41fc4eb25ade48a358511564f3911a4605c1c31 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Wed, 5 Jun 2013 22:20:48 +0100 Subject: [PATCH 1/2] Avoid a deadlock where a script can attempt to take a ScriptInstance.m_Scripts lock then a lock on SP.m_attachments whilst SP.MakeRootAgent() attempts to take in the opposite order. This is because scripts (at least on XEngine) start unsuspended - deceptively the ResumeScripts() calls in various places in the code are actually completely redundant (and useless). The solution chosen here is to use a copy of the SP attachments and not have the list locked whilst creating the scripts when an avatar enters the region. This looks to address http://opensimulator.org/mantis/view.php?id=6557 --- .../Region/Framework/Scenes/ScenePresence.cs | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs index b8ff7f772f..bab14dd3de 100644 --- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs +++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs @@ -121,6 +121,8 @@ namespace OpenSim.Region.Framework.Scenes /// /// TODO: For some reason, we effectively have a list both here and in Appearance. Need to work out if this is /// necessary. + /// NOTE: To avoid deadlocks, do not lock m_attachments and then perform other tasks under that lock. Take a copy + /// of the list and act on that instead. /// private List m_attachments = new List(); @@ -971,19 +973,27 @@ namespace OpenSim.Region.Framework.Scenes // and CHANGED_REGION) when the attachments have been rezzed in the new region. This cannot currently // be done in AttachmentsModule.CopyAttachments(AgentData ad, IScenePresence sp) itself since we are // not transporting the required data. - lock (m_attachments) - { - if (HasAttachments()) - { - m_log.DebugFormat( - "[SCENE PRESENCE]: Restarting scripts in attachments for {0} in {1}", Name, Scene.Name); + // + // We must take a copy of the attachments list here (rather than locking) to avoid a deadlock where a script in one of + // the attachments may start processing an event (which locks ScriptInstance.m_Script) that then calls a method here + // which needs to lock m_attachments. ResumeScripts() needs to take a ScriptInstance.m_Script lock to try to unset the Suspend status. + // + // FIXME: In theory, this deadlock should not arise since scripts should not be processing events until ResumeScripts(). + // But XEngine starts all scripts unsuspended. Starting them suspended will not currently work because script rezzing + // is placed in an asynchronous queue in XEngine and so the ResumeScripts() call will almost certainly execute before the + // script is rezzed. This means the ResumeScripts() does absolutely nothing when using XEngine. + List attachments = GetAttachments(); - // Resume scripts - foreach (SceneObjectGroup sog in m_attachments) - { - sog.RootPart.ParentGroup.CreateScriptInstances(0, false, m_scene.DefaultScriptEngine, GetStateSource()); - sog.ResumeScripts(); - } + if (attachments.Count > 0) + { + m_log.DebugFormat( + "[SCENE PRESENCE]: Restarting scripts in attachments for {0} in {1}", Name, Scene.Name); + + // Resume scripts + foreach (SceneObjectGroup sog in attachments) + { + sog.RootPart.ParentGroup.CreateScriptInstances(0, false, m_scene.DefaultScriptEngine, GetStateSource()); + sog.ResumeScripts(); } } } From a7dbafb0e383ca5043a71284cdc35569acc5e2be Mon Sep 17 00:00:00 2001 From: Melanie Date: Wed, 5 Jun 2013 23:42:50 +0100 Subject: [PATCH 2/2] Port Avination's inventory send throttling --- OpenSim/Framework/Util.cs | 108 +++++++ .../Linden/Caps/WebFetchInvDescModule.cs | 269 ++++++++++++++---- prebuild.xml | 1 + 3 files changed, 324 insertions(+), 54 deletions(-) diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index ada4e89b00..7f0850fba0 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -2233,4 +2233,112 @@ namespace OpenSim.Framework return str.Replace("_", "\\_").Replace("%", "\\%"); } } + + public class DoubleQueue where T:class + { + private Queue m_lowQueue = new Queue(); + private Queue m_highQueue = new Queue(); + + private object m_syncRoot = new object(); + private Semaphore m_s = new Semaphore(0, 1); + + public DoubleQueue() + { + } + + public virtual int Count + { + get { return m_highQueue.Count + m_lowQueue.Count; } + } + + public virtual void Enqueue(T data) + { + Enqueue(m_lowQueue, data); + } + + public virtual void EnqueueLow(T data) + { + Enqueue(m_lowQueue, data); + } + + public virtual void EnqueueHigh(T data) + { + Enqueue(m_highQueue, data); + } + + private void Enqueue(Queue q, T data) + { + lock (m_syncRoot) + { + m_lowQueue.Enqueue(data); + m_s.WaitOne(0); + m_s.Release(); + } + } + + public virtual T Dequeue() + { + return Dequeue(Timeout.Infinite); + } + + public virtual T Dequeue(int tmo) + { + return Dequeue(TimeSpan.FromMilliseconds(tmo)); + } + + public virtual T Dequeue(TimeSpan wait) + { + T res = null; + + if (!Dequeue(wait, ref res)) + return null; + + return res; + } + + public bool Dequeue(int timeout, ref T res) + { + return Dequeue(TimeSpan.FromMilliseconds(timeout), ref res); + } + + public bool Dequeue(TimeSpan wait, ref T res) + { + if (!m_s.WaitOne(wait)) + return false; + + lock (m_syncRoot) + { + if (m_highQueue.Count > 0) + res = m_highQueue.Dequeue(); + else + res = m_lowQueue.Dequeue(); + + if (m_highQueue.Count == 0 && m_lowQueue.Count == 0) + return true; + + try + { + m_s.Release(); + } + catch + { + } + + return true; + } + } + + public virtual void Clear() + { + + lock (m_syncRoot) + { + // Make sure sem count is 0 + m_s.WaitOne(0); + + m_lowQueue.Clear(); + m_highQueue.Clear(); + } + } + } } diff --git a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs index 6890f4a7e6..7dd97701af 100644 --- a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs +++ b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs @@ -27,18 +27,25 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Reflection; +using System.Threading; using log4net; using Nini.Config; using Mono.Addins; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; +using OpenSim.Framework.Servers; using OpenSim.Framework.Servers.HttpServer; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; +using OpenSim.Framework.Capabilities; using OpenSim.Services.Interfaces; using Caps = OpenSim.Framework.Capabilities.Caps; using OpenSim.Capabilities.Handlers; +using OpenMetaverse; +using OpenMetaverse.StructuredData; namespace OpenSim.Region.ClientStack.Linden { @@ -48,67 +55,74 @@ namespace OpenSim.Region.ClientStack.Linden [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "WebFetchInvDescModule")] public class WebFetchInvDescModule : INonSharedRegionModule { -// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + class aPollRequest + { + public PollServiceInventoryEventArgs thepoll; + public UUID reqID; + public Hashtable request; + public ScenePresence presence; + public List folders; + } + + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private Scene m_scene; private IInventoryService m_InventoryService; private ILibraryService m_LibraryService; - private bool m_Enabled; + private static WebFetchInvDescHandler m_webFetchHandler; - private string m_fetchInventoryDescendents2Url; - private string m_webFetchInventoryDescendentsUrl; + private Dictionary m_capsDict = new Dictionary(); + private static Thread[] m_workerThreads = null; - private WebFetchInvDescHandler m_webFetchHandler; + private static DoubleQueue m_queue = + new DoubleQueue(); #region ISharedRegionModule Members public void Initialise(IConfigSource source) { - IConfig config = source.Configs["ClientStack.LindenCaps"]; - if (config == null) - return; - - m_fetchInventoryDescendents2Url = config.GetString("Cap_FetchInventoryDescendents2", string.Empty); - m_webFetchInventoryDescendentsUrl = config.GetString("Cap_WebFetchInventoryDescendents", string.Empty); - - if (m_fetchInventoryDescendents2Url != string.Empty || m_webFetchInventoryDescendentsUrl != string.Empty) - { - m_Enabled = true; - } } public void AddRegion(Scene s) { - if (!m_Enabled) - return; - m_scene = s; } public void RemoveRegion(Scene s) { - if (!m_Enabled) - return; - m_scene.EventManager.OnRegisterCaps -= RegisterCaps; + m_scene.EventManager.OnDeregisterCaps -= DeregisterCaps; m_scene = null; } public void RegionLoaded(Scene s) { - if (!m_Enabled) - return; - m_InventoryService = m_scene.InventoryService; m_LibraryService = m_scene.LibraryService; // We'll reuse the same handler for all requests. - if (m_fetchInventoryDescendents2Url == "localhost" || m_webFetchInventoryDescendentsUrl == "localhost") - m_webFetchHandler = new WebFetchInvDescHandler(m_InventoryService, m_LibraryService); + m_webFetchHandler = new WebFetchInvDescHandler(m_InventoryService, m_LibraryService); m_scene.EventManager.OnRegisterCaps += RegisterCaps; + m_scene.EventManager.OnDeregisterCaps += DeregisterCaps; + + if (m_workerThreads == null) + { + m_workerThreads = new Thread[2]; + + for (uint i = 0; i < 2; i++) + { + m_workerThreads[i] = Watchdog.StartThread(DoInventoryRequests, + String.Format("InventoryWorkerThread{0}", i), + ThreadPriority.Normal, + false, + true, + null, + int.MaxValue); + } + } } public void PostInitialise() @@ -126,43 +140,190 @@ namespace OpenSim.Region.ClientStack.Linden #endregion - private void RegisterCaps(UUID agentID, Caps caps) + ~WebFetchInvDescModule() { - if (m_webFetchInventoryDescendentsUrl != "") - RegisterFetchCap(agentID, caps, "WebFetchInventoryDescendents", m_webFetchInventoryDescendentsUrl); - - if (m_fetchInventoryDescendents2Url != "") - RegisterFetchCap(agentID, caps, "FetchInventoryDescendents2", m_fetchInventoryDescendents2Url); + foreach (Thread t in m_workerThreads) + Watchdog.AbortThread(t.ManagedThreadId); } - private void RegisterFetchCap(UUID agentID, Caps caps, string capName, string url) + private class PollServiceInventoryEventArgs : PollServiceEventArgs + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + private Dictionary responses = + new Dictionary(); + + private Scene m_scene; + + public PollServiceInventoryEventArgs(Scene scene, UUID pId) : + base(null, null, null, null, pId) + { + m_scene = scene; + + HasEvents = (x, y) => { lock (responses) return responses.ContainsKey(x); }; + GetEvents = (x, y, z) => + { + lock (responses) + { + try + { + return responses[x]; + } + finally + { + responses.Remove(x); + } + } + }; + + Request = (x, y) => + { + ScenePresence sp = m_scene.GetScenePresence(Id); + if (sp == null) + { + m_log.ErrorFormat("[INVENTORY]: Unable to find ScenePresence for {0}", Id); + return; + } + + aPollRequest reqinfo = new aPollRequest(); + reqinfo.thepoll = this; + reqinfo.reqID = x; + reqinfo.request = y; + reqinfo.presence = sp; + reqinfo.folders = new List(); + + // Decode the request here + string request = y["body"].ToString(); + + request = request.Replace("00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000000"); + + request = request.Replace("fetch_folders0", "fetch_folders0"); + request = request.Replace("fetch_folders1", "fetch_folders1"); + + Hashtable hash = new Hashtable(); + try + { + hash = (Hashtable)LLSD.LLSDDeserialize(Utils.StringToBytes(request)); + } + catch (LLSD.LLSDParseException e) + { + m_log.ErrorFormat("[INVENTORY]: Fetch error: {0}{1}" + e.Message, e.StackTrace); + m_log.Error("Request: " + request); + return; + } + catch (System.Xml.XmlException) + { + m_log.ErrorFormat("[INVENTORY]: XML Format error"); + } + + ArrayList foldersrequested = (ArrayList)hash["folders"]; + + bool highPriority = false; + + for (int i = 0; i < foldersrequested.Count; i++) + { + Hashtable inventoryhash = (Hashtable)foldersrequested[i]; + string folder = inventoryhash["folder_id"].ToString(); + UUID folderID; + if (UUID.TryParse(folder, out folderID)) + { + if (!reqinfo.folders.Contains(folderID)) + { + //TODO: Port COF handling from Avination + reqinfo.folders.Add(folderID); + } + } + } + + if (highPriority) + m_queue.EnqueueHigh(reqinfo); + else + m_queue.EnqueueLow(reqinfo); + }; + + NoEvents = (x, y) => + { +/* + lock (requests) + { + Hashtable request = requests.Find(id => id["RequestID"].ToString() == x.ToString()); + requests.Remove(request); + } +*/ + Hashtable response = new Hashtable(); + + response["int_response_code"] = 500; + response["str_response_string"] = "Script timeout"; + response["content_type"] = "text/plain"; + response["keepalive"] = false; + response["reusecontext"] = false; + + return response; + }; + } + + public void Process(aPollRequest requestinfo) + { + UUID requestID = requestinfo.reqID; + + Hashtable response = new Hashtable(); + + response["int_response_code"] = 200; + response["content_type"] = "text/plain"; + response["keepalive"] = false; + response["reusecontext"] = false; + + response["str_response_string"] = m_webFetchHandler.FetchInventoryDescendentsRequest( + requestinfo.request["body"].ToString(), String.Empty, String.Empty, null, null); + + lock (responses) + responses[requestID] = response; + } + } + + private void RegisterCaps(UUID agentID, Caps caps) + { + string capUrl = "/CAPS/" + UUID.Random() + "/"; + + // Register this as a poll service + PollServiceInventoryEventArgs args = new PollServiceInventoryEventArgs(m_scene, agentID); + + MainServer.Instance.AddPollServiceHTTPHandler(capUrl, args); + + string hostName = m_scene.RegionInfo.ExternalHostName; + uint port = (MainServer.Instance == null) ? 0 : MainServer.Instance.Port; + string protocol = "http"; + + if (MainServer.Instance.UseSSL) + { + hostName = MainServer.Instance.SSLCommonName; + port = MainServer.Instance.SSLPort; + protocol = "https"; + } + caps.RegisterHandler("FetchInventoryDescendents2", String.Format("{0}://{1}:{2}{3}", protocol, hostName, port, capUrl)); + + m_capsDict[agentID] = capUrl; + } + + private void DeregisterCaps(UUID agentID, Caps caps) { string capUrl; - if (url == "localhost") + if (m_capsDict.TryGetValue(agentID, out capUrl)) { - capUrl = "/CAPS/" + UUID.Random(); - - IRequestHandler reqHandler - = new RestStreamHandler( - "POST", - capUrl, - m_webFetchHandler.FetchInventoryDescendentsRequest, - "FetchInventoryDescendents2", - agentID.ToString()); - - caps.RegisterHandler(capName, reqHandler); + MainServer.Instance.RemoveHTTPHandler("", capUrl); + m_capsDict.Remove(agentID); } - else + } + + private void DoInventoryRequests() + { + while (true) { - capUrl = url; + aPollRequest poolreq = m_queue.Dequeue(); - caps.RegisterHandler(capName, capUrl); + poolreq.thepoll.Process(poolreq); } - -// m_log.DebugFormat( -// "[WEB FETCH INV DESC MODULE]: Registered capability {0} at {1} in region {2} for {3}", -// capName, capUrl, m_scene.RegionInfo.RegionName, agentID); } } -} \ No newline at end of file +} diff --git a/prebuild.xml b/prebuild.xml index 03cac764ce..5531558960 100644 --- a/prebuild.xml +++ b/prebuild.xml @@ -1531,6 +1531,7 @@ +