From d4720bd721b518efbc681df2f4e7d1ca35aa0f3c Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 16 Jul 2013 17:53:05 -0700 Subject: [PATCH 01/12] Added config var to fiddle with the Interval for the service throttle thread --- .../Framework/ServiceThrottle/ServiceThrottleModule.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs index 553e4caa29..a3ca6d69d0 100644 --- a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs +++ b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs @@ -54,11 +54,14 @@ namespace OpenSim.Region.CoreModules.Framework // private OpenSim.Framework.DoubleQueue m_RequestQueue = new OpenSim.Framework.DoubleQueue(); //private Queue m_RequestQueue = new Queue(); private Queue m_RequestQueue = new Queue(); + private int m_Interval; #region ISharedRegionModule public void Initialise(IConfigSource config) { + m_Interval = Util.GetConfigVarFromSections(config, "Interval", new string[] { "ServiceThrottle" }, 2000); + m_timer = new System.Timers.Timer(); m_timer.AutoReset = false; m_timer.Enabled = true; @@ -131,7 +134,7 @@ namespace OpenSim.Region.CoreModules.Framework { if (!m_timer.Enabled) { - m_timer.Interval = 1000; + m_timer.Interval = m_Interval; m_timer.Enabled = true; m_timer.Start(); } From 5f27aaa6ddb857dbfbbe0219d0e29c42ad903375 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 16 Jul 2013 18:22:42 -0700 Subject: [PATCH 02/12] UserManagementModule: in the continuation, call the method that also looks up the cache, because the resource may be here in the meantime --- .../Framework/UserManagement/UserManagementModule.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs b/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs index da28a530f5..e8bdcc9692 100644 --- a/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs +++ b/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs @@ -172,11 +172,11 @@ namespace OpenSim.Region.CoreModules.Framework.UserManagement return; } - // Not found in cache, get it from services + // Not found in cache, queue continuation m_ServiceThrottle.Enqueue(delegate { //m_log.DebugFormat("[YYY]: Name request {0}", uuid); - bool foundRealName = TryGetUserNamesFromServices(uuid, names); + bool foundRealName = TryGetUserNames(uuid, names); if (names.Length == 2) { From 9432f3c94d0b0345132e5ff9eaf966b96cf218c2 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 16 Jul 2013 19:04:30 -0700 Subject: [PATCH 03/12] Improvements to the ServiceThrottleModule: added a category and an itemid to the interface, so that duplicate requests aren't enqueued more than once. --- .../ServiceThrottle/ServiceThrottleModule.cs | 35 ++++++++++++++----- .../UserManagement/UserManagementModule.cs | 2 +- .../Interfaces/IServiceThrottleModule.cs | 10 +++++- bin/OpenSimDefaults.ini | 4 +++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs index a3ca6d69d0..1554b3ed8c 100644 --- a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs +++ b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs @@ -50,17 +50,15 @@ namespace OpenSim.Region.CoreModules.Framework private readonly List m_scenes = new List(); private System.Timers.Timer m_timer = new System.Timers.Timer(); - //private OpenSim.Framework.BlockingQueue m_RequestQueue = new OpenSim.Framework.BlockingQueue(); - // private OpenSim.Framework.DoubleQueue m_RequestQueue = new OpenSim.Framework.DoubleQueue(); - //private Queue m_RequestQueue = new Queue(); private Queue m_RequestQueue = new Queue(); + private Dictionary> m_Pending = new Dictionary>(); private int m_Interval; #region ISharedRegionModule public void Initialise(IConfigSource config) { - m_Interval = Util.GetConfigVarFromSections(config, "Interval", new string[] { "ServiceThrottle" }, 2000); + m_Interval = Util.GetConfigVarFromSections(config, "Interval", new string[] { "ServiceThrottle" }, 5000); m_timer = new System.Timers.Timer(); m_timer.AutoReset = false; @@ -159,18 +157,37 @@ namespace OpenSim.Region.CoreModules.Framework client.SendRegionHandle(regionID, r.RegionHandle); }; - lock (m_RequestQueue) - m_RequestQueue.Enqueue(action); - + Enqueue("region", regionID.ToString(), action); } #endregion Events #region IServiceThrottleModule - public void Enqueue(Action continuation) + public void Enqueue(string category, string itemid, Action continuation) { - m_RequestQueue.Enqueue(continuation); + lock (m_RequestQueue) + { + if (m_Pending.ContainsKey(category)) + { + if (m_Pending[category].Contains(itemid)) + // Don't enqueue, it's already pending + return; + } + else + m_Pending.Add(category, new List()); + + m_Pending[category].Add(itemid); + + m_RequestQueue.Enqueue(delegate + { + continuation(); + lock (m_RequestQueue) + { + m_Pending[category].Remove(itemid); + } + }); + } } #endregion IServiceThrottleModule diff --git a/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs b/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs index e8bdcc9692..a91adfab13 100644 --- a/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs +++ b/OpenSim/Region/CoreModules/Framework/UserManagement/UserManagementModule.cs @@ -173,7 +173,7 @@ namespace OpenSim.Region.CoreModules.Framework.UserManagement } // Not found in cache, queue continuation - m_ServiceThrottle.Enqueue(delegate + m_ServiceThrottle.Enqueue("name", uuid.ToString(), delegate { //m_log.DebugFormat("[YYY]: Name request {0}", uuid); bool foundRealName = TryGetUserNames(uuid, names); diff --git a/OpenSim/Region/Framework/Interfaces/IServiceThrottleModule.cs b/OpenSim/Region/Framework/Interfaces/IServiceThrottleModule.cs index bb6a8b4ffa..198256fdde 100644 --- a/OpenSim/Region/Framework/Interfaces/IServiceThrottleModule.cs +++ b/OpenSim/Region/Framework/Interfaces/IServiceThrottleModule.cs @@ -5,7 +5,15 @@ namespace OpenSim.Region.Framework.Interfaces { public interface IServiceThrottleModule { - void Enqueue(Action continuation); + /// + /// Enqueue a continuation meant to get a resource from elsewhere. + /// As usual with CPS, caller beware: if that continuation is a never-ending computation, + /// the whole thread will be blocked, and no requests are processed + /// + /// Category of the resource (e.g. name, region) + /// The resource identifier + /// The continuation to be executed + void Enqueue(string category, string itemid, Action continuation); } } diff --git a/bin/OpenSimDefaults.ini b/bin/OpenSimDefaults.ini index 4a3104ee81..8079632368 100644 --- a/bin/OpenSimDefaults.ini +++ b/bin/OpenSimDefaults.ini @@ -1727,5 +1727,9 @@ MaxStringSpace = 0 ;; {MaxDistance} {} {Cut-off distance at which sounds will not be sent to users} {100.0} MaxDistance = 100.0 +[ServiceThrottle] + ;; Default time interval (in ms) for the throttle service thread to wake up + Interval = 5000 + [Modules] Include-modules = "addon-modules/*/config/*.ini" From 894554faf61fe8ae02c1348612845ae2553efbd4 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 16 Jul 2013 20:28:48 -0700 Subject: [PATCH 04/12] Removed the MapItems thread. Redirected the map items requests to the services throttle thread. Didn't change anything in how that processor is implemented, for better or for worse. --- .../ServiceThrottle/ServiceThrottleModule.cs | 5 +- .../World/WorldMap/WorldMapModule.cs | 49 ++++++++++++++++--- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs index 1554b3ed8c..a70261e938 100644 --- a/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs +++ b/OpenSim/Region/CoreModules/Framework/ServiceThrottle/ServiceThrottleModule.cs @@ -181,11 +181,10 @@ namespace OpenSim.Region.CoreModules.Framework m_RequestQueue.Enqueue(delegate { - continuation(); lock (m_RequestQueue) - { m_Pending[category].Remove(itemid); - } + + continuation(); }); } } diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs index c50ab64ad6..a26a5f03e9 100644 --- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs +++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs @@ -81,6 +81,8 @@ namespace OpenSim.Region.CoreModules.World.WorldMap private List m_rootAgents = new List(); private volatile bool threadrunning = false; + private IServiceThrottleModule m_ServiceThrottle; + //private int CacheRegionsDistance = 256; #region INonSharedRegionModule Members @@ -131,6 +133,10 @@ namespace OpenSim.Region.CoreModules.World.WorldMap public virtual void RegionLoaded (Scene scene) { + if (!m_Enabled) + return; + + m_ServiceThrottle = scene.RequestModuleInterface(); } @@ -170,13 +176,13 @@ namespace OpenSim.Region.CoreModules.World.WorldMap m_scene.EventManager.OnMakeRootAgent += MakeRootAgent; m_scene.EventManager.OnRegionUp += OnRegionUp; - StartThread(new object()); +// StartThread(new object()); } // this has to be called with a lock on m_scene protected virtual void RemoveHandlers() { - StopThread(); +// StopThread(); m_scene.EventManager.OnRegionUp -= OnRegionUp; m_scene.EventManager.OnMakeRootAgent -= MakeRootAgent; @@ -526,7 +532,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap /// public void process() { - const int MAX_ASYNC_REQUESTS = 20; + //const int MAX_ASYNC_REQUESTS = 20; try { while (true) @@ -571,13 +577,44 @@ namespace OpenSim.Region.CoreModules.World.WorldMap Watchdog.RemoveThread(); } + const int MAX_ASYNC_REQUESTS = 20; + /// - /// Enqueues the map item request into the processing thread + /// Enqueues the map item request into the services throttle processing thread /// /// - public void EnqueueMapItemRequest(MapRequestState state) + public void EnqueueMapItemRequest(MapRequestState st) { - requests.Enqueue(state); + + m_ServiceThrottle.Enqueue("map-item", st.regionhandle.ToString() + st.agentID.ToString(), delegate + { + if (st.agentID != UUID.Zero) + { + bool dorequest = true; + lock (m_rootAgents) + { + if (!m_rootAgents.Contains(st.agentID)) + dorequest = false; + } + + if (dorequest && !m_blacklistedregions.ContainsKey(st.regionhandle)) + { + if (nAsyncRequests >= MAX_ASYNC_REQUESTS) // hit the break + { + // AH!!! Recursive ! + // Put this request back in the queue and return + EnqueueMapItemRequest(st); + return; + } + + RequestMapItemsDelegate d = RequestMapItemsAsync; + d.BeginInvoke(st.agentID, st.flags, st.EstateID, st.godlike, st.itemtype, st.regionhandle, RequestMapItemsCompleted, null); + //OSDMap response = RequestMapItemsAsync(st.agentID, st.flags, st.EstateID, st.godlike, st.itemtype, st.regionhandle); + //RequestMapItemsCompleted(response); + Interlocked.Increment(ref nAsyncRequests); + } + } + }); } /// From 2c8bf4aaa66a4cc9fc8fd9f89825e7cd1b43f3d6 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Wed, 17 Jul 2013 10:19:44 -0700 Subject: [PATCH 05/12] BulletSim: fix small bug where everything looked like it was colliding before the first simulator step. --- .../Region/Physics/BulletSPlugin/BSPhysObject.cs | 13 +++++++------ OpenSim/Region/Physics/BulletSPlugin/BSScene.cs | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/OpenSim/Region/Physics/BulletSPlugin/BSPhysObject.cs b/OpenSim/Region/Physics/BulletSPlugin/BSPhysObject.cs index a41eaf8617..fc4545f4e3 100755 --- a/OpenSim/Region/Physics/BulletSPlugin/BSPhysObject.cs +++ b/OpenSim/Region/Physics/BulletSPlugin/BSPhysObject.cs @@ -103,9 +103,10 @@ public abstract class BSPhysObject : PhysicsActor CollisionsLastTickStep = -1; SubscribedEventsMs = 0; - CollidingStep = 0; - CollidingGroundStep = 0; - CollisionAccumulation = 0; + // Crazy values that will never be true + CollidingStep = BSScene.NotASimulationStep; + CollidingGroundStep = BSScene.NotASimulationStep; + CollisionAccumulation = BSScene.NotASimulationStep; ColliderIsMoving = false; CollisionScore = 0; @@ -349,7 +350,7 @@ public abstract class BSPhysObject : PhysicsActor if (value) CollidingStep = PhysScene.SimulationStep; else - CollidingStep = 0; + CollidingStep = BSScene.NotASimulationStep; } } public override bool CollidingGround { @@ -359,7 +360,7 @@ public abstract class BSPhysObject : PhysicsActor if (value) CollidingGroundStep = PhysScene.SimulationStep; else - CollidingGroundStep = 0; + CollidingGroundStep = BSScene.NotASimulationStep; } } public override bool CollidingObj { @@ -368,7 +369,7 @@ public abstract class BSPhysObject : PhysicsActor if (value) CollidingObjectStep = PhysScene.SimulationStep; else - CollidingObjectStep = 0; + CollidingObjectStep = BSScene.NotASimulationStep; } } diff --git a/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs b/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs index 214271b247..41aca3b2a7 100644 --- a/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs +++ b/OpenSim/Region/Physics/BulletSPlugin/BSScene.cs @@ -97,6 +97,9 @@ public sealed class BSScene : PhysicsScene, IPhysicsParameters internal long m_simulationStep = 0; // The current simulation step. public long SimulationStep { get { return m_simulationStep; } } + // A number to use for SimulationStep that is probably not any step value + // Used by the collision code (which remembers the step when a collision happens) to remember not any simulation step. + public static long NotASimulationStep = -1234; internal float LastTimeStep { get; private set; } // The simulation time from the last invocation of Simulate() From e46459ef21e1ee5ceaeca70365a7c881d33b09ce Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 11:19:36 -0700 Subject: [PATCH 06/12] Cleared up much confusion in PollServiceRequestManager. Here's the history: When Melanie added the web fetch inventory throttle to core, she made the long poll requests (EQs) effectively be handled on an active loop. All those requests, if they existed, were being constantly dequeued, checked for events (which most often they didn't have), and requeued again. This was an active loop thread on a 100ms cycle! This fixes the issue. Now the inventory requests, if they aren't ready to be served, are placed directly back in the queue, but the long poll requests aren't placed there until there are events ready to be sent or timeout has been reached. This puts the LongPollServiceWatcherThread back to 1sec cycle, as it was before. --- OpenSim/Framework/BlockingQueue.cs | 2 +- .../HttpServer/PollServiceEventArgs.cs | 4 +- .../HttpServer/PollServiceRequestManager.cs | 79 ++++++++----------- .../Caps/EventQueue/EventQueueGetModule.cs | 8 +- 4 files changed, 41 insertions(+), 52 deletions(-) diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs index aef1192364..e607e648e9 100644 --- a/OpenSim/Framework/BlockingQueue.cs +++ b/OpenSim/Framework/BlockingQueue.cs @@ -76,7 +76,7 @@ namespace OpenSim.Framework { lock (m_queueSync) { - if (m_queue.Count < 1 && m_pqueue.Count < 1) + while (m_queue.Count < 1 && m_pqueue.Count < 1) { Monitor.Wait(m_queueSync, msTimeout); } diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs index 020bfd5baf..947710094d 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs @@ -50,7 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer public enum EventType : int { - Normal = 0, + LongPoll = 0, LslHttp = 1, Inventory = 2 } @@ -80,7 +80,7 @@ namespace OpenSim.Framework.Servers.HttpServer NoEvents = pNoEvents; Id = pId; TimeOutms = pTimeOutms; - Type = EventType.Normal; + Type = EventType.LongPoll; } } } diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 1b9010acf0..4cb551cd19 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -47,12 +47,11 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; private BlockingQueue m_requests = new BlockingQueue(); - private static Queue m_slowRequests = new Queue(); - private static Queue m_retryRequests = new Queue(); + private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; private Thread[] m_workerThreads; - private Thread m_retrysThread; + private Thread m_longPollThread; private bool m_running = true; private int slowCount = 0; @@ -84,9 +83,9 @@ namespace OpenSim.Framework.Servers.HttpServer int.MaxValue); } - m_retrysThread = Watchdog.StartThread( - this.CheckRetries, - string.Format("PollServiceWatcherThread:{0}", m_server.Port), + m_longPollThread = Watchdog.StartThread( + this.CheckLongPollThreads, + string.Format("LongPollServiceWatcherThread:{0}", m_server.Port), ThreadPriority.Normal, false, true, @@ -97,48 +96,47 @@ namespace OpenSim.Framework.Servers.HttpServer private void ReQueueEvent(PollServiceHttpRequest req) { if (m_running) - { - lock (m_retryRequests) - m_retryRequests.Enqueue(req); - } + m_requests.Enqueue(req); } public void Enqueue(PollServiceHttpRequest req) { if (m_running) { - if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal) + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) { - m_requests.Enqueue(req); + lock (m_longPollRequests) + m_longPollRequests.Enqueue(req); } else - { - lock (m_slowRequests) - m_slowRequests.Enqueue(req); - } + m_requests.Enqueue(req); } } - private void CheckRetries() + private void CheckLongPollThreads() { + // The only purpose of this thread is to check the EQs for events. + // If there are events, that thread will be placed in the "ready-to-serve" queue, m_requests. + // If there are no events, that thread will be back to its "waiting" queue, m_longPollRequests. + // All other types of tasks (Inventory handlers) don't have the long-poll nature, + // so if they aren't ready to be served by a worker thread (no events), they are placed + // directly back in the "ready-to-serve" queue by the worker thread. while (m_running) { - Thread.Sleep(100); // let the world move .. back to faster rate + Thread.Sleep(1000); Watchdog.UpdateThread(); - lock (m_retryRequests) - { - while (m_retryRequests.Count > 0 && m_running) - m_requests.Enqueue(m_retryRequests.Dequeue()); - } - slowCount++; - if (slowCount >= 10) - { - slowCount = 0; - lock (m_slowRequests) + PollServiceHttpRequest req; + lock (m_longPollRequests) + { + while (m_longPollRequests.Count > 0 && m_running) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + req = m_longPollRequests.Dequeue(); + if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ + (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout + m_requests.Enqueue(req); + else + m_longPollRequests.Enqueue(req); } } } @@ -153,24 +151,12 @@ namespace OpenSim.Framework.Servers.HttpServer foreach (Thread t in m_workerThreads) Watchdog.AbortThread(t.ManagedThreadId); - try - { - foreach (PollServiceHttpRequest req in m_retryRequests) - { - req.DoHTTPGruntWork(m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); - } - } - catch - { - } - PollServiceHttpRequest wreq; - m_retryRequests.Clear(); - lock (m_slowRequests) + lock (m_longPollRequests) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + while (m_longPollRequests.Count > 0 && m_running) + m_requests.Enqueue(m_longPollRequests.Dequeue()); } while (m_requests.Count() > 0) @@ -196,6 +182,7 @@ namespace OpenSim.Framework.Servers.HttpServer while (m_running) { PollServiceHttpRequest req = m_requests.Dequeue(5000); + //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); Watchdog.UpdateThread(); if (req != null) @@ -209,7 +196,7 @@ namespace OpenSim.Framework.Servers.HttpServer if (responsedata == null) continue; - if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Normal) // This is the event queue + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue { try { diff --git a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs index 1835a72a4a..f0445ff3c3 100644 --- a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs +++ b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs @@ -364,8 +364,7 @@ namespace OpenSim.Region.ClientStack.Linden caps.RegisterPollHandler( "EventQueueGet", - new PollServiceEventArgs( - null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000)); + new PollServiceEventArgs(null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000)); Random rnd = new Random(Environment.TickCount); lock (m_ids) @@ -383,7 +382,10 @@ namespace OpenSim.Region.ClientStack.Linden Queue queue = GetQueue(agentID); if (queue != null) lock (queue) + { + //m_log.WarnFormat("POLLED FOR EVENTS BY {0} in {1} -- {2}", agentID, m_scene.RegionInfo.RegionName, queue.Count); return queue.Count > 0; + } return false; } @@ -406,7 +408,7 @@ namespace OpenSim.Region.ClientStack.Linden public Hashtable GetEvents(UUID requestID, UUID pAgentId) { if (DebugLevel >= 2) - m_log.DebugFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName); + m_log.WarnFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName); Queue queue = TryGetQueue(pAgentId); OSD element; From 0f5b616fb0ebf9207b3cc81771622ed1290ea7d6 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 12:02:00 -0700 Subject: [PATCH 07/12] Didn't mean to commit this change in BlockingQueue.cs --- OpenSim/Framework/BlockingQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs index e607e648e9..aef1192364 100644 --- a/OpenSim/Framework/BlockingQueue.cs +++ b/OpenSim/Framework/BlockingQueue.cs @@ -76,7 +76,7 @@ namespace OpenSim.Framework { lock (m_queueSync) { - while (m_queue.Count < 1 && m_pqueue.Count < 1) + if (m_queue.Count < 1 && m_pqueue.Count < 1) { Monitor.Wait(m_queueSync, msTimeout); } From f4317dc26d670c853d0ea64b401b00f718f09474 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 12:57:34 -0700 Subject: [PATCH 08/12] Putting the requests back in the queue while testing for count >0 is not the smartest move... --- .../Servers/HttpServer/PollServiceRequestManager.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 4cb551cd19..c50df5abf4 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -126,18 +126,22 @@ namespace OpenSim.Framework.Servers.HttpServer Thread.Sleep(1000); Watchdog.UpdateThread(); - PollServiceHttpRequest req; + List not_ready = new List(); lock (m_longPollRequests) { while (m_longPollRequests.Count > 0 && m_running) { - req = m_longPollRequests.Dequeue(); + PollServiceHttpRequest req = m_longPollRequests.Dequeue(); if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout m_requests.Enqueue(req); else - m_longPollRequests.Enqueue(req); + not_ready.Add(req); } + + foreach (PollServiceHttpRequest req in not_ready) + m_longPollRequests.Enqueue(req); + } } } From af792bc7f2504e9ccf1c8ae7568919785dc397c9 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 13:23:29 -0700 Subject: [PATCH 09/12] Do the same trick that dahlia did for Dequeue(timeout) --- OpenSim/Framework/BlockingQueue.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs index aef1192364..fb74a24784 100644 --- a/OpenSim/Framework/BlockingQueue.cs +++ b/OpenSim/Framework/BlockingQueue.cs @@ -76,9 +76,10 @@ namespace OpenSim.Framework { lock (m_queueSync) { - if (m_queue.Count < 1 && m_pqueue.Count < 1) + bool timedout = false; + while (m_queue.Count < 1 && m_pqueue.Count < 1 && !timedout) { - Monitor.Wait(m_queueSync, msTimeout); + timedout = Monitor.Wait(m_queueSync, msTimeout); } if (m_pqueue.Count > 0) From 1d3deda10cf85abd68a5f904d6698ae597a67cc0 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 13:26:15 -0700 Subject: [PATCH 10/12] I confuse myself. Let's try this variable name instead. --- OpenSim/Framework/BlockingQueue.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs index fb74a24784..3e90fac569 100644 --- a/OpenSim/Framework/BlockingQueue.cs +++ b/OpenSim/Framework/BlockingQueue.cs @@ -76,10 +76,10 @@ namespace OpenSim.Framework { lock (m_queueSync) { - bool timedout = false; - while (m_queue.Count < 1 && m_pqueue.Count < 1 && !timedout) + bool success = true; + while (m_queue.Count < 1 && m_pqueue.Count < 1 && success) { - timedout = Monitor.Wait(m_queueSync, msTimeout); + success = Monitor.Wait(m_queueSync, msTimeout); } if (m_pqueue.Count > 0) From 5f95f4d78e8c7d17b8ba866907156fe6d4444c04 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 14:09:04 -0700 Subject: [PATCH 11/12] Now trying DoubleQueue instead of BlockingQueue for the PollServiceRequestManager. --- .../Framework/Servers/HttpServer/PollServiceRequestManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index c50df5abf4..bad28e47d2 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -46,7 +46,7 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; - private BlockingQueue m_requests = new BlockingQueue(); + private DoubleQueue m_requests = new DoubleQueue(); private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; @@ -163,7 +163,7 @@ namespace OpenSim.Framework.Servers.HttpServer m_requests.Enqueue(m_longPollRequests.Dequeue()); } - while (m_requests.Count() > 0) + while (m_requests.Count > 0) { try { From 5232ab0496eb4fe6903a0fd328974ac69df29ad8 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 14:36:55 -0700 Subject: [PATCH 12/12] This is a completely unreasonable thing to do, effectively defying the purpose of BlockingQueues. Trying this, to see the effect on CPU. --- .../HttpServer/PollServiceRequestManager.cs | 94 +++++++++++-------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index bad28e47d2..b8f57436cd 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -46,7 +46,7 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; - private DoubleQueue m_requests = new DoubleQueue(); + private BlockingQueue m_requests = new BlockingQueue(); private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; @@ -163,7 +163,7 @@ namespace OpenSim.Framework.Servers.HttpServer m_requests.Enqueue(m_longPollRequests.Dequeue()); } - while (m_requests.Count > 0) + while (m_requests.Count() > 0) { try { @@ -185,35 +185,33 @@ namespace OpenSim.Framework.Servers.HttpServer { while (m_running) { - PollServiceHttpRequest req = m_requests.Dequeue(5000); - //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); - Watchdog.UpdateThread(); - if (req != null) + + PollServiceHttpRequest req = null; + lock (m_requests) { - try + if (m_requests.Count() > 0) + req = m_requests.Dequeue(); + } + if (req == null) + Thread.Sleep(100); + else + { + //PollServiceHttpRequest req = m_requests.Dequeue(5000); + //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); + + if (req != null) { - if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id)) + try { - Hashtable responsedata = req.PollServiceArgs.GetEvents(req.RequestID, req.PollServiceArgs.Id); - - if (responsedata == null) - continue; - - if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue + if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id)) { - try - { - req.DoHTTPGruntWork(m_server, responsedata); - } - catch (ObjectDisposedException) // Browser aborted before we could read body, server closed the stream - { - // Ignore it, no need to reply - } - } - else - { - m_threadPool.QueueWorkItem(x => + Hashtable responsedata = req.PollServiceArgs.GetEvents(req.RequestID, req.PollServiceArgs.Id); + + if (responsedata == null) + continue; + + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue { try { @@ -223,27 +221,41 @@ namespace OpenSim.Framework.Servers.HttpServer { // Ignore it, no need to reply } + } + else + { + m_threadPool.QueueWorkItem(x => + { + try + { + req.DoHTTPGruntWork(m_server, responsedata); + } + catch (ObjectDisposedException) // Browser aborted before we could read body, server closed the stream + { + // Ignore it, no need to reply + } - return null; - }, null); - } - } - else - { - if ((Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) - { - req.DoHTTPGruntWork( - m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); + return null; + }, null); + } } else { - ReQueueEvent(req); + if ((Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) + { + req.DoHTTPGruntWork( + m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); + } + else + { + ReQueueEvent(req); + } } } - } - catch (Exception e) - { - m_log.ErrorFormat("Exception in poll service thread: " + e.ToString()); + catch (Exception e) + { + m_log.ErrorFormat("Exception in poll service thread: " + e.ToString()); + } } } }