From e46459ef21e1ee5ceaeca70365a7c881d33b09ce Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Wed, 17 Jul 2013 11:19:36 -0700 Subject: [PATCH] Cleared up much confusion in PollServiceRequestManager. Here's the history: When Melanie added the web fetch inventory throttle to core, she made the long poll requests (EQs) effectively be handled on an active loop. All those requests, if they existed, were being constantly dequeued, checked for events (which most often they didn't have), and requeued again. This was an active loop thread on a 100ms cycle! This fixes the issue. Now the inventory requests, if they aren't ready to be served, are placed directly back in the queue, but the long poll requests aren't placed there until there are events ready to be sent or timeout has been reached. This puts the LongPollServiceWatcherThread back to 1sec cycle, as it was before. --- OpenSim/Framework/BlockingQueue.cs | 2 +- .../HttpServer/PollServiceEventArgs.cs | 4 +- .../HttpServer/PollServiceRequestManager.cs | 79 ++++++++----------- .../Caps/EventQueue/EventQueueGetModule.cs | 8 +- 4 files changed, 41 insertions(+), 52 deletions(-) diff --git a/OpenSim/Framework/BlockingQueue.cs b/OpenSim/Framework/BlockingQueue.cs index aef1192364..e607e648e9 100644 --- a/OpenSim/Framework/BlockingQueue.cs +++ b/OpenSim/Framework/BlockingQueue.cs @@ -76,7 +76,7 @@ namespace OpenSim.Framework { lock (m_queueSync) { - if (m_queue.Count < 1 && m_pqueue.Count < 1) + while (m_queue.Count < 1 && m_pqueue.Count < 1) { Monitor.Wait(m_queueSync, msTimeout); } diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs index 020bfd5baf..947710094d 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs @@ -50,7 +50,7 @@ namespace OpenSim.Framework.Servers.HttpServer public enum EventType : int { - Normal = 0, + LongPoll = 0, LslHttp = 1, Inventory = 2 } @@ -80,7 +80,7 @@ namespace OpenSim.Framework.Servers.HttpServer NoEvents = pNoEvents; Id = pId; TimeOutms = pTimeOutms; - Type = EventType.Normal; + Type = EventType.LongPoll; } } } diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 1b9010acf0..4cb551cd19 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -47,12 +47,11 @@ namespace OpenSim.Framework.Servers.HttpServer private readonly BaseHttpServer m_server; private BlockingQueue m_requests = new BlockingQueue(); - private static Queue m_slowRequests = new Queue(); - private static Queue m_retryRequests = new Queue(); + private static Queue m_longPollRequests = new Queue(); private uint m_WorkerThreadCount = 0; private Thread[] m_workerThreads; - private Thread m_retrysThread; + private Thread m_longPollThread; private bool m_running = true; private int slowCount = 0; @@ -84,9 +83,9 @@ namespace OpenSim.Framework.Servers.HttpServer int.MaxValue); } - m_retrysThread = Watchdog.StartThread( - this.CheckRetries, - string.Format("PollServiceWatcherThread:{0}", m_server.Port), + m_longPollThread = Watchdog.StartThread( + this.CheckLongPollThreads, + string.Format("LongPollServiceWatcherThread:{0}", m_server.Port), ThreadPriority.Normal, false, true, @@ -97,48 +96,47 @@ namespace OpenSim.Framework.Servers.HttpServer private void ReQueueEvent(PollServiceHttpRequest req) { if (m_running) - { - lock (m_retryRequests) - m_retryRequests.Enqueue(req); - } + m_requests.Enqueue(req); } public void Enqueue(PollServiceHttpRequest req) { if (m_running) { - if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal) + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) { - m_requests.Enqueue(req); + lock (m_longPollRequests) + m_longPollRequests.Enqueue(req); } else - { - lock (m_slowRequests) - m_slowRequests.Enqueue(req); - } + m_requests.Enqueue(req); } } - private void CheckRetries() + private void CheckLongPollThreads() { + // The only purpose of this thread is to check the EQs for events. + // If there are events, that thread will be placed in the "ready-to-serve" queue, m_requests. + // If there are no events, that thread will be back to its "waiting" queue, m_longPollRequests. + // All other types of tasks (Inventory handlers) don't have the long-poll nature, + // so if they aren't ready to be served by a worker thread (no events), they are placed + // directly back in the "ready-to-serve" queue by the worker thread. while (m_running) { - Thread.Sleep(100); // let the world move .. back to faster rate + Thread.Sleep(1000); Watchdog.UpdateThread(); - lock (m_retryRequests) - { - while (m_retryRequests.Count > 0 && m_running) - m_requests.Enqueue(m_retryRequests.Dequeue()); - } - slowCount++; - if (slowCount >= 10) - { - slowCount = 0; - lock (m_slowRequests) + PollServiceHttpRequest req; + lock (m_longPollRequests) + { + while (m_longPollRequests.Count > 0 && m_running) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + req = m_longPollRequests.Dequeue(); + if (req.PollServiceArgs.HasEvents(req.RequestID, req.PollServiceArgs.Id) || // there are events in this EQ + (Environment.TickCount - req.RequestTime) > req.PollServiceArgs.TimeOutms) // no events, but timeout + m_requests.Enqueue(req); + else + m_longPollRequests.Enqueue(req); } } } @@ -153,24 +151,12 @@ namespace OpenSim.Framework.Servers.HttpServer foreach (Thread t in m_workerThreads) Watchdog.AbortThread(t.ManagedThreadId); - try - { - foreach (PollServiceHttpRequest req in m_retryRequests) - { - req.DoHTTPGruntWork(m_server, req.PollServiceArgs.NoEvents(req.RequestID, req.PollServiceArgs.Id)); - } - } - catch - { - } - PollServiceHttpRequest wreq; - m_retryRequests.Clear(); - lock (m_slowRequests) + lock (m_longPollRequests) { - while (m_slowRequests.Count > 0 && m_running) - m_requests.Enqueue(m_slowRequests.Dequeue()); + while (m_longPollRequests.Count > 0 && m_running) + m_requests.Enqueue(m_longPollRequests.Dequeue()); } while (m_requests.Count() > 0) @@ -196,6 +182,7 @@ namespace OpenSim.Framework.Servers.HttpServer while (m_running) { PollServiceHttpRequest req = m_requests.Dequeue(5000); + //m_log.WarnFormat("[YYY]: Dequeued {0}", (req == null ? "null" : req.PollServiceArgs.Type.ToString())); Watchdog.UpdateThread(); if (req != null) @@ -209,7 +196,7 @@ namespace OpenSim.Framework.Servers.HttpServer if (responsedata == null) continue; - if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Normal) // This is the event queue + if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LongPoll) // This is the event queue { try { diff --git a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs index 1835a72a4a..f0445ff3c3 100644 --- a/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs +++ b/OpenSim/Region/ClientStack/Linden/Caps/EventQueue/EventQueueGetModule.cs @@ -364,8 +364,7 @@ namespace OpenSim.Region.ClientStack.Linden caps.RegisterPollHandler( "EventQueueGet", - new PollServiceEventArgs( - null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000)); + new PollServiceEventArgs(null, GenerateEqgCapPath(eventQueueGetUUID), HasEvents, GetEvents, NoEvents, agentID, 40000)); Random rnd = new Random(Environment.TickCount); lock (m_ids) @@ -383,7 +382,10 @@ namespace OpenSim.Region.ClientStack.Linden Queue queue = GetQueue(agentID); if (queue != null) lock (queue) + { + //m_log.WarnFormat("POLLED FOR EVENTS BY {0} in {1} -- {2}", agentID, m_scene.RegionInfo.RegionName, queue.Count); return queue.Count > 0; + } return false; } @@ -406,7 +408,7 @@ namespace OpenSim.Region.ClientStack.Linden public Hashtable GetEvents(UUID requestID, UUID pAgentId) { if (DebugLevel >= 2) - m_log.DebugFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName); + m_log.WarnFormat("POLLED FOR EQ MESSAGES BY {0} in {1}", pAgentId, m_scene.RegionInfo.RegionName); Queue queue = TryGetQueue(pAgentId); OSD element;