From ac7bc78555c1dd51724790032f0711b24bc8c67d Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Tue, 8 Feb 2011 12:06:14 -0800 Subject: [PATCH] Added emergency monitoring of UDP Outgoing packets thread. Just type "emergency-monitoring on/off" --- .../ClientStack/LindenUDP/LLUDPServer.cs | 100 ++++++++++++++++++ OpenSim/Region/Framework/Scenes/Scene.cs | 2 + .../Agent/UDP/Linden/LindenUDPInfoModule.cs | 30 +++++- 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs index 5ff9aeee8b..72257d28ce 100644 --- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Net; using System.Net.Sockets; @@ -1053,6 +1054,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP #endregion Update Timers + if (m_scene.EmergencyMonitoring) + clientPacketHandler = MonitoredClientOutgoingPacketHandler; + // Handle outgoing packets, resends, acknowledgements, and pings for each // client. m_packetSent will be set to true if a packet is sent m_scene.ForEachClient(clientPacketHandler); @@ -1068,6 +1072,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP { m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler loop threw an exception: " + ex.Message, ex); } + } Watchdog.RemoveThread(); @@ -1105,6 +1110,101 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } + #region Emergency Monitoring + private Stopwatch watch1 = new Stopwatch(); + private Stopwatch watch2 = new Stopwatch(); + + private float avgProcessingTicks = 0; + private float avgResendUnackedTicks = 0; + private float avgSendAcksTicks = 0; + private float avgSendPingTicks = 0; + private float avgDequeueTicks = 0; + private long nticks = 0; + private long nticksUnack = 0; + private long nticksAck = 0; + private long nticksPing = 0; + + private void MonitoredClientOutgoingPacketHandler(IClientAPI client) + { + nticks++; + watch1.Start(); + try + { + if (client is LLClientView) + { + LLUDPClient udpClient = ((LLClientView)client).UDPClient; + + if (udpClient.IsConnected) + { + if (m_resendUnacked) + { + nticksUnack++; + watch2.Start(); + + ResendUnacked(udpClient); + + watch2.Stop(); + avgResendUnackedTicks = (nticksUnack - 1)/(float)nticksUnack * avgResendUnackedTicks + (watch2.ElapsedTicks / (float)nticksUnack); + watch2.Reset(); + } + + if (m_sendAcks) + { + nticksAck++; + watch2.Start(); + + SendAcks(udpClient); + + watch2.Stop(); + avgSendAcksTicks = (nticksAck - 1) / (float)nticksAck * avgSendAcksTicks + (watch2.ElapsedTicks / (float)nticksAck); + watch2.Reset(); + } + + if (m_sendPing) + { + nticksPing++; + watch2.Start(); + + SendPing(udpClient); + + watch2.Stop(); + avgSendPingTicks = (nticksPing - 1) / (float)nticksPing * avgSendPingTicks + (watch2.ElapsedTicks / (float)nticksPing); + watch2.Reset(); + } + + watch2.Start(); + // Dequeue any outgoing packets that are within the throttle limits + if (udpClient.DequeueOutgoing()) + m_packetSent = true; + watch2.Stop(); + avgDequeueTicks = (nticks - 1) / (float)nticks * avgDequeueTicks + (watch2.ElapsedTicks / (float)nticks); + watch2.Reset(); + + } + else + m_log.WarnFormat("[LLUDPSERVER]: Client is not connected"); + } + } + catch (Exception ex) + { + m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler iteration for " + client.Name + + " threw an exception: " + ex.Message, ex); + } + watch1.Stop(); + avgProcessingTicks = (nticks - 1) / (float)nticks * avgProcessingTicks + (watch1.ElapsedTicks / (float)nticks); + watch1.Reset(); + + // reuse this -- it's every 100ms + if (m_scene.EmergencyMonitoring && nticks % 100 == 0) + { + m_log.InfoFormat("[LLUDPSERVER]: avg processing ticks: {0} avg unacked: {1} avg acks: {2} avg ping: {3} avg dequeue: {4} (pack sent? {5})", + avgProcessingTicks, avgResendUnackedTicks, avgSendAcksTicks, avgSendPingTicks, avgDequeueTicks, m_packetSent); + } + + } + + #endregion + private void ProcessInPacket(object state) { IncomingPacket incomingPacket = (IncomingPacket)state; diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index 4fca2610d2..355671cde3 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -64,6 +64,8 @@ namespace OpenSim.Region.Framework.Scenes #region Fields + public bool EmergencyMonitoring = false; + public SynchronizeSceneHandler SynchronizeScene; public SimStatsReporter StatsReporter; public List NorthBorders = new List(); diff --git a/OpenSim/Region/OptionalModules/Agent/UDP/Linden/LindenUDPInfoModule.cs b/OpenSim/Region/OptionalModules/Agent/UDP/Linden/LindenUDPInfoModule.cs index 6630edb15e..53cebb2189 100644 --- a/OpenSim/Region/OptionalModules/Agent/UDP/Linden/LindenUDPInfoModule.cs +++ b/OpenSim/Region/OptionalModules/Agent/UDP/Linden/LindenUDPInfoModule.cs @@ -95,7 +95,15 @@ namespace OpenSim.Region.CoreModules.UDP.Linden "Show throttle settings for each client and for the server overall", "Without the 'full' option, only root agents are shown." + " With the 'full' option child agents are also shown.", - ShowThrottlesReport); + ShowThrottlesReport); + + scene.AddCommand( + this, "emergency-monitoring", + "Go on/off emergency monitoring mode", + "Go on/off emergency monitoring mode", + "Go on/off emergency monitoring mode", + EmergencyMonitoring); + } public void RemoveRegion(Scene scene) @@ -120,7 +128,25 @@ namespace OpenSim.Region.CoreModules.UDP.Linden { MainConsole.Instance.Output(GetThrottlesReport(cmd)); } - + + protected void EmergencyMonitoring(string module, string[] cmd) + { + bool mode = true; + if (cmd.Length == 1 || (cmd.Length > 1 && cmd[1] == "on")) + { + mode = true; + MainConsole.Instance.Output("Emergency Monitoring ON"); + } + else + { + mode = false; + MainConsole.Instance.Output("Emergency Monitoring OFF"); + } + + foreach (Scene s in m_scenes.Values) + s.EmergencyMonitoring = mode; + } + protected string GetColumnEntry(string entry, int maxLength, int columnPadding) { return string.Format(