* Rex merge, rest of clientstack. Clean merge.

OpenSim/Region/ClientStack/PacketQueue.cs

@@ -29,14 +29,19 @@ using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Timers;
+using libsecondlife;
 using libsecondlife.Packets;
 using OpenSim.Framework;
+using OpenSim.Framework.Statistics;
+using OpenSim.Framework.Statistics.Interfaces;
 using Timer=System.Timers.Timer;
 
 namespace OpenSim.Region.ClientStack
 {
-    public class PacketQueue
+    public class PacketQueue : IPullStatsProvider
     {
+        //private static readonly log4net.ILog m_log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         private bool m_enabled = true;
 
         private BlockingQueue<QueItem> SendQueue;
@@ -75,7 +80,9 @@ namespace OpenSim.Region.ClientStack
         // private long ThrottleInterval;
         private Timer throttleTimer;
         
-        public PacketQueue()
+        private LLUUID m_agentId;
+
+        public PacketQueue(LLUUID agentId)
         {
             // While working on this, the BlockingQueue had me fooled for a bit.
             // The Blocking queue causes the thread to stop until there's something 
@@ -114,6 +121,13 @@ namespace OpenSim.Region.ClientStack
             // TIMERS needed for this
             // LastThrottle = DateTime.Now.Ticks;
             // ThrottleInterval = (long)(throttletimems/throttleTimeDivisor);
+            
+            m_agentId = agentId;
+            
+            if (StatsManager.SimExtraStats != null)            
+            {
+                StatsManager.SimExtraStats.RegisterPacketQueueStatsProvider(m_agentId, this);
+            }
         }
 
         /* STANDARD QUEUE MANIPULATION INTERFACES */
@@ -204,7 +218,7 @@ namespace OpenSim.Region.ClientStack
                         SendQueue.Enqueue(AssetOutgoingPacketQueue.Dequeue());
                     }
                 }
-                // MainLog.Instance.Verbose("THROTTLE", "Processed " + throttleLoops + " packets");
+                // m_log.Info("[THROTTLE]: Processed " + throttleLoops + " packets");
             }
         }
 
@@ -212,6 +226,11 @@ namespace OpenSim.Region.ClientStack
         {
             m_enabled = false;
             throttleTimer.Stop();
+            
+            if (StatsManager.SimExtraStats != null)            
+            {
+                StatsManager.SimExtraStats.DeregisterPacketQueueStatsProvider(m_agentId);
+            }            
         }
 
         private void ResetCounters()
@@ -253,7 +272,7 @@ namespace OpenSim.Region.ClientStack
             lock (this)
             {
                 ResetCounters();
-                // MainLog.Instance.Verbose("THROTTLE", "Entering Throttle");
+                // m_log.Info("[THROTTLE]: Entering Throttle");
                 while (TotalThrottle.UnderLimit() && PacketsWaiting() &&
                        (throttleLoops <= MaxThrottleLoops))
                 {
@@ -316,7 +335,7 @@ namespace OpenSim.Region.ClientStack
                         AssetThrottle.Add(qpack.Packet.ToBytes().Length);
                     }
                 }
-                // MainLog.Instance.Verbose("THROTTLE", "Processed " + throttleLoops + " packets");
+                // m_log.Info("[THROTTLE]: Processed " + throttleLoops + " packets");
             }
         }
 
@@ -356,7 +375,7 @@ namespace OpenSim.Region.ClientStack
             return (int) (((float) value/(float) curmax)*newmax);
         }
 
-        private byte[] GetThrottlesPacked(float multiplier)
+        public byte[] GetThrottlesPacked(float multiplier)
         {
             int singlefloat = 4;
             float tResend = ResendThrottle.Throttle*multiplier;
@@ -426,7 +445,7 @@ namespace OpenSim.Region.ClientStack
 
             tall = tResend + tLand + tWind + tCloud + tTask + tTexture + tAsset;
             /*
-              MainLog.Instance.Verbose("CLIENT", "Client AgentThrottle - Got throttle:resendbytes=" + tResend +
+              m_log.Info("[CLIENT]: Client AgentThrottle - Got throttle:resendbytes=" + tResend +
               " landbytes=" + tLand +
               " windbytes=" + tWind +
               " cloudbytes=" + tCloud +
@@ -481,5 +500,21 @@ namespace OpenSim.Region.ClientStack
             // effectively wiggling the slider causes things reset
             ResetCounters();
         }
+        
+        // See IPullStatsProvider
+        public string GetStats()
+        {
+            return string.Format("{0,7}  {1,7}  {2,7}  {3,7}  {4,7}  {5,7}  {6,7}  {7,7}  {8,7}  {9,7}",
+                                 SendQueue.Count(),
+                                 IncomingPacketQueue.Count,
+                                 OutgoingPacketQueue.Count,
+                                 ResendOutgoingPacketQueue.Count,
+                                 LandOutgoingPacketQueue.Count,
+                                 WindOutgoingPacketQueue.Count,
+                                 CloudOutgoingPacketQueue.Count,
+                                 TaskOutgoingPacketQueue.Count,
+                                 TextureOutgoingPacketQueue.Count,
+                                 AssetOutgoingPacketQueue.Count);                                     
+        }
     }
 }

OpenSim/Region/ClientStack/PacketServer.cs

@@ -25,6 +25,8 @@
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * 
 */
+
+using System;
 using System.Net;
 using System.Net.Sockets;
 using libsecondlife;
@@ -36,6 +38,9 @@ namespace OpenSim.Region.ClientStack
 {
     public class PacketServer
     {
+        //private static readonly log4net.ILog m_log 
+        //    = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         private ClientStackNetworkHandler m_networkHandler;
         private IScene m_scene;
 
@@ -130,9 +135,16 @@ namespace OpenSim.Region.ClientStack
             //m_scene.ClientManager.CloseAllAgents(circuitcode);
         }
 
+        /// <summary>
+        /// Completely close down the given client.
+        /// </summary>
+        /// <param name="client"></param>
         public virtual void CloseClient(IClientAPI client)
         {
+            //m_log.Info("PacketServer:CloseClient()");
+         
             CloseCircuit(client.CircuitCode);
+            m_scene.ClientManager.Remove(client.CircuitCode);                
             client.Close(false);
         }
     }

OpenSim/Region/ClientStack/RegionApplicationBase.cs

@@ -41,17 +41,17 @@ using OpenSim.Region.Communications.VoiceChat;
 
 namespace OpenSim.Region.ClientStack
 {
-    public abstract class RegionApplicationBase
+    public abstract class RegionApplicationBase : BaseOpenSimServer
     {
+        private static readonly log4net.ILog m_log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         protected AssetCache m_assetCache;
         protected Dictionary<EndPoint, uint> m_clientCircuits = new Dictionary<EndPoint, uint>();
-        protected DateTime m_startuptime;
         protected NetworkServersInfo m_networkServersInfo;
 
         protected BaseHttpServer m_httpServer;
         protected uint m_httpServerPort;
 
-        protected LogBase m_log;
         protected CommunicationsManager m_commsManager;
 
         protected SceneManager m_sceneManager = new SceneManager();
@@ -59,6 +59,10 @@ namespace OpenSim.Region.ClientStack
         protected StorageManager m_storageManager;
         protected string m_storageConnectionString;
         
+        // An attribute to indicate whether prim inventories should be persisted.
+        // Probably will be temporary until this stops being experimental.
+        protected bool m_storagePersistPrimInventories;
+
         protected VoiceChatServer m_voiceChatServer;
 
         public SceneManager SceneManager
@@ -66,11 +70,6 @@ namespace OpenSim.Region.ClientStack
             get { return m_sceneManager; }
         }
 
-        public RegionApplicationBase()
-        {
-            m_startuptime = DateTime.Now;
-        }
-
         public virtual void StartUp()
         {
             ClientView.TerrainManager = new TerrainManager(new SecondLife());
@@ -81,19 +80,20 @@ namespace OpenSim.Region.ClientStack
 
             m_httpServer = new BaseHttpServer(m_httpServerPort);
 
-            m_log.Status("REGION", "Starting HTTP server");
+            m_log.Info("[REGION]: Starting HTTP server");
+
             m_httpServer.Start();
         }
 
         protected abstract void Initialize();
 
-        protected void StartLog()
+        protected void StartConsole()
         {
-            m_log = CreateLog();
+            m_console = CreateConsole();
-            MainLog.Instance = m_log;
+            MainConsole.Instance = m_console;
         }
 
-        protected abstract LogBase CreateLog();
+        protected abstract ConsoleBase CreateConsole();
         protected abstract PhysicsScene GetPhysicsScene();
         protected abstract StorageManager CreateStorageManager(string connectionstring);
 
@@ -108,7 +108,13 @@ namespace OpenSim.Region.ClientStack
         protected Scene SetupScene(RegionInfo regionInfo, out UDPServer udpServer, bool m_permissions)
         {
             AgentCircuitManager circuitManager = new AgentCircuitManager();
-            udpServer = new UDPServer((uint) regionInfo.InternalEndPoint.Port, m_assetCache, m_log, circuitManager);
+            IPAddress listenIP = regionInfo.InternalEndPoint.Address;
+            //if (!IPAddress.TryParse(regionInfo.InternalEndPoint, out listenIP))
+            //    listenIP = IPAddress.Parse("0.0.0.0");
+
+            uint port = (uint) regionInfo.InternalEndPoint.Port;
+            udpServer = new UDPServer(listenIP, ref port, regionInfo.m_allow_alternate_ports, m_assetCache, circuitManager);
+            regionInfo.InternalEndPoint.Port = (int)port;
 
             Scene scene = CreateScene(regionInfo, m_storageManager, circuitManager);
             m_voiceChatServer = new VoiceChatServer(scene);
@@ -138,12 +144,12 @@ namespace OpenSim.Region.ClientStack
 
             if (masterAvatar != null)
             {
-                m_log.Verbose("PARCEL", "Found master avatar [" + masterAvatar.UUID.ToString() + "]");
+                m_log.Info("[PARCEL]: Found master avatar [" + masterAvatar.UUID.ToString() + "]");
                 scene.RegionInfo.MasterAvatarAssignedUUID = masterAvatar.UUID;
             }
             else
             {
-                m_log.Verbose("PARCEL", "No master avatar found, using null.");
+                m_log.Info("[PARCEL]: No master avatar found, using null.");
                 scene.RegionInfo.MasterAvatarAssignedUUID = LLUUID.Zero;
             }
 

OpenSim/Region/ClientStack/UDPServer.cs

@@ -39,6 +39,8 @@ namespace OpenSim.Region.ClientStack
 {
     public class UDPServer : ClientStackNetworkHandler
     {
+        private static readonly log4net.ILog m_log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         protected Dictionary<EndPoint, uint> clientCircuits = new Dictionary<EndPoint, uint>();
         public Dictionary<uint, EndPoint> clientCircuits_reverse = new Dictionary<uint, EndPoint>();
         public Socket Server;
@@ -52,9 +54,10 @@ namespace OpenSim.Region.ClientStack
         protected ulong m_regionHandle;
 
         protected uint listenPort;
+        protected bool Allow_Alternate_Port;
+        protected IPAddress listenIP = IPAddress.Parse("0.0.0.0");
         protected IScene m_localScene;
         protected AssetCache m_assetCache;
-        protected LogBase m_log;
         protected AgentCircuitManager m_authenticateSessionsClass;
 
         public PacketServer PacketServer
@@ -82,13 +85,19 @@ namespace OpenSim.Region.ClientStack
         {
         }
 
-        public UDPServer(uint port, AssetCache assetCache, LogBase console, AgentCircuitManager authenticateClass)
+        public UDPServer(IPAddress _listenIP, ref uint port, bool allow_alternate_port, AssetCache assetCache, AgentCircuitManager authenticateClass)
         {
+            listenIP = _listenIP;
             listenPort = port;
+            Allow_Alternate_Port = allow_alternate_port;
             m_assetCache = assetCache;
-            m_log = console;
             m_authenticateSessionsClass = authenticateClass;
             CreatePacketServer();
+
+            // Return new port
+            // This because in Grid mode it is not really important what port the region listens to as long as it is correctly registered.
+            // So the option allow_alternate_ports="true" was added to default.xml
+            port = listenPort;
         }
 
         protected virtual void CreatePacketServer()
@@ -98,11 +107,11 @@ namespace OpenSim.Region.ClientStack
 
         protected virtual void OnReceivedData(IAsyncResult result)
         {
-            ipeSender = new IPEndPoint(IPAddress.Parse("0.0.0.0"), 0);
+            ipeSender = new IPEndPoint(listenIP, 0);
             epSender = (EndPoint) ipeSender;
             Packet packet = null;
 
-            int numBytes;
+            int numBytes = 1;
 
             try
             {
@@ -115,6 +124,7 @@ namespace OpenSim.Region.ClientStack
             {
                 // TODO : Actually only handle those states that we have control over, re-throw everything else,
                 // TODO: implement cases as we encounter them.
+                //m_log.Error("[UDPSERVER]: Connection Error! - " + e.ToString());
                 switch (e.SocketErrorCode)
                 {
                     case SocketError.AlreadyInProgress:
@@ -127,7 +137,7 @@ namespace OpenSim.Region.ClientStack
                         }
                         catch (Exception a)
                         {
-                            MainLog.Instance.Verbose("UDPSERVER", a.ToString());
+                            m_log.Info("[UDPSERVER]: " + a.ToString());
                         }
                         try
                         {
@@ -152,7 +162,7 @@ namespace OpenSim.Region.ClientStack
                         }
                         catch (Exception)
                         {
-                            //MainLog.Instance.Verbose("UDPSERVER", a.ToString());
+                            //m_log.Info("[UDPSERVER]" + a.ToString());
                         }
                         try
                         {
@@ -166,8 +176,9 @@ namespace OpenSim.Region.ClientStack
                             // Stupid I know..  
                             // but Flusing the buffer would be even more stupid...  so, we're stuck with this ugly method.
                         }
-                        catch (SocketException)
+                        catch (SocketException e2)
                         {
+                            m_log.Error("[UDPSERVER]: " + e2.ToString());
                         }
 
                         // Here's some reference code!   :D  
@@ -180,86 +191,194 @@ namespace OpenSim.Region.ClientStack
                         break;
                 }
 
-                return;
+                //return;
             }
-            catch (ObjectDisposedException)
+            catch (ObjectDisposedException e)
             {
-                //MainLog.Instance.Debug("UDPSERVER", e.ToString());
+                m_log.Debug("[UDPSERVER]: " + e.ToString());
-                return;
+                try
+                {
+                    Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender,
+                                            ReceivedData, null);
+
+                    // Ter: For some stupid reason ConnectionReset basically kills our async event structure..  
+                    // so therefore..  we've got to tell the server to BeginReceiveFrom again.
+                    // This will happen over and over until we've gone through all packets 
+                    // sent to and from this particular user.
+                    // Stupid I know..  
+                    // but Flusing the buffer would be even more stupid...  so, we're stuck with this ugly method.
+                }
+                catch (SocketException e2)
+                {
+                    m_log.Error("[UDPSERVER]: " + e2.ToString());
+                }
+                //return;
             }
 
             int packetEnd = numBytes - 1;
 
             try
             {
-                packet = PacketPool.Instance.GetPacket(RecvBuffer, ref packetEnd, ZeroBuffer);
+               packet = PacketPool.Instance.GetPacket(RecvBuffer, ref packetEnd, ZeroBuffer);
+               
             }
-            catch (Exception)
+            catch (Exception e)
             {
-                //MainLog.Instance.Debug("UDPSERVER", e.ToString());
+                m_log.Debug("[UDPSERVER]: " + e.ToString());
+            }
+
+            try
+            {
+                Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender, ReceivedData, null);
+            }
+            catch (SocketException e4)
+            {
+                try
+                {
+                    CloseEndPoint(epSender);
+                }
+                catch (Exception a)
+                {
+                    m_log.Info("[UDPSERVER]: " + a.ToString());
+                }
+                try
+                {
+                    Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender,
+                                            ReceivedData, null);
+
+                    // Ter: For some stupid reason ConnectionReset basically kills our async event structure..  
+                    // so therefore..  we've got to tell the server to BeginReceiveFrom again.
+                    // This will happen over and over until we've gone through all packets 
+                    // sent to and from this particular user.
+                    // Stupid I know..  
+                    // but Flusing the buffer would be even more stupid...  so, we're stuck with this ugly method.
+                }
+                catch (SocketException e5)
+                {
+                    m_log.Error("[UDPSERVER]: " + e5.ToString());
+                }
+
             }
 
             if (packet != null)
             {
-                // do we already have a circuit for this endpoint
+                try
-                uint circuit;
-                if (clientCircuits.TryGetValue(epSender, out circuit))
                 {
-                    //if so then send packet to the packetserver
+                    // do we already have a circuit for this endpoint
-                    //MainLog.Instance.Warn("UDPSERVER", "ALREADY HAVE Circuit!");
+                    uint circuit;
-                    m_packetServer.InPacket(circuit, packet);
+
+                    bool ret = false;
+                    lock (clientCircuits)
+                    {
+                        ret = clientCircuits.TryGetValue(epSender, out circuit);
+                    }
+                    if (ret)
+                    {
+                        //if so then send packet to the packetserver
+                        //m_log.Warn("[UDPSERVER]: ALREADY HAVE Circuit!");
+                        m_packetServer.InPacket(circuit, packet);
+                    }
+                    else if (packet.Type == PacketType.UseCircuitCode)
+                    {
+                        // new client
+                        m_log.Debug("[UDPSERVER]: Adding New Client");
+                        AddNewClient(packet);
+                    }
+                    else
+                    {
+                        // invalid client
+                        //CFK: This message seems to have served its usefullness as of 12-15 so I am commenting it out for now
+                        //m_log.Warn("[UDPSERVER]: Got a packet from an invalid client - " + packet.ToString());
+
+                    }
                 }
-                else if (packet.Type == PacketType.UseCircuitCode)
+                catch (Exception ex)
                 {
-                    // new client
+                    m_log.Error("[UDPSERVER]: Exception in processing packet.");
-                    MainLog.Instance.Debug("UDPSERVER", "Adding New Client");
+                    m_log.Debug("[UDPSERVER]: Adding New Client");
-                    AddNewClient(packet);
+                    try
-                }
+                    {
-                else
+                        AddNewClient(packet);
-                {
+                    }
-                    // invalid client
+                    catch (Exception e3)
-                    //CFK: This message seems to have served its usefullness as of 12-15 so I am commenting it out for now
+                    {
-                    //m_log.Warn("client", "Got a packet from an invalid client - " + epSender.ToString());
+                        m_log.Error("[UDPSERVER]: Adding New Client threw exception " + e3.ToString());
+                        Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender,
+                                                    ReceivedData, null);
+                    }
                 }
             }
             
-            Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender, ReceivedData, null);
         }
 
         private void CloseEndPoint(EndPoint sender)
         {
             uint circuit;
-            if (clientCircuits.TryGetValue(sender, out circuit))
+            lock (clientCircuits)
             {
-                m_packetServer.CloseCircuit(circuit);
+                if (clientCircuits.TryGetValue(sender, out circuit))
+                {
+                    m_packetServer.CloseCircuit(circuit);
+                }
             }
         }
 
         protected virtual void AddNewClient(Packet packet)
         {
             UseCircuitCodePacket useCircuit = (UseCircuitCodePacket) packet;
-            clientCircuits.Add(epSender, useCircuit.CircuitCode.Code);
+            lock (clientCircuits)
-            clientCircuits_reverse.Add(useCircuit.CircuitCode.Code, epSender);
+            {
+                if (!clientCircuits.ContainsKey(epSender))
+                    clientCircuits.Add(epSender, useCircuit.CircuitCode.Code);
+                else
+                    m_log.Error("[UDPSERVER]: clientCircuits already contans entry for user " + useCircuit.CircuitCode.Code.ToString() + ". NOT adding.");
+            }
+            lock (clientCircuits_reverse)
+            {
+                if (!clientCircuits_reverse.ContainsKey(useCircuit.CircuitCode.Code))
+                    clientCircuits_reverse.Add(useCircuit.CircuitCode.Code, epSender);
+                else
+                    m_log.Error("[UDPSERVER]: clientCurcuits_reverse already contains entry for user " + useCircuit.CircuitCode.Code.ToString() + ". NOT adding.");
+            }
 
             PacketServer.AddNewClient(epSender, useCircuit, m_assetCache, m_authenticateSessionsClass);
         }
 
         public void ServerListener()
         {
-            m_log.Verbose("SERVER", "Opening UDP socket on " + listenPort.ToString());
+            uint newPort = listenPort;
+            for (uint i = 0; i < 20; i++)
+            {
+                newPort = listenPort + i;
+                m_log.Info("[SERVER]: Opening UDP socket on " + listenIP.ToString() + " " + newPort + ".");// Allow alternate ports: " + Allow_Alternate_Port.ToString());
+                try
+                {
+                    ServerIncoming = new IPEndPoint(listenIP, (int) newPort);
+                    Server = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+                    Server.Bind(ServerIncoming);
+                    listenPort = newPort;
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    // We are not looking for alternate ports?
+                    //if (!Allow_Alternate_Port)
+                        throw (ex);
 
-            ServerIncoming = new IPEndPoint(IPAddress.Parse("0.0.0.0"), (int) listenPort);
+                    // We are looking for alternate ports!
-            Server = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+                    m_log.Info("[SERVER]: UDP socket on " + listenIP.ToString() + " " + listenPort.ToString() + " is not available, trying next.");
-            Server.Bind(ServerIncoming);
+                }
+                System.Threading.Thread.Sleep(100); // Wait before we retry socket
+            }
 
-            m_log.Verbose("SERVER", "UDP socket bound, getting ready to listen");
+            m_log.Info("[SERVER]: UDP socket bound, getting ready to listen");
 
-            ipeSender = new IPEndPoint(IPAddress.Parse("0.0.0.0"), 0);
+            ipeSender = new IPEndPoint(listenIP, 0);
             epSender = (EndPoint) ipeSender;
             ReceivedData = new AsyncCallback(OnReceivedData);
             Server.BeginReceiveFrom(RecvBuffer, 0, RecvBuffer.Length, SocketFlags.None, ref epSender, ReceivedData, null);
 
-            m_log.Status("SERVER", "Listening...");
+            m_log.Info("[SERVER]: Listening on port " + newPort);
         }
 
         public virtual void RegisterPacketServer(PacketServer server)
@@ -272,30 +391,12 @@ namespace OpenSim.Region.ClientStack
         {
             // find the endpoint for this circuit
             EndPoint sendto = null;
-            if (clientCircuits_reverse.TryGetValue(circuitcode, out sendto))
+            lock (clientCircuits_reverse)
             {
-                //we found the endpoint so send the packet to it
+                if (clientCircuits_reverse.TryGetValue(circuitcode, out sendto))
-                while (true)
                 {
-                    try
+                    //we found the endpoint so send the packet to it
-                    {
+                    Server.SendTo(buffer, size, flags, sendto);
-                        Server.SendTo(buffer, size, flags, sendto);
-                        return;
-                    }
-                    catch (SocketException e)
-                    {
-                        if (e.ErrorCode == 10055)
-                        {
-                            //Send buffer full, halt for half second and retry
-                            MainLog.Instance.Warn("SERVER", "Socket send buffer was full, halting for 200ms");
-                            System.Threading.Thread.Sleep(200);
-                        }
-                        else
-                        {
-                            //Rethrow
-                            throw e;
-                        }
-                    }
                 }
             }
         }
@@ -303,12 +404,15 @@ namespace OpenSim.Region.ClientStack
         public virtual void RemoveClientCircuit(uint circuitcode)
         {
             EndPoint sendto = null;
-            if (clientCircuits_reverse.TryGetValue(circuitcode, out sendto))
+            lock (clientCircuits_reverse)
             {
-                clientCircuits.Remove(sendto);
+                if (clientCircuits_reverse.TryGetValue(circuitcode, out sendto))
+                {
+                    clientCircuits.Remove(sendto);
 
 
-                clientCircuits_reverse.Remove(circuitcode);
+                    clientCircuits_reverse.Remove(circuitcode);
+                }
             }
         }
     }