Add optional pool for the UDPPacketBuffer objects that handle all incoming UDP data.

Even when an avatar is standing still, it's sending in a constant stream of AgentUpdate packets that the client creates new UDPPacketBuffer objects to handle.
This option pools those objects.  This reduces memory churn.
Currently off by default.  Works but the scope can be expanded.

OpenSim/Framework/Pool.cs

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the OpenSimulator Project nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Collections.Generic;
+
+namespace OpenSim.Framework
+{
+    /// <summary>
+    /// Naive pool implementation.
+    /// </summary>
+    /// <remarks>
+    /// Currently assumes that objects are in a useable state when returned.
+    /// </remarks>
+    public class Pool<T>
+    {
+        private Stack<T> m_pool;
+
+        private int m_maxPoolSize;
+
+        private Func<T> m_createFunction;
+
+        public Pool(Func<T> createFunction, int maxSize)
+        {
+            m_maxPoolSize = maxSize;
+            m_createFunction = createFunction;
+            m_pool = new Stack<T>(m_maxPoolSize);
+        }
+
+        public T GetObject()
+        {
+            lock (m_pool)
+            {
+                if (m_pool.Count > 0)
+                    return m_pool.Pop();
+                else
+                    return m_createFunction();
+            }
+        }
+
+        public void ReturnObject(T obj)
+        {
+            lock (m_pool)
+            {
+                if (m_pool.Count >= m_maxPoolSize)
+                    return;
+                else
+                    m_pool.Push(obj);
+            }
+        }
+    }
+}

OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs

@@ -188,7 +188,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// </summary>
         private IClientAPI m_currentIncomingClient;
 
-        public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
+        public LLUDPServer(
+            IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port,
+            IConfigSource configSource, AgentCircuitManager circuitManager)
             : base(listenIP, (int)port)
         {
             #region Environment.TickCount Measurement
@@ -242,6 +244,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 PacketPool.Instance.RecyclePackets = packetConfig.GetBoolean("RecyclePackets", true);
                 PacketPool.Instance.RecycleDataBlocks = packetConfig.GetBoolean("RecycleDataBlocks", true);
+                UsePools = packetConfig.GetBoolean("RecycleBaseUDPPackets", false);
             }
 
             #region BinaryStats
@@ -284,8 +287,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private void StartInbound()
         {
             m_log.InfoFormat(
-                "[LLUDPSERVER]: Starting inbound packet processing for the LLUDP server in {0} mode",
+                "[LLUDPSERVER]: Starting inbound packet processing for the LLUDP server in {0} mode with UsePools = {1}",
-                m_asyncPacketHandling ? "asynchronous" : "synchronous");
+                m_asyncPacketHandling ? "asynchronous" : "synchronous", UsePools);
 
             base.StartInbound(m_recvBufferSize, m_asyncPacketHandling);
 
@@ -300,7 +303,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS);
         }
 
-        private void StartOutbound()
+        private new void StartOutbound()
         {
             m_log.Info("[LLUDPSERVER]: Starting outbound packet processing for the LLUDP server");
 
@@ -317,7 +320,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS);
         }
 
-        public new void Stop()
+        public void Stop()
         {
             m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
             base.StopOutbound();
@@ -806,7 +809,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             LLUDPClient udpClient = null;
             Packet packet = null;
             int packetEnd = buffer.DataLength - 1;
-            IPEndPoint address = (IPEndPoint)buffer.RemoteEndPoint;
+            IPEndPoint endPoint = (IPEndPoint)buffer.RemoteEndPoint;
 
             #region Decoding
 
@@ -816,7 +819,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 //                    "[LLUDPSERVER]: Dropping undersized packet with {0} bytes received from {1} in {2}",
 //                    buffer.DataLength, buffer.RemoteEndPoint, m_scene.RegionInfo.RegionName);
 
-                return; // Drop undersizd packet
+                return; // Drop undersized packet
             }
 
             int headerLen = 7;
@@ -842,6 +845,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 //                packet = Packet.BuildPacket(buffer.Data, ref packetEnd,
 //                    // Only allocate a buffer for zerodecoding if the packet is zerocoded
 //                    ((buffer.Data[0] & Helpers.MSG_ZEROCODED) != 0) ? new byte[4096] : null);
+                // If OpenSimUDPBase.UsePool == true (which is currently separate from the PacketPool) then we
+                // assume that packet construction does not retain a reference to byte[] buffer.Data (instead, all
+                // bytes are copied out).
                 packet = PacketPool.Instance.GetPacket(buffer.Data, ref packetEnd,
                     // Only allocate a buffer for zerodecoding if the packet is zerocoded
                     ((buffer.Data[0] & Helpers.MSG_ZEROCODED) != 0) ? new byte[4096] : null);
@@ -884,7 +890,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             // UseCircuitCode handling
             if (packet.Type == PacketType.UseCircuitCode)
             {
-                object[] array = new object[] { buffer, packet };
+                // We need to copy the endpoint so that it doesn't get changed when another thread reuses the
+                // buffer.
+                object[] array = new object[] { new IPEndPoint(endPoint.Address, endPoint.Port), packet };
 
                 Util.FireAndForget(HandleUseCircuitCode, array);
 
@@ -893,7 +901,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             // Determine which agent this packet came from
             IClientAPI client;
-            if (!m_scene.TryGetClient(address, out client) || !(client is LLClientView))
+            if (!m_scene.TryGetClient(endPoint, out client) || !(client is LLClientView))
             {
                 //m_log.Debug("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address + " in " + m_scene.RegionInfo.RegionName);
                 return;
@@ -1091,21 +1099,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
         private void HandleUseCircuitCode(object o)
         {
-            IPEndPoint remoteEndPoint = null;
+            IPEndPoint endPoint = null;
             IClientAPI client = null;
 
             try
             {
     //            DateTime startTime = DateTime.Now;
                 object[] array = (object[])o;
-                UDPPacketBuffer buffer = (UDPPacketBuffer)array[0];
+                endPoint = (IPEndPoint)array[0];
                 UseCircuitCodePacket uccp = (UseCircuitCodePacket)array[1];
 
                 m_log.DebugFormat(
                     "[LLUDPSERVER]: Handling UseCircuitCode request for circuit {0} to {1} from IP {2}",
-                    uccp.CircuitCode.Code, m_scene.RegionInfo.RegionName, buffer.RemoteEndPoint);
+                    uccp.CircuitCode.Code, m_scene.RegionInfo.RegionName, endPoint);
-    
-                remoteEndPoint = (IPEndPoint)buffer.RemoteEndPoint;
     
                 AuthenticateResponse sessionInfo;
                 if (IsClientAuthorized(uccp, out sessionInfo))
@@ -1116,13 +1122,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                             uccp.CircuitCode.Code,
                             uccp.CircuitCode.ID,
                             uccp.CircuitCode.SessionID,
-                            remoteEndPoint,
+                            endPoint,
                             sessionInfo);
             
                     // Send ack straight away to let the viewer know that the connection is active.
                     // The client will be null if it already exists (e.g. if on a region crossing the client sends a use
                     // circuit code to the existing child agent.  This is not particularly obvious.
-                    SendAckImmediate(remoteEndPoint, uccp.Header.Sequence);
+                    SendAckImmediate(endPoint, uccp.Header.Sequence);
             
                     // We only want to send initial data to new clients, not ones which are being converted from child to root.
                     if (client != null)
@@ -1133,7 +1139,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // Don't create clients for unauthorized requesters.
                     m_log.WarnFormat(
                         "[LLUDPSERVER]: Ignoring connection request for {0} to {1} with unknown circuit code {2} from IP {3}",
-                        uccp.CircuitCode.ID, m_scene.RegionInfo.RegionName, uccp.CircuitCode.Code, remoteEndPoint);
+                        uccp.CircuitCode.ID, m_scene.RegionInfo.RegionName, uccp.CircuitCode.Code, endPoint);
                 }
     
                 //            m_log.DebugFormat(
@@ -1145,7 +1151,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 m_log.ErrorFormat(
                     "[LLUDPSERVER]: UseCircuitCode handling from endpoint {0}, client {1} {2} failed.  Exception {3}{4}",
-                    remoteEndPoint != null ? remoteEndPoint.ToString() : "n/a",
+                    endPoint != null ? endPoint.ToString() : "n/a",
                     client != null ? client.Name : "unknown",
                     client != null ? client.AgentId.ToString() : "unknown",
                     e.Message,

OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs

@@ -30,6 +30,7 @@ using System.Net;
 using System.Net.Sockets;
 using System.Threading;
 using log4net;
+using OpenSim.Framework;
 
 namespace OpenMetaverse
 {
@@ -58,6 +59,16 @@ namespace OpenMetaverse
         /// <summary>Flag to process packets asynchronously or synchronously</summary>
         private bool m_asyncPacketHandling;
 
+        /// <summary>
+        /// Pool to use for handling data.  May be null if UsePools = false;
+        /// </summary>
+        protected OpenSim.Framework.Pool<UDPPacketBuffer> m_pool;
+
+        /// <summary>
+        /// Are we to use object pool(s) to reduce memory churn when receiving data?
+        /// </summary>
+        public bool UsePools { get; protected set; }
+
         /// <summary>Returns true if the server is currently listening for inbound packets, otherwise false</summary>
         public bool IsRunningInbound { get; private set; }
 
@@ -70,6 +81,7 @@ namespace OpenMetaverse
         /// </summary>
         /// <param name="bindAddress">Local IP address to bind the server to</param>
         /// <param name="port">Port to listening for incoming UDP packets on</param>
+        /// /// <param name="usePool">Are we to use an object pool to get objects for handing inbound data?</param>
         public OpenSimUDPBase(IPAddress bindAddress, int port)
         {
             m_localBindAddress = bindAddress;
@@ -94,6 +106,11 @@ namespace OpenMetaverse
         /// necessary</remarks>
         public void StartInbound(int recvBufferSize, bool asyncPacketHandling)
         {
+            if (UsePools)
+                m_pool = new Pool<UDPPacketBuffer>(() => new UDPPacketBuffer(), 500);
+            else
+                m_pool = null;
+
             m_asyncPacketHandling = asyncPacketHandling;
 
             if (!IsRunningInbound)
@@ -165,9 +182,12 @@ namespace OpenMetaverse
 
         private void AsyncBeginReceive()
         {
-            // allocate a packet buffer
+            UDPPacketBuffer buf;
-            //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
+
-            UDPPacketBuffer buf = new UDPPacketBuffer();
+            if (UsePools)
+                buf = m_pool.GetObject();
+            else
+                buf = new UDPPacketBuffer();
 
             if (IsRunningInbound)
             {
@@ -231,8 +251,6 @@ namespace OpenMetaverse
 
                 // get the buffer that was created in AsyncBeginReceive
                 // this is the received data
-                //WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
-                //UDPPacketBuffer buffer = wrappedBuffer.Instance;
                 UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
 
                 try
@@ -249,7 +267,8 @@ namespace OpenMetaverse
                 catch (ObjectDisposedException) { }
                 finally
                 {
-                    //wrappedBuffer.Dispose();
+                    if (UsePools)
+                        m_pool.ReturnObject(buffer);
 
                     // Synchronous mode waits until the packet callback completes
                     // before starting the receive to fetch another packet

bin/OpenSimDefaults.ini

@@ -1602,10 +1602,14 @@
 
 
 [PacketPool]
-    ; Enables the experimental packet pool. Yes, we've been here before.
     ;RecyclePackets = true;
     ;RecycleDataBlocks = true;
 
+    ; If true, then the basic packet objects used to receive data are also recycled, not just the LLUDP packets.
+    ; This reduces data churn
+    ; This setting is currently experimental and defaults to false.
+    RecycleBaseUDPPackets = false;
+
 
 [InterestManagement]
      ; This section controls how state updates are prioritized for each client