* Minimized the number of times textures are pulled off the priority queue

* OnQueueEmpty is still called async, but will not be called for a given category if the previous callback for that category is still running. This is the most balanced behavior I could find, and seems to work well
* Added support for the old [ClientStack.LindenUDP] settings (including setting the receive buffer size) and added the new token bucket and global throttle settings
* Added the AssetLoaderEnabled config variable to optionally disable loading assets from XML every startup. This gives a dramatic improvement in startup times for those who don't need the functionality every startup

OpenSim/Region/ClientStack/LindenUDP/J2KImage.cs

@@ -72,14 +72,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             m_imageManager = imageManager;
         }
 
-        public bool SendPackets(LLClientView client, int maxpack)
+        /// <summary>
+        /// Sends packets for this texture to a client until packetsToSend is 
+        /// hit or the transfer completes
+        /// </summary>
+        /// <param name="client">Reference to the client that the packets are destined for</param>
+        /// <param name="packetsToSend">Maximum number of packets to send during this call</param>
+        /// <param name="packetsSent">Number of packets sent during this call</param>
+        /// <returns>True if the transfer completes at the current discard level, otherwise false</returns>
+        public bool SendPackets(LLClientView client, int packetsToSend, out int packetsSent)
         {
-            if (client == null)
+            packetsSent = 0;
-                return false;
 
             if (m_currentPacket <= m_stopPacket)
             {
-                int count = 0;
                 bool sendMore = true;
 
                 if (!m_sentInfo || (m_currentPacket == 0))
@@ -88,25 +94,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
                     m_sentInfo = true;
                     ++m_currentPacket;
-                    ++count;
+                    ++packetsSent;
                 }
                 if (m_currentPacket < 2)
                 {
                     m_currentPacket = 2;
                 }
                 
-                while (sendMore && count < maxpack && m_currentPacket <= m_stopPacket)
+                while (sendMore && packetsSent < packetsToSend && m_currentPacket <= m_stopPacket)
                 {
                     sendMore = SendPacket(client);
                     ++m_currentPacket;
-                    ++count;
+                    ++packetsSent;
                 }
-
-                if (m_currentPacket > m_stopPacket)
-                    return true;
             }
 
-            return false;
+            return (m_currentPacket > m_stopPacket);
         }
 
         public void RunUpdate()

OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs

@@ -313,10 +313,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         protected int m_primFullUpdatesPerPacket = 14;
         protected int m_primTerseUpdateRate = 10;
         protected int m_primFullUpdateRate = 14;
-        protected int m_textureSendLimit   = 20;
-        protected int m_textureDataLimit   = 10;
         protected int m_avatarTerseUpdateRate = 50;
         protected int m_avatarTerseUpdatesPerPacket = 5;
+        /// <summary>Number of texture packets to put on the queue each time the
+        /// OnQueueEmpty event is triggered for the texture category</summary>
+        protected int m_textureSendLimit = 20;
         protected IAssetService m_assetService;
 
         #endregion Class Members
@@ -3453,7 +3454,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         void ProcessTextureRequests()
         {
             if (m_imageManager != null)
-                m_imageManager.ProcessImageQueue(m_textureSendLimit, m_textureDataLimit);
+                m_imageManager.ProcessImageQueue(m_textureSendLimit);
         }
 
         void ProcessPrimFullUpdates(object sender, ElapsedEventArgs e)

OpenSim/Region/ClientStack/LindenUDP/LLImageManager.cs

@@ -162,32 +162,38 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             }
         }
 
-        public bool ProcessImageQueue(int count, int maxpack)
+        public bool ProcessImageQueue(int packetsToSend)
         {
-            J2KImage imagereq;
-            int numCollected = 0;
-
             m_lastloopprocessed = DateTime.Now.Ticks;
+            int packetsSent = 0;
 
-            // This can happen during Close()
+            while (packetsSent < packetsToSend)
-            if (m_client == null)
-                return false;
-
-            while ((imagereq = GetHighestPriorityImage()) != null)
             {
-                if (imagereq.IsDecoded == true)
+                J2KImage image = GetHighestPriorityImage();
+
+                // If null was returned, the texture priority queue is currently empty
+                if (image == null)
+                    return false;
+
+                if (image.IsDecoded)
                 {
-                    ++numCollected;
+                    int sent;
+                    bool imageDone = image.SendPackets(m_client, packetsToSend - packetsSent, out sent);
 
-                    if (imagereq.SendPackets(m_client, maxpack))
+                    packetsSent += sent;
-                    {
+
-                        // Send complete. Destroy any knowledge of this transfer
+                    // If the send is complete, destroy any knowledge of this transfer
-                        RemoveImageFromQueue(imagereq);
+                    if (imageDone)
-                    }
+                        RemoveImageFromQueue(image);
+                }
+                else
+                {
+                    // TODO: This is a limitation of how LLImageManager is currently
+                    // written. Undecoded textures should not be going into the priority
+                    // queue, because a high priority undecoded texture will clog up the
+                    // pipeline for a client
+                    return true;
                 }
-
-                if (numCollected == count)
-                    break;
             }
 
             return m_priorityQueue.Count > 0;
@@ -199,10 +205,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public void Close()
         {
             m_shuttingdown = true;
-            m_priorityQueue = null;
-            m_j2kDecodeModule = null;
-            m_assetCache = null;
-            m_client = null;
         }
 
         #region Priority Queue Helpers

OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
+using log4net;
 using OpenSim.Framework;
 using OpenMetaverse;
 
@@ -59,6 +60,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
     /// </summary>
     public sealed class LLUDPClient
     {
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
         // FIXME: Make this a config setting
         /// <summary>Percentage of the task throttle category that is allocated to avatar and prim
         /// state updates</summary>
@@ -136,9 +139,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         /// <summary>A container that can hold one packet for each outbox, used to store
         /// dequeued packets that are being held for throttling</summary>
         private readonly OutgoingPacket[] nextPackets = new OutgoingPacket[THROTTLE_CATEGORY_COUNT];
-        /// <summary>An optimization to store the length of dequeued packets being held
+        /// <summary>Flags to prevent queue empty callbacks from stacking up on
-        /// for throttling. This avoids expensive calls to Packet.Length</summary>
+        /// top of each other</summary>
-        private readonly int[] nextPacketLengths = new int[THROTTLE_CATEGORY_COUNT];
+        private readonly bool[] onQueueEmptyRunning = new bool[THROTTLE_CATEGORY_COUNT];
         /// <summary>A reference to the LLUDPServer that is managing this client</summary>
         private readonly LLUDPServer udpServer;
 
@@ -163,7 +166,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
                 packetOutboxes[i] = new OpenSim.Framework.LocklessQueue<OutgoingPacket>();
 
-            throttle = new TokenBucket(parentThrottle, 0, 0);
+            throttle = new TokenBucket(parentThrottle, rates.TotalLimit, rates.Total);
             throttleCategories = new TokenBucket[THROTTLE_CATEGORY_COUNT];
             throttleCategories[(int)ThrottleOutPacketType.Resend] = new TokenBucket(throttle, rates.ResendLimit, rates.Resend);
             throttleCategories[(int)ThrottleOutPacketType.Land] = new TokenBucket(throttle, rates.LandLimit, rates.Land);
@@ -401,10 +404,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                     // This bucket was empty the last time we tried to send a packet,
                     // leaving a dequeued packet still waiting to be sent out. Try to
                     // send it again
-                    if (bucket.RemoveTokens(nextPacketLengths[i]))
+                    OutgoingPacket nextPacket = nextPackets[i];
+                    if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
                     {
                         // Send the packet
-                        udpServer.SendPacketFinal(nextPackets[i]);
+                        udpServer.SendPacketFinal(nextPacket);
                         nextPackets[i] = null;
                         packetSent = true;
                     }
@@ -426,23 +430,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                         }
                         else
                         {
-                            // Save the dequeued packet and the length calculation for
+                            // Save the dequeued packet for the next iteration
-                            // the next iteration
                             nextPackets[i] = packet;
-                            nextPacketLengths[i] = packet.Buffer.DataLength;
                         }
 
                         // If the queue is empty after this dequeue, fire the queue
                         // empty callback now so it has a chance to fill before we 
                         // get back here
                         if (queue.Count == 0)
-                            FireQueueEmpty(i);
+                            BeginFireQueueEmpty(i);
                     }
                     else
                     {
                         // No packets in this queue. Fire the queue empty callback
                         // if it has not been called recently
-                        FireQueueEmpty(i);
+                        BeginFireQueueEmpty(i);
                     }
                 }
             }
@@ -450,6 +452,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             return packetSent;
         }
 
+        /// <summary>
+        /// Called when an ACK packet is received and a round-trip time for a
+        /// packet is calculated. This is used to calculate the smoothed
+        /// round-trip time, round trip time variance, and finally the
+        /// retransmission timeout
+        /// </summary>
+        /// <param name="r">Round-trip time of a single packet and its
+        /// acknowledgement</param>
         public void UpdateRoundTrip(float r)
         {
             const float ALPHA = 0.125f;
@@ -475,11 +485,40 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             //    RTTVAR + " based on new RTT of " + r + "ms");
         }
 
-        private void FireQueueEmpty(int queueIndex)
+        /// <summary>
+        /// Does an early check to see if this queue empty callback is already
+        /// running, then asynchronously firing the event
+        /// </summary>
+        /// <param name="throttleIndex">Throttle category to fire the callback
+        /// for</param>
+        private void BeginFireQueueEmpty(int throttleIndex)
         {
+            if (!onQueueEmptyRunning[throttleIndex])
+                Util.FireAndForget(FireQueueEmpty, throttleIndex);
+        }
+
+        /// <summary>
+        /// Checks to see if this queue empty callback is already running,
+        /// then firing the event
+        /// </summary>
+        /// <param name="o">Throttle category to fire the callback for, stored
+        /// as an object to match the WaitCallback delegate signature</param>
+        private void FireQueueEmpty(object o)
+        {
+            int i = (int)o;
+            ThrottleOutPacketType type = (ThrottleOutPacketType)i;
             QueueEmpty callback = OnQueueEmpty;
+
             if (callback != null)
-                Util.FireAndForget(delegate(object o) { callback((ThrottleOutPacketType)(int)o); }, queueIndex);
+            {
+                if (!onQueueEmptyRunning[i])
+                {
+                    onQueueEmptyRunning[i] = true;
+                    try { callback(type); }
+                    catch (Exception e) { m_log.Error("[LLUDPCLIENT]: OnQueueEmpty(" + type + ") threw an exception: " + e.Message, e); }
+                    onQueueEmptyRunning[i] = false;
+                }
+            }
         }
     }
 }

OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs

@@ -109,6 +109,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         private Location m_location;
         /// <summary>The measured resolution of Environment.TickCount</summary>
         private float m_tickCountResolution;
+        /// <summary>The size of the receive buffer for the UDP socket. This value
+        /// is passed up to the operating system and used in the system networking
+        /// stack. Use zero to leave this value as the default</summary>
+        private int m_recvBufferSize;
 
         /// <summary>The measured resolution of Environment.TickCount</summary>
         public float TickCountResolution { get { return m_tickCountResolution; } }
@@ -135,6 +139,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
 
             m_circuitManager = circuitManager;
 
+            IConfig config = configSource.Configs["ClientStack.LindenUDP"];
+            if (config != null)
+            {
+                m_recvBufferSize = config.GetInt("client_socket_rcvbuf_size", 0);
+            }
+
             // TODO: Config support for throttling the entire connection
             m_throttle = new TokenBucket(null, 0, 0);
             m_throttleRates = new ThrottleRates(configSource);
@@ -145,7 +155,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             if (m_scene == null)
                 throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
 
-            base.Start();
+            base.Start(m_recvBufferSize);
 
             // Start the incoming packet processing thread
             Thread incomingThread = new Thread(IncomingPacketHandler);

OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs

@@ -73,6 +73,7 @@ namespace OpenMetaverse
         /// </summary>
         /// <param name="bindAddress">Local IP address to bind the server to</param>
         /// <param name="port">Port to listening for incoming UDP packets on</param>
+        /// 
         public OpenSimUDPBase(IPAddress bindAddress, int port)
         {
             m_localBindAddress = bindAddress;
@@ -82,25 +83,31 @@ namespace OpenMetaverse
         /// <summary>
         /// Start the UDP server
         /// </summary>
+        /// <param name="recvBufferSize">The size of the receive buffer for 
+        /// the UDP socket. This value is passed up to the operating system 
+        /// and used in the system networking stack. Use zero to leave this
+        /// value as the default</param>
         /// <remarks>This method will attempt to set the SIO_UDP_CONNRESET flag
         /// on the socket to get newer versions of Windows to behave in a sane
         /// manner (not throwing an exception when the remote side resets the
         /// connection). This call is ignored on Mono where the flag is not
         /// necessary</remarks>
-        public void Start()
+        public void Start(int recvBufferSize)
         {
             if (m_shutdownFlag)
             {
                 const int SIO_UDP_CONNRESET = -1744830452;
 
                 IPEndPoint ipep = new IPEndPoint(m_localBindAddress, m_udpPort);
+
                 m_udpSocket = new Socket(
                     AddressFamily.InterNetwork,
                     SocketType.Dgram,
                     ProtocolType.Udp);
+
                 try
                 {
-                    // this udp socket flag is not supported under mono, 
+                    // This udp socket flag is not supported under mono, 
                     // so we'll catch the exception and continue
                     m_udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
                     m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag set");
@@ -109,6 +116,10 @@ namespace OpenMetaverse
                 {
                     m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag not supported on this platform, ignoring");
                 }
+
+                if (recvBufferSize != 0)
+                    m_udpSocket.ReceiveBufferSize = recvBufferSize;
+
                 m_udpSocket.Bind(ipep);
 
                 // we're not shutting down, we're starting up

OpenSim/Region/ClientStack/LindenUDP/ThrottleRates.cs

@@ -51,6 +51,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public int Texture;
         /// <summary>Drip rate for asset packets</summary>
         public int Asset;
+        /// <summary>Drip rate for the parent token bucket</summary>
+        public int Total;
 
         /// <summary>Maximum burst rate for resent packets</summary>
         public int ResendLimit;
@@ -66,6 +68,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
         public int TextureLimit;
         /// <summary>Maximum burst rate for asset packets</summary>
         public int AssetLimit;
+        /// <summary>Burst rate for the parent token bucket</summary>
+        public int TotalLimit;
 
         /// <summary>
         /// Default constructor
@@ -77,21 +81,25 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             {
                 IConfig throttleConfig = config.Configs["ClientStack.LindenUDP"];
 
-                Resend = throttleConfig.GetInt("ResendDefault", 12500);
+                Resend = throttleConfig.GetInt("resend_default", 12500);
-                Land = throttleConfig.GetInt("LandDefault", 500);
+                Land = throttleConfig.GetInt("land_default", 500);
-                Wind = throttleConfig.GetInt("WindDefault", 500);
+                Wind = throttleConfig.GetInt("wind_default", 500);
-                Cloud = throttleConfig.GetInt("CloudDefault", 500);
+                Cloud = throttleConfig.GetInt("cloud_default", 500);
-                Task = throttleConfig.GetInt("TaskDefault", 500);
+                Task = throttleConfig.GetInt("task_default", 500);
-                Texture = throttleConfig.GetInt("TextureDefault", 500);
+                Texture = throttleConfig.GetInt("texture_default", 500);
-                Asset = throttleConfig.GetInt("AssetDefault", 500);
+                Asset = throttleConfig.GetInt("asset_default", 500);
 
-                ResendLimit = throttleConfig.GetInt("ResendLimit", 18750);
+                Total = throttleConfig.GetInt("client_throttle_max_bps", 0);
-                LandLimit = throttleConfig.GetInt("LandLimit", 29750);
+
-                WindLimit = throttleConfig.GetInt("WindLimit", 18750);
+                ResendLimit = throttleConfig.GetInt("resend_limit", 18750);
-                CloudLimit = throttleConfig.GetInt("CloudLimit", 18750);
+                LandLimit = throttleConfig.GetInt("land_limit", 29750);
-                TaskLimit = throttleConfig.GetInt("TaskLimit", 55750);
+                WindLimit = throttleConfig.GetInt("wind_limit", 18750);
-                TextureLimit = throttleConfig.GetInt("TextureLimit", 55750);
+                CloudLimit = throttleConfig.GetInt("cloud_limit", 18750);
-                AssetLimit = throttleConfig.GetInt("AssetLimit", 27500);
+                TaskLimit = throttleConfig.GetInt("task_limit", 55750);
+                TextureLimit = throttleConfig.GetInt("texture_limit", 55750);
+                AssetLimit = throttleConfig.GetInt("asset_limit", 27500);
+
+                TotalLimit = throttleConfig.GetInt("client_throttle_max_bps", 0);
             }
             catch (Exception) { }
         }

OpenSim/Services/AssetService/AssetService.cs

@@ -64,12 +64,17 @@ namespace OpenSim.Services.AssetService
                 string loaderArgs = assetConfig.GetString("AssetLoaderArgs",
                         String.Empty);
 
-                m_log.InfoFormat("[ASSET]: Loading default asset set from {0}", loaderArgs);
+                bool assetLoaderEnabled = assetConfig.GetBoolean("AssetLoaderEnabled", true);
-                m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs,
+
-                        delegate(AssetBase a)
+                if (assetLoaderEnabled)
-                        {
+                {
-                            Store(a);
+                    m_log.InfoFormat("[ASSET]: Loading default asset set from {0}", loaderArgs);
-                        });
+                    m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs,
+                            delegate(AssetBase a)
+                            {
+                                Store(a);
+                            });
+                }
                 
                 m_log.Info("[ASSET CONNECTOR]: Local asset service enabled");
             }

bin/OpenSim.ini.example

@@ -341,25 +341,6 @@
 
 
 [ClientStack.LindenUDP]
-    ; This is the multiplier applied to all client throttles for outgoing UDP network data
-    ; If it is set to 1, then we obey the throttle settings as given to us by the client.  If it is set to 3, for example, then we
-    ; multiply that setting by 3 (e.g. if the client gives us a setting of 250 kilobits per second then we
-    ; will actually push down data at a maximum rate of 750 kilobits per second).
-    ;
-    ; In principle, setting a multiplier greater than 1 will allow data to be pushed down to a client much faster
-    ; than its UI allows the setting to go.  This may be okay in some situations, such as standalone OpenSim
-    ; applications on a LAN.  However, the greater the multipler, the higher the risk of packet drop, resulting
-    ; in symptoms such as missing terrain or objects.  A much better solution is to change the client UI to allow
-    ; higher network bandwidth settings directly, though this isn't always possible.
-    ;
-    ; Currently this setting is 2 by default because we currently send much more texture data than is strictly
-    ; necessary.  A setting of 1 could result in slow texture transfer.  This will be fixed when the transfer
-    ; of textures at different levels of quality is improved.
-    ;
-    ; Pre r7113, this setting was not exposed but was effectively 8.  You may want to try this if you encounter
-    ; unexpected difficulties
-    client_throttle_multiplier = 2;
-
     ; the client socket receive buffer size determines how many
     ; incoming requests we can process; the default on .NET is 8192
     ; which is about 2 4k-sized UDP datagrams. On mono this is
@@ -374,12 +355,39 @@
     ; by the system's settings for the maximum client receive buffer
     ; size (on linux systems you can set that with "sysctl -w
     ; net.core.rmem_max=X")
-    ;
+    ;client_socket_rcvbuf_size = 8388608
-    ; client_socket_rcvbuf_size = 8388608
+    
+    ; Maximum outbound bits per second for a single scene. This can be used to
+    ; throttle total outbound UDP traffic for a simulator. The default value is
+    ; 0, meaning no throttling at the scene level. The example given here is
+    ; 20 megabits
+    ;scene_throttle_max_bps = 20971520
 
-    ; Maximum bits per second to send to any single client. This will override the user's viewer preference settings.
+    ; Maximum bits per second to send to any single client. This will override 
-
+    ; the user's viewer preference settings. The default value is 0, meaning no
-    ; client_throttle_max_bps = 1500000
+    ; aggregate throttling on clients (only per-category throttling). The
+    ; example given here is 1.5 megabits
+    ;client_throttle_max_bps = 1572864
+    
+    ; Per-client bits per second rates for the various throttle categories.
+    ; These are default values that will be overriden by clients
+    ;resend_default = 12500
+    ;land_default = 500
+    ;wind_default = 500
+    ;cloud_default = 50
+    ;task_default = 500
+    ;texture_default = 500
+    ;asset_default = 500
+    
+    ; Per-client maximum burst rates in bits per second for the various throttle
+    ; categories. These are default values that will be overriden by clients
+    ;resend_limit = 18750
+    ;land_limit = 29750
+    ;wind_limit = 18750
+    ;cloud_limit = 18750
+    ;task_limit = 55750
+    ;texture_limit = 55750
+    ;asset_limit = 27500
 
 [Chat]
     ; Controls whether the chat module is enabled.  Default is true.
@@ -1381,6 +1389,10 @@
 [AssetService]
     DefaultAssetLoader = "OpenSim.Framework.AssetLoader.Filesystem.dll"
     AssetLoaderArgs = "assets/AssetSets.xml"
+    
+    ; Disable this to prevent the default asset set from being inserted into the
+    ; asset store each time the region starts
+    AssetLoaderEnabled = true
 
 [GridService]
     ;; default standalone, overridable in StandaloneCommon.ini