diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs index 8f14806206..de9185651a 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPClient.cs @@ -229,7 +229,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP m_throttleClient = new AdaptiveTokenBucket( string.Format("adaptive throttle for {0} in {1}", AgentID, server.Scene.Name), - parentThrottle, 0, rates.Total, rates.AdaptiveThrottlesEnabled); + parentThrottle, 0, rates.Total, rates.MinimumAdaptiveThrottleRate, rates.AdaptiveThrottlesEnabled); // Create an array of token buckets for this clients different throttle categories m_throttleCategories = new TokenBucket[THROTTLE_CATEGORY_COUNT]; diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs index 94300f846b..9bee3ad20d 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/OpenSimUDPBase.cs @@ -107,6 +107,62 @@ namespace OpenMetaverse /// public float AverageReceiveTicksForLastSamplePeriod { get; private set; } + #region PacketDropDebugging + /// + /// For debugging purposes only... random number generator for dropping + /// outbound packets. + /// + private Random m_dropRandomGenerator; + + /// + /// For debugging purposes only... parameters for a simplified + /// model of packet loss with bursts, overall drop rate should + /// be roughly 1 - m_dropLengthProbability / (m_dropProbabiliy + m_dropLengthProbability) + /// which is about 1% for parameters 0.0015 and 0.15 + /// + private double m_dropProbability = 0.0030; + private double m_dropLengthProbability = 0.15; + private bool m_dropState = false; + + /// + /// For debugging purposes only... parameters to control the time + /// duration over which packet loss bursts can occur, if no packets + /// have been sent for m_dropResetTicks milliseconds, then reset the + /// state of the packet dropper to its default. + /// + private int m_dropLastTick = 0; + private int m_dropResetTicks = 500; + + /// + /// Debugging code used to simulate dropped packets with bursts + /// + private bool DropOutgoingPacket() + { + double rnum = m_dropRandomGenerator.NextDouble(); + + // if the connection has been idle for awhile (more than m_dropResetTicks) then + // reset the state to the default state, don't continue a burst + int curtick = Util.EnvironmentTickCount(); + if (Util.EnvironmentTickCountSubtract(curtick, m_dropLastTick) > m_dropResetTicks) + m_dropState = false; + + m_dropLastTick = curtick; + + // if we are dropping packets, then the probability of dropping + // this packet is the probability that we stay in the burst + if (m_dropState) + { + m_dropState = (rnum < (1.0 - m_dropLengthProbability)) ? true : false; + } + else + { + m_dropState = (rnum < m_dropProbability) ? true : false; + } + + return m_dropState; + } + #endregion PacketDropDebugging + /// /// Default constructor /// @@ -117,6 +173,10 @@ namespace OpenMetaverse { m_localBindAddress = bindAddress; m_udpPort = port; + + // for debugging purposes only, initializes the random number generator + // used for simulating packet loss + // m_dropRandomGenerator = new Random(); } /// @@ -395,6 +455,12 @@ namespace OpenMetaverse { // if (IsRunningOutbound) // { + + // This is strictly for debugging purposes to simulate dropped + // packets when testing throttles & retransmission code + // if (DropOutgoingPacket()) + // return; + try { m_udpSocket.BeginSendTo( diff --git a/OpenSim/Region/ClientStack/Linden/UDP/Tests/ThrottleTests.cs b/OpenSim/Region/ClientStack/Linden/UDP/Tests/ThrottleTests.cs index 0560b9bc32..3c82a7882f 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/Tests/ThrottleTests.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/Tests/ThrottleTests.cs @@ -141,7 +141,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP.Tests udpServer.Throttle.DebugLevel = 1; udpClient.ThrottleDebugLevel = 1; - + int resendBytes = 1000; int landBytes = 2000; int windBytes = 3000; @@ -173,6 +173,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP.Tests IniConfigSource ics = new IniConfigSource(); IConfig config = ics.AddConfig("ClientStack.LindenUDP"); config.Set("enable_adaptive_throttles", true); + config.Set("adaptive_throttle_min_bps", 32000); + TestLLUDPServer udpServer = ClientStackHelpers.AddUdpServer(scene, ics); ScenePresence sp @@ -184,8 +186,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP.Tests udpServer.Throttle.DebugLevel = 1; udpClient.ThrottleDebugLevel = 1; - // Total is 280000 - int resendBytes = 10000; + // Total is 275000 + int resendBytes = 5000; // this is set low to test the minimum throttle override int landBytes = 20000; int windBytes = 30000; int cloudBytes = 40000; @@ -197,28 +199,28 @@ namespace OpenSim.Region.ClientStack.LindenUDP.Tests SetThrottles( udpClient, resendBytes, landBytes, windBytes, cloudBytes, taskBytes, textureBytes, assetBytes); - // Ratio of current adaptive drip rate to requested bytes - // XXX: Should hard code this as below so we don't rely on values given by tested code to construct - // expected values. - double commitRatio = (double)udpClient.FlowThrottle.AdjustedDripRate / udpClient.FlowThrottle.TargetDripRate; + // Ratio of current adaptive drip rate to requested bytes, minimum rate is 32000 + double commitRatio = 32000.0 / totalBytes; AssertThrottles( udpClient, LLUDPServer.MTU, landBytes * commitRatio, windBytes * commitRatio, cloudBytes * commitRatio, taskBytes * commitRatio, textureBytes * commitRatio, assetBytes * commitRatio, udpClient.FlowThrottle.AdjustedDripRate, totalBytes, 0); - // Test an increase in target throttle - udpClient.FlowThrottle.AcknowledgePackets(35000); - commitRatio = 0.2; + // Test an increase in target throttle, ack of 20 packets adds 20 * LLUDPServer.MTU bytes + // to the throttle, recompute commitratio from those numbers + udpClient.FlowThrottle.AcknowledgePackets(20); + commitRatio = (32000.0 + 20.0 * LLUDPServer.MTU) / totalBytes; AssertThrottles( udpClient, - resendBytes * commitRatio, landBytes * commitRatio, windBytes * commitRatio, cloudBytes * commitRatio, taskBytes * commitRatio, + LLUDPServer.MTU, landBytes * commitRatio, windBytes * commitRatio, cloudBytes * commitRatio, taskBytes * commitRatio, textureBytes * commitRatio, assetBytes * commitRatio, udpClient.FlowThrottle.AdjustedDripRate, totalBytes, 0); - // Test a decrease in target throttle + // Test a decrease in target throttle, adaptive throttle should cut the rate by 50% with a floor + // set by the minimum adaptive rate udpClient.FlowThrottle.ExpirePackets(1); - commitRatio = 0.1; + commitRatio = (32000.0 + (20.0 * LLUDPServer.MTU)/Math.Pow(2,1)) / totalBytes; AssertThrottles( udpClient, diff --git a/OpenSim/Region/ClientStack/Linden/UDP/ThrottleRates.cs b/OpenSim/Region/ClientStack/Linden/UDP/ThrottleRates.cs index dd15cc7d4f..7a2756bab1 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/ThrottleRates.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/ThrottleRates.cs @@ -58,7 +58,14 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// Flag used to enable adaptive throttles public bool AdaptiveThrottlesEnabled; - + + /// + /// Set the minimum rate that the adaptive throttles can set. The viewer + /// can still throttle lower than this, but the adaptive throttles will + /// never decrease rates below this no matter how many packets are dropped + /// + public Int64 MinimumAdaptiveThrottleRate; + /// Amount of the texture throttle to steal for the task throttle public double CannibalizeTextureRate; @@ -84,7 +91,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP Total = throttleConfig.GetInt("client_throttle_max_bps", 0); AdaptiveThrottlesEnabled = throttleConfig.GetBoolean("enable_adaptive_throttles", false); - + MinimumAdaptiveThrottleRate = throttleConfig.GetInt("adaptive_throttle_min_bps", 32000); + CannibalizeTextureRate = (double)throttleConfig.GetFloat("CannibalizeTextureRate", 0.0f); CannibalizeTextureRate = Util.Clamp(CannibalizeTextureRate,0.0, 0.9); } diff --git a/OpenSim/Region/ClientStack/Linden/UDP/TokenBucket.cs b/OpenSim/Region/ClientStack/Linden/UDP/TokenBucket.cs index 4ca83ff207..4616203181 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/TokenBucket.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/TokenBucket.cs @@ -61,7 +61,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP /// /// - protected const Int32 m_minimumDripRate = 1400; + protected const Int32 m_minimumDripRate = LLUDPServer.MTU; /// Time of the last drip, in system ticks protected Int32 m_lastDrip; @@ -392,13 +392,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP } /// - /// The minimum rate for flow control. Minimum drip rate is one - /// packet per second. Open the throttle to 15 packets per second - /// or about 160kbps. + /// The minimum rate for adaptive flow control. /// - protected const Int64 m_minimumFlow = m_minimumDripRate * 15; + protected Int64 m_minimumFlow = 32000; - public AdaptiveTokenBucket(string identifier, TokenBucket parent, Int64 requestedDripRate, Int64 maxDripRate, bool enabled) + /// + /// Constructor for the AdaptiveTokenBucket class + /// Unique identifier for the client + /// Parent bucket in the hierarchy + /// + /// The ceiling rate for adaptation + /// The floor rate for adaptation + /// + public AdaptiveTokenBucket(string identifier, TokenBucket parent, Int64 requestedDripRate, Int64 maxDripRate, Int64 minDripRate, bool enabled) : base(identifier, parent, requestedDripRate, maxDripRate) { AdaptiveEnabled = enabled; @@ -406,34 +412,53 @@ namespace OpenSim.Region.ClientStack.LindenUDP if (AdaptiveEnabled) { // m_log.DebugFormat("[TOKENBUCKET]: Adaptive throttle enabled"); + m_minimumFlow = minDripRate; TargetDripRate = m_minimumFlow; AdjustedDripRate = m_minimumFlow; } } - // - // Reliable packets sent to the client for which we never received an ack adjust the drip rate down. - // - public void ExpirePackets(Int32 count) + /// + /// Reliable packets sent to the client for which we never received an ack adjust the drip rate down. + /// Number of packets that expired without successful delivery + /// + public void ExpirePackets(Int32 packets) { if (AdaptiveEnabled) { if (DebugLevel > 0) m_log.WarnFormat( "[ADAPTIVEBUCKET] drop {0} by {1} expired packets for {2}", - AdjustedDripRate, count, Identifier); + AdjustedDripRate, packets, Identifier); - AdjustedDripRate = (Int64) (AdjustedDripRate / Math.Pow(2,count)); + // AdjustedDripRate = (Int64) (AdjustedDripRate / Math.Pow(2,packets)); + + // Compute the fallback solely on the rate allocated beyond the minimum, this + // should smooth out the fallback to the minimum rate + AdjustedDripRate = m_minimumFlow + (Int64) ((AdjustedDripRate - m_minimumFlow) / Math.Pow(2, packets)); } } - // - // Reliable packets acked by the client adjust the drip rate up. - // - public void AcknowledgePackets(Int32 count) + /// + /// Reliable packets acked by the client adjust the drip rate up. + /// Number of packets successfully acknowledged + /// + public void AcknowledgePackets(Int32 packets) { if (AdaptiveEnabled) - AdjustedDripRate = AdjustedDripRate + count; + AdjustedDripRate = AdjustedDripRate + packets * LLUDPServer.MTU; + } + + /// + /// Adjust the minimum flow level for the adaptive throttle, this will drop adjusted + /// throttles back to the minimum levels + /// minDripRate--the new minimum flow + /// + public void ResetMinimumAdaptiveFlow(Int64 minDripRate) + { + m_minimumFlow = minDripRate; + TargetDripRate = m_minimumFlow; + AdjustedDripRate = m_minimumFlow; } } } \ No newline at end of file diff --git a/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs b/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs index 9d6c09e612..b546a99603 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/UnackedPacketCollection.cs @@ -31,6 +31,9 @@ using System.Net; using System.Threading; using OpenMetaverse; +//using System.Reflection; +//using log4net; + namespace OpenSim.Region.ClientStack.LindenUDP { /// @@ -60,6 +63,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } + //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + /// Holds the actual unacked packet data, sorted by sequence number private Dictionary m_packets = new Dictionary(); /// Holds packets that need to be added to the unacknowledged list @@ -164,8 +169,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP } } - //if (expiredPackets != null) - // m_log.DebugFormat("[UNACKED PACKET COLLECTION]: Found {0} expired packets on timeout of {1}", expiredPackets.Count, timeoutMS); + // if (expiredPackets != null) + // m_log.DebugFormat("[UNACKED PACKET COLLECTION]: Found {0} expired packets on timeout of {1}", expiredPackets.Count, timeoutMS); return expiredPackets; } @@ -192,7 +197,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP // As with other network applications, assume that an acknowledged packet is an // indication that the network can handle a little more load, speed up the transmission - ackedPacket.Client.FlowThrottle.AcknowledgePackets(ackedPacket.Buffer.DataLength); + ackedPacket.Client.FlowThrottle.AcknowledgePackets(1); // Update stats Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength); @@ -207,9 +212,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP } else { - //m_log.WarnFormat("[UNACKED PACKET COLLECTION]: Could not find packet with sequence number {0} to ack", pendingAcknowledgement.SequenceNumber); + // m_log.WarnFormat("[UNACKED PACKET COLLECTION]: found null packet for sequence number {0} to ack", + // pendingAcknowledgement.SequenceNumber); } } + else + { + // m_log.WarnFormat("[UNACKED PACKET COLLECTION]: Could not find packet with sequence number {0} to ack", + // pendingAcknowledgement.SequenceNumber); + } } uint pendingRemove; diff --git a/bin/OpenSimDefaults.ini b/bin/OpenSimDefaults.ini index 3e9514ebbf..212baabc7f 100644 --- a/bin/OpenSimDefaults.ini +++ b/bin/OpenSimDefaults.ini @@ -544,6 +544,13 @@ ; ;client_throttle_max_bps = 187500 + ; Minimum bytes per second to send to any single client as a result of + ; adaptive throttling. Viewer preferences set to a lower number will + ; override the settin. The example given here ensures that adaptive + ; throttling will never decrease per client bandwidth below 256 kbps. + ; + ;adaptive_throttle_min_bps = 32000 + ; Adaptive throttling attempts to limit network overload when multiple ; clients login by starting each connection more slowly. Disabled by ; default