OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* Replaced the UnackedPacketCollection with a lockless implementation. The tiny amount of time spent in the locks turned into a lot of time when the rest of the LLUDP implementation went lockless 

* Changed the timer tracking numbers for each client to not have "memory". It will no longer queue up calls to functions like ResendUnacked
* Reverted Jim's WaitHandle code. Although it was technically more correct, it exhibited the exact same behavior as the old code but spent more cycles. The 20ms has been replaced with the minimum amount of time before a token bucket could receive a drip, and an else { sleep(0); } was added to make sure the outgoing packet handler always yields at least a minimum amount
prioritization
John Hurliman 2009-10-21 11:59:48 -07:00
parent c0c845aea4
commit 9178537e94
4 changed files with 128 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
OpenSim/Region/ClientStack/LindenUDP/LLClientView.cs
View File

@ -1228,10 +1228,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
StartPingCheckPacket pc = (StartPingCheckPacket)PacketPool.Instance.GetPacket(PacketType.StartPingCheck);
pc.Header.Reliable = false;
OutgoingPacket oldestPacket = m_udpClient.NeedAcks.GetOldest();
pc.PingID.PingID = seq;
pc.PingID.OldestUnacked = (oldestPacket != null) ? oldestPacket.SequenceNumber : 0;
// We *could* get OldestUnacked, but it would hurt performance and not provide any benefit
pc.PingID.OldestUnacked = 0;
OutPacket(pc, ThrottleOutPacketType.Unknown);
}
@ -3320,8 +3319,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// If we received an update about our own avatar, process the avatar update priority queue immediately
if (data.AgentID == m_agentId)
ProcessAvatarTerseUpdates();
else
m_udpServer.SignalOutgoingPacketHandler();
}
private void ProcessAvatarTerseUpdates()
@ -3409,8 +3406,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
lock (m_primFullUpdates.SyncRoot)
m_primFullUpdates.Enqueue(data.priority, objectData, data.localID);
m_udpServer.SignalOutgoingPacketHandler();
}
void ProcessPrimFullUpdates()
@ -3454,8 +3449,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
lock (m_primTerseUpdates.SyncRoot)
m_primTerseUpdates.Enqueue(data.Priority, objectData, data.LocalID);
m_udpServer.SignalOutgoingPacketHandler();
}
void ProcessPrimTerseUpdates()

33
OpenSim/Region/ClientStack/LindenUDP/LLUDPClient.cs
View File

@ -202,7 +202,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public void Shutdown()
{
IsConnected = false;
NeedAcks.Clear();
for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
{
m_packetOutboxes[i].Clear();
@ -394,7 +393,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
// Not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
m_udpServer.SignalOutgoingPacketHandler();
return true;
}
}
@ -411,15 +409,13 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// </summary>
/// <remarks>This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe</remarks>
/// <returns>The minimum amount of time before the next packet
/// can be sent to this client</returns>
public int DequeueOutgoing()
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing()
{
OutgoingPacket packet;
OpenSim.Framework.LocklessQueue<OutgoingPacket> queue;
TokenBucket bucket;
int dataLength;
int minTimeout = Int32.MaxValue;
bool packetSent = false;
//string queueDebugOutput = String.Empty; // Serious debug business
@ -434,18 +430,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// leaving a dequeued packet still waiting to be sent out. Try to
// send it again
OutgoingPacket nextPacket = m_nextPackets[i];
dataLength = nextPacket.Buffer.DataLength;
if (bucket.RemoveTokens(dataLength))
if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextPackets[i] = null;
minTimeout = 0;
}
else if (minTimeout != 0)
{
// Check the minimum amount of time we would have to wait before this packet can be sent out
minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1);
packetSent = true;
}
}
else
@ -457,23 +447,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
dataLength = packet.Buffer.DataLength;
if (bucket.RemoveTokens(dataLength))
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
minTimeout = 0;
packetSent = true;
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
if (minTimeout != 0)
{
// Check the minimum amount of time we would have to wait before this packet can be sent out
minTimeout = Math.Min(minTimeout, ((dataLength - bucket.Content) / bucket.DripPerMS) + 1);
}
}
// If the queue is empty after this dequeue, fire the queue
@ -492,7 +475,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return minTimeout;
return packetSent;
}
/// <summary>

97
OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
View File

@ -121,12 +121,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
private int m_recvBufferSize;
/// <summary>Flag to process packets asynchronously or synchronously</summary>
private bool m_asyncPacketHandling;
/// <summary>Track the minimum amount of time needed to send the next packet in the
/// OutgoingPacketHandler loop so we know when to sleep</summary>
private int m_minTimeout = Int32.MaxValue;
/// <summary>EventWaitHandle to signify the outgoing packet handler thread that
/// there is more work to do</summary>
private EventWaitHandle m_outgoingWaitHandle;
/// <summary>Tracks whether or not a packet was sent each round so we know
/// whether or not to sleep</summary>
private bool m_packetSent;
public Socket Server { get { return null; } }
@ -174,8 +171,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
base.Start(m_recvBufferSize, m_asyncPacketHandling);
m_outgoingWaitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);
// Start the incoming packet processing thread
Thread incomingThread = new Thread(IncomingPacketHandler);
incomingThread.Name = "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")";
@ -190,8 +185,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
base.Stop();
m_outgoingWaitHandle.Close();
}
public void AddScene(IScene scene)
@ -374,10 +367,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
StartPingCheckPacket pc = (StartPingCheckPacket)PacketPool.Instance.GetPacket(PacketType.StartPingCheck);
pc.Header.Reliable = false;
OutgoingPacket oldestPacket = udpClient.NeedAcks.GetOldest();
pc.PingID.PingID = (byte)udpClient.CurrentPingSequence++;
pc.PingID.OldestUnacked = (oldestPacket != null) ? oldestPacket.SequenceNumber : 0;
// We *could* get OldestUnacked, but it would hurt performance and not provide any benefit
pc.PingID.OldestUnacked = 0;
SendPacket(udpClient, pc, ThrottleOutPacketType.Unknown, false);
}
@ -397,14 +389,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
return;
}
if (udpClient.NeedAcks.Count > 0)
{
// Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
List<OutgoingPacket> expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
if (expiredPackets != null)
{
m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID);
m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO);
// Resend packets
for (int i = 0; i < expiredPackets.Count; i++)
@ -432,7 +422,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
}
}
}
public void Flush(LLUDPClient udpClient)
{
@ -576,12 +565,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// Handle appended ACKs
if (packet.Header.AppendedAcks && packet.Header.AckList != null)
{
lock (udpClient.NeedAcks.SyncRoot)
{
for (int i = 0; i < packet.Header.AckList.Length; i++)
AcknowledgePacket(udpClient, packet.Header.AckList[i], now, packet.Header.Resent);
}
udpClient.NeedAcks.Remove(packet.Header.AckList[i], now, packet.Header.Resent);
}
// Handle PacketAck packets
@ -589,11 +575,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
PacketAckPacket ackPacket = (PacketAckPacket)packet;
lock (udpClient.NeedAcks.SyncRoot)
{
for (int i = 0; i < ackPacket.Packets.Length; i++)
AcknowledgePacket(udpClient, ackPacket.Packets[i].ID, now, packet.Header.Resent);
}
udpClient.NeedAcks.Remove(ackPacket.Packets[i].ID, now, packet.Header.Resent);
// We don't need to do anything else with PacketAck packets
return;
@ -734,21 +717,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
client.Close();
}
private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend)
{
OutgoingPacket ackedPacket;
if (client.NeedAcks.RemoveUnsafe(ack, out ackedPacket) && !fromResend)
{
// Update stats
Interlocked.Add(ref client.UnackedBytes, -ackedPacket.Buffer.DataLength);
// Calculate the round-trip time for this packet and its ACK
int rtt = currentTime - ackedPacket.TickCount;
if (rtt > 0)
client.UpdateRoundTrip(rtt);
}
}
private void IncomingPacketHandler()
{
// Set this culture for the thread that incoming packets are received
@ -775,11 +743,6 @@ namespace OpenSim.Region.ClientStack.LindenUDP
packetInbox.Clear();
}
public bool SignalOutgoingPacketHandler()
{
return m_outgoingWaitHandle.Set();
}
private void OutgoingPacketHandler()
{
// Set this culture for the thread that outgoing packets are sent
@ -790,28 +753,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
try
{
m_minTimeout = Int32.MaxValue;
m_packetSent = false;
// Handle outgoing packets, resends, acknowledgements, and pings for each
// client. m_minTimeout will be set to 0 if more packets are waiting in the
// queues with bandwidth to spare, or the number of milliseconds we need to
// wait before at least one packet can be sent to a client
// client. m_packetSent will be set to true if a packet is sent
m_scene.ClientManager.ForEachSync(ClientOutgoingPacketHandler);
// Can't wait for a negative amount of time, and put a 100ms ceiling on our
// maximum wait time
m_minTimeout = Utils.Clamp(m_minTimeout, 0, 100);
if (m_minTimeout > 0)
{
// Don't bother waiting for a shorter interval than our TickCountResolution
// since the token buckets wouldn't update anyways
m_minTimeout = Math.Max(m_minTimeout, (int)TickCountResolution);
// Wait for someone to signal that packets are ready to be sent, or for our
// sleep interval to expire
m_outgoingWaitHandle.WaitOne(m_minTimeout);
}
// If a packet was sent, only do a minimum length context switch to allow
// other parts of the code to do work. If nothing was sent, sleep for the
// minimum amount of time before a token bucket could get more tokens
if (m_packetSent)
Thread.Sleep(0);
else
Thread.Sleep((int)TickCountResolution);
}
catch (Exception ex)
{
@ -841,7 +795,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.ElapsedMSOutgoingPacketHandler >= 100)
{
ResendUnacked(udpClient);
udpClient.ElapsedMSOutgoingPacketHandler -= 100;
udpClient.ElapsedMSOutgoingPacketHandler = 0;
udpClient.Elapsed100MSOutgoingPacketHandler += 1;
}
@ -849,7 +803,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.Elapsed100MSOutgoingPacketHandler >= 5)
{
SendAcks(udpClient);
udpClient.Elapsed100MSOutgoingPacketHandler -= 5;
udpClient.Elapsed100MSOutgoingPacketHandler = 0;
udpClient.Elapsed500MSOutgoingPacketHandler += 1;
}
@ -857,19 +811,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (udpClient.Elapsed500MSOutgoingPacketHandler >= 10)
{
SendPing(udpClient);
udpClient.Elapsed500MSOutgoingPacketHandler -= 10;
udpClient.Elapsed500MSOutgoingPacketHandler = 0;
}
// Dequeue any outgoing packets that are within the throttle limits
// and get the minimum time we would have to sleep before this client
// could send a packet out
int minTimeoutThisLoop = udpClient.DequeueOutgoing();
// Although this is not thread safe, it is cheaper than locking and the
// worst that will happen is we sleep for slightly longer than the
// minimum necessary interval
if (minTimeoutThisLoop < m_minTimeout)
m_minTimeout = minTimeoutThisLoop;
if (udpClient.DequeueOutgoing())
m_packetSent = true;
}
udpClient.TickLastOutgoingPacketHandler = thisTick;

151
OpenSim/Region/ClientStack/LindenUDP/UnackedPacketCollection.cs
View File

@ -37,22 +37,34 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// </summary>
public sealed class UnackedPacketCollection
{
/// <summary>Synchronization primitive. A lock must be acquired on this
/// object before calling any of the unsafe methods</summary>
public object SyncRoot = new object();
/// <summary>
/// Holds information about a pending acknowledgement
/// </summary>
private struct PendingAck
{
/// <summary>Sequence number of the packet to remove</summary>
public uint SequenceNumber;
/// <summary>Environment.TickCount value when the remove was queued.
/// This is used to update round-trip times for packets</summary>
public int RemoveTime;
/// <summary>Whether or not this acknowledgement was attached to a
/// resent packet. If so, round-trip time will not be calculated</summary>
public bool FromResend;
public PendingAck(uint sequenceNumber, int currentTime, bool fromResend)
{
SequenceNumber = sequenceNumber;
RemoveTime = currentTime;
FromResend = fromResend;
}
}
/// <summary>Holds the actual unacked packet data, sorted by sequence number</summary>
private SortedDictionary<uint, OutgoingPacket> packets = new SortedDictionary<uint, OutgoingPacket>();
/// <summary>Gets the total number of unacked packets</summary>
public int Count { get { return packets.Count; } }
/// <summary>
/// Default constructor
/// </summary>
public UnackedPacketCollection()
{
}
private SortedDictionary<uint, OutgoingPacket> m_packets = new SortedDictionary<uint, OutgoingPacket>();
/// <summary>Holds packets that need to be added to the unacknowledged list</summary>
private LocklessQueue<OutgoingPacket> m_pendingAdds = new LocklessQueue<OutgoingPacket>();
/// <summary>Holds information about pending acknowledgements</summary>
private LocklessQueue<PendingAck> m_pendingRemoves = new LocklessQueue<PendingAck>();
/// <summary>
/// Add an unacked packet to the collection
@ -60,74 +72,22 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
public bool Add(OutgoingPacket packet)
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
lock (SyncRoot)
{
if (!packets.ContainsKey(packet.SequenceNumber))
{
packets.Add(packet.SequenceNumber, packet);
return true;
}
return false;
}
m_pendingAdds.Enqueue(packet);
}
/// <summary>
/// Removes a packet from the collection without attempting to obtain a
/// lock first
/// Marks a packet as acknowledged
/// </summary>
/// <param name="sequenceNumber">Sequence number of the packet to remove</param>
/// <returns>True if the packet was found and removed, otherwise false</returns>
public bool RemoveUnsafe(uint sequenceNumber)
/// <param name="sequenceNumber">Sequence number of the packet to
/// acknowledge</param>
/// <param name="currentTime">Current value of Environment.TickCount</param>
public void Remove(uint sequenceNumber, int currentTime, bool fromResend)
{
return packets.Remove(sequenceNumber);
}
/// <summary>
/// Removes a packet from the collection without attempting to obtain a
/// lock first
/// </summary>
/// <param name="sequenceNumber">Sequence number of the packet to remove</param>
/// <param name="packet">Returns the removed packet</param>
/// <returns>True if the packet was found and removed, otherwise false</returns>
public bool RemoveUnsafe(uint sequenceNumber, out OutgoingPacket packet)
{
if (packets.TryGetValue(sequenceNumber, out packet))
{
packets.Remove(sequenceNumber);
return true;
}
return false;
}
/// <summary>
/// Removes all elements from the collection
/// </summary>
public void Clear()
{
lock (SyncRoot)
packets.Clear();
}
/// <summary>
/// Gets the packet with the lowest sequence number
/// </summary>
/// <returns>The packet with the lowest sequence number, or null if the
/// collection is empty</returns>
public OutgoingPacket GetOldest()
{
lock (SyncRoot)
{
using (SortedDictionary<uint, OutgoingPacket>.ValueCollection.Enumerator e = packets.Values.GetEnumerator())
{
if (e.MoveNext())
return e.Current;
else
return null;
}
}
m_pendingRemoves.Enqueue(new PendingAck(sequenceNumber, currentTime, fromResend));
}
/// <summary>
@ -138,14 +98,19 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// packet is considered expired</param>
/// <returns>A list of all expired packets according to the given
/// expiration timeout</returns>
/// <remarks>This function is not thread safe, and cannot be called
/// multiple times concurrently</remarks>
public List<OutgoingPacket> GetExpiredPackets(int timeoutMS)
{
ProcessQueues();
List<OutgoingPacket> expiredPackets = null;
lock (SyncRoot)
if (m_packets.Count > 0)
{
int now = Environment.TickCount;
foreach (OutgoingPacket packet in packets.Values)
foreach (OutgoingPacket packet in m_packets.Values)
{
// TickCount of zero means a packet is in the resend queue
// but hasn't actually been sent over the wire yet
@ -167,5 +132,35 @@ namespace OpenSim.Region.ClientStack.LindenUDP
return expiredPackets;
}
private void ProcessQueues()
{
// Process all the pending adds
OutgoingPacket pendingAdd;
while (m_pendingAdds.Dequeue(out pendingAdd))
m_packets[pendingAdd.SequenceNumber] = pendingAdd;
// Process all the pending removes, including updating statistics and round-trip times
PendingAck pendingRemove;
OutgoingPacket ackedPacket;
while (m_pendingRemoves.Dequeue(out pendingRemove))
{
if (m_packets.TryGetValue(pendingRemove.SequenceNumber, out ackedPacket))
{
m_packets.Remove(pendingRemove.SequenceNumber);
// Update stats
System.Threading.Interlocked.Add(ref ackedPacket.Client.UnackedBytes, -ackedPacket.Buffer.DataLength);
if (!pendingRemove.FromResend)
{
// Calculate the round-trip time for this packet and its ACK
int rtt = pendingRemove.RemoveTime - ackedPacket.TickCount;
if (rtt > 0)
ackedPacket.Client.UpdateRoundTrip(rtt);
}
}
}
}
}
}