* Added a lock object for the write functions in LLUDPClientCollection (immutable != concurrent write safety)

* Allow the UDP server to bind to a user-specified port again
* Updated to a newer version of OpenSimUDPBase that streamlines the code even more. This also reintroduces the highly concurrent packet handling which needs more testing
prioritization
John Hurliman 2009-10-09 01:53:06 -07:00
parent 56a27c37d3
commit a5b9971fd7
3 changed files with 169 additions and 210 deletions

View File

@ -36,6 +36,9 @@ using ReaderWriterLockImpl = OpenMetaverse.ReaderWriterLockSlim;
namespace OpenSim.Region.ClientStack.LindenUDP namespace OpenSim.Region.ClientStack.LindenUDP
{ {
/// <summary>
/// A thread safe mapping from endpoints to client references
/// </summary>
public sealed class UDPClientCollection public sealed class UDPClientCollection
{ {
#region IComparers #region IComparers
@ -52,43 +55,80 @@ namespace OpenSim.Region.ClientStack.LindenUDP
#endregion IComparers #endregion IComparers
/// <summary>An immutable dictionary mapping from <seealso cref="IPEndPoint"/>
/// to <seealso cref="LLUDPClient"/> references</summary>
private ImmutableMap<IPEndPoint, LLUDPClient> m_dict; private ImmutableMap<IPEndPoint, LLUDPClient> m_dict;
/// <summary>Immutability grants thread safety for concurrent reads and
/// read-writes, but not concurrent writes</summary>
private object m_writeLock;
/// <summary>Number of clients in the collection</summary>
public int Count { get { return m_dict.Count; } }
/// <summary>
/// Default constructor
/// </summary>
public UDPClientCollection() public UDPClientCollection()
{ {
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer()); m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
} }
/// <summary>
/// Add a client reference to the collection
/// </summary>
/// <param name="key">Remote endpoint of the client</param>
/// <param name="value">Reference to the client object</param>
public void Add(IPEndPoint key, LLUDPClient value) public void Add(IPEndPoint key, LLUDPClient value)
{ {
lock (m_writeLock)
m_dict = m_dict.Add(key, value); m_dict = m_dict.Add(key, value);
} }
/// <summary>
/// Remove a client from the collection
/// </summary>
/// <param name="key">Remote endpoint of the client</param>
public void Remove(IPEndPoint key) public void Remove(IPEndPoint key)
{ {
lock (m_writeLock)
m_dict = m_dict.Delete(key); m_dict = m_dict.Delete(key);
} }
/// <summary>
/// Resets the client collection
/// </summary>
public void Clear() public void Clear()
{ {
lock (m_writeLock)
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer()); m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
} }
public int Count /// <summary>
{ /// Checks if an endpoint is in the collection
get { return m_dict.Count; } /// </summary>
} /// <param name="key">Endpoint to check for</param>
/// <returns>True if the endpoint was found in the collection, otherwise false</returns>
public bool ContainsKey(IPEndPoint key) public bool ContainsKey(IPEndPoint key)
{ {
return m_dict.ContainsKey(key); return m_dict.ContainsKey(key);
} }
/// <summary>
/// Attempts to fetch a value out of the collection
/// </summary>
/// <param name="key">Endpoint of the client to retrieve</param>
/// <param name="value">Retrieved client, or null on lookup failure</param>
/// <returns>True if the lookup succeeded, otherwise false</returns>
public bool TryGetValue(IPEndPoint key, out LLUDPClient value) public bool TryGetValue(IPEndPoint key, out LLUDPClient value)
{ {
return m_dict.TryGetValue(key, out value); return m_dict.TryGetValue(key, out value);
} }
/// <summary>
/// Performs a given task in parallel for each of the elements in the
/// collection
/// </summary>
/// <param name="action">Action to perform on each element</param>
public void ForEach(Action<LLUDPClient> action) public void ForEach(Action<LLUDPClient> action)
{ {
Parallel.ForEach<LLUDPClient>(m_dict.Values, action); Parallel.ForEach<LLUDPClient>(m_dict.Values, action);

View File

@ -96,7 +96,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// <summary>Incoming packets that are awaiting handling</summary> /// <summary>Incoming packets that are awaiting handling</summary>
private OpenMetaverse.BlockingQueue<IncomingPacket> packetInbox = new OpenMetaverse.BlockingQueue<IncomingPacket>(); private OpenMetaverse.BlockingQueue<IncomingPacket> packetInbox = new OpenMetaverse.BlockingQueue<IncomingPacket>();
/// <summary></summary> /// <summary></summary>
private UDPClientCollection clients = new UDPClientCollection(); private UDPClientCollection m_clients = new UDPClientCollection();
/// <summary>Bandwidth throttle for this UDP server</summary> /// <summary>Bandwidth throttle for this UDP server</summary>
private TokenBucket m_throttle; private TokenBucket m_throttle;
/// <summary>Bandwidth throttle rates for this UDP server</summary> /// <summary>Bandwidth throttle rates for this UDP server</summary>
@ -115,7 +115,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public Socket Server { get { return null; } } public Socket Server { get { return null; } }
public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager) public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
: base((int)port) : base(listenIP, (int)port)
{ {
#region Environment.TickCount Measurement #region Environment.TickCount Measurement
@ -143,7 +143,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public new void Start() public new void Start()
{ {
if (m_scene == null) if (m_scene == null)
throw new InvalidOperationException("Cannot LLUDPServer.Start() without an IScene reference"); throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
base.Start(); base.Start();
@ -188,7 +188,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
m_scene.ClientManager.Remove(udpClient.CircuitCode); m_scene.ClientManager.Remove(udpClient.CircuitCode);
udpClient.ClientAPI.Close(false); udpClient.ClientAPI.Close(false);
udpClient.Shutdown(); udpClient.Shutdown();
clients.Remove(udpClient.RemoteEndPoint); m_clients.Remove(udpClient.RemoteEndPoint);
} }
public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting) public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting)
@ -208,7 +208,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
for (int i = 0; i < packetCount; i++) for (int i = 0; i < packetCount; i++)
{ {
byte[] data = datas[i]; byte[] data = datas[i];
clients.ForEach( m_clients.ForEach(
delegate(LLUDPClient client) delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); }); { SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
} }
@ -216,7 +216,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
else else
{ {
byte[] data = packet.ToBytes(); byte[] data = packet.ToBytes();
clients.ForEach( m_clients.ForEach(
delegate(LLUDPClient client) delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); }); { SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
} }
@ -502,7 +502,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
} }
// Determine which agent this packet came from // Determine which agent this packet came from
if (!clients.TryGetValue(address, out client)) if (!m_clients.TryGetValue(address, out client))
{ {
m_log.Warn("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address); m_log.Warn("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address);
return; return;
@ -606,7 +606,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (m_scene.RegionStatus != RegionStatus.SlaveScene) if (m_scene.RegionStatus != RegionStatus.SlaveScene)
{ {
AuthenticateResponse sessionInfo; AuthenticateResponse sessionInfo;
bool isNewCircuit = !clients.ContainsKey(remoteEndPoint); bool isNewCircuit = !m_clients.ContainsKey(remoteEndPoint);
if (!IsClientAuthorized(useCircuitCode, out sessionInfo)) if (!IsClientAuthorized(useCircuitCode, out sessionInfo))
{ {
@ -648,7 +648,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
udpClient.ClientAPI = clientApi; udpClient.ClientAPI = clientApi;
// Add the new client to our list of tracked clients // Add the new client to our list of tracked clients
clients.Add(udpClient.RemoteEndPoint, udpClient); m_clients.Add(udpClient.RemoteEndPoint, udpClient);
} }
private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend) private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend)
@ -726,7 +726,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
elapsed500MS = 0; elapsed500MS = 0;
} }
clients.ForEach( m_clients.ForEach(
delegate(LLUDPClient client) delegate(LLUDPClient client)
{ {
if (client.DequeueOutgoing()) if (client.DequeueOutgoing())

View File

@ -29,101 +29,90 @@ using System;
using System.Net; using System.Net;
using System.Net.Sockets; using System.Net.Sockets;
using System.Threading; using System.Threading;
using OpenMetaverse; using log4net;
namespace OpenSim.Region.ClientStack.LindenUDP namespace OpenMetaverse
{ {
/// <summary> /// <summary>
/// /// Base UDP server
/// </summary> /// </summary>
public abstract class OpenSimUDPBase public abstract class OpenSimUDPBase
{ {
// these abstract methods must be implemented in a derived class to actually do private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
// something with the packets that are sent and received.
/// <summary>
/// This method is called when an incoming packet is received
/// </summary>
/// <param name="buffer">Incoming packet buffer</param>
protected abstract void PacketReceived(UDPPacketBuffer buffer); protected abstract void PacketReceived(UDPPacketBuffer buffer);
/// <summary>
/// This method is called when an outgoing packet is sent
/// </summary>
/// <param name="buffer">Outgoing packet buffer</param>
/// <param name="bytesSent">Number of bytes written to the wire</param>
protected abstract void PacketSent(UDPPacketBuffer buffer, int bytesSent); protected abstract void PacketSent(UDPPacketBuffer buffer, int bytesSent);
// the port to listen on /// <summary>UDP port to bind to in server mode</summary>
internal int udpPort; protected int m_udpPort;
// the UDP socket /// <summary>Local IP address to bind to in server mode</summary>
private Socket udpSocket; protected IPAddress m_localBindAddress;
// the ReaderWriterLock is used solely for the purposes of shutdown (Stop()). /// <summary>UDP socket, used in either client or server mode</summary>
// since there are potentially many "reader" threads in the internal .NET IOCP private Socket m_udpSocket;
// thread pool, this is a cheaper synchronization primitive than using
// a Mutex object. This allows many UDP socket "reads" concurrently - when
// Stop() is called, it attempts to obtain a writer lock which will then
// wait until all outstanding operations are completed before shutting down.
// this avoids the problem of closing the socket with outstanding operations
// and trying to catch the inevitable ObjectDisposedException.
private ReaderWriterLock rwLock = new ReaderWriterLock();
// number of outstanding operations. This is a reference count /// <summary>The all important shutdown flag</summary>
// which we use to ensure that the threads exit cleanly. Note that private volatile bool m_shutdownFlag = true;
// we need this because the threads will potentially still need to process
// data even after the socket is closed.
private int rwOperationCount = 0;
// the all important shutdownFlag. This is synchronized through the ReaderWriterLock.
private volatile bool shutdownFlag = true;
// the remote endpoint to communicate with
protected IPEndPoint remoteEndPoint = null;
/// <summary>Returns true if the server is currently listening, otherwise false</summary>
public bool IsRunning { get { return !m_shutdownFlag; } }
/// <summary> /// <summary>
/// Initialize the UDP packet handler in server mode /// Default constructor
/// </summary> /// </summary>
/// <param name="bindAddress">Local IP address to bind the server to</param>
/// <param name="port">Port to listening for incoming UDP packets on</param> /// <param name="port">Port to listening for incoming UDP packets on</param>
public OpenSimUDPBase(int port) public OpenSimUDPBase(IPAddress bindAddress, int port)
{ {
udpPort = port; m_localBindAddress = bindAddress;
m_udpPort = port;
} }
/// <summary> /// <summary>
/// Initialize the UDP packet handler in client mode /// Start the UDP server
/// </summary>
/// <param name="endPoint">Remote UDP server to connect to</param>
public OpenSimUDPBase(IPEndPoint endPoint)
{
remoteEndPoint = endPoint;
udpPort = 0;
}
/// <summary>
///
/// </summary> /// </summary>
/// <remarks>This method will attempt to set the SIO_UDP_CONNRESET flag
/// on the socket to get newer versions of Windows to behave in a sane
/// manner (not throwing an exception when the remote side resets the
/// connection). This call is ignored on Mono where the flag is not
/// necessary</remarks>
public void Start() public void Start()
{ {
if (shutdownFlag) if (m_shutdownFlag)
{ {
if (remoteEndPoint == null) const int SIO_UDP_CONNRESET = -1744830452;
{
// Server mode
// create and bind the socket IPEndPoint ipep = new IPEndPoint(m_localBindAddress, m_udpPort);
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort); m_udpSocket = new Socket(
udpSocket = new Socket(
AddressFamily.InterNetwork, AddressFamily.InterNetwork,
SocketType.Dgram, SocketType.Dgram,
ProtocolType.Udp); ProtocolType.Udp);
udpSocket.Bind(ipep); try
}
else
{ {
// Client mode // this udp socket flag is not supported under mono,
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort); // so we'll catch the exception and continue
udpSocket = new Socket( m_udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
AddressFamily.InterNetwork, m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag set");
SocketType.Dgram,
ProtocolType.Udp);
udpSocket.Bind(ipep);
//udpSocket.Connect(remoteEndPoint);
} }
catch (SocketException)
{
m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag not supported on this platform, ignoring");
}
m_udpSocket.Bind(ipep);
// we're not shutting down, we're starting up // we're not shutting down, we're starting up
shutdownFlag = false; m_shutdownFlag = false;
// kick off an async receive. The Start() method will return, the // kick off an async receive. The Start() method will return, the
// actual receives will occur asynchronously and will be caught in // actual receives will occur asynchronously and will be caught in
@ -133,104 +122,85 @@ namespace OpenSim.Region.ClientStack.LindenUDP
} }
/// <summary> /// <summary>
/// /// Stops the UDP server
/// </summary> /// </summary>
public void Stop() public void Stop()
{ {
if (!shutdownFlag) if (!m_shutdownFlag)
{ {
// wait indefinitely for a writer lock. Once this is called, the .NET runtime // wait indefinitely for a writer lock. Once this is called, the .NET runtime
// will deny any more reader locks, in effect blocking all other send/receive // will deny any more reader locks, in effect blocking all other send/receive
// threads. Once we have the lock, we set shutdownFlag to inform the other // threads. Once we have the lock, we set shutdownFlag to inform the other
// threads that the socket is closed. // threads that the socket is closed.
rwLock.AcquireWriterLock(-1); m_shutdownFlag = true;
shutdownFlag = true; m_udpSocket.Close();
udpSocket.Close();
rwLock.ReleaseWriterLock();
// wait for any pending operations to complete on other
// threads before exiting.
const int FORCE_STOP = 100;
int i = 0;
while (rwOperationCount > 0 && i < FORCE_STOP)
{
Thread.Sleep(10);
++i;
} }
if (i >= FORCE_STOP)
{
Logger.Log("UDPBase.Stop() forced shutdown while waiting on pending operations",
Helpers.LogLevel.Warning);
}
}
}
/// <summary>
///
/// </summary>
public bool IsRunning
{
get { return !shutdownFlag; }
} }
private void AsyncBeginReceive() private void AsyncBeginReceive()
{ {
// this method actually kicks off the async read on the socket.
// we aquire a reader lock here to ensure that no other thread
// is trying to set shutdownFlag and close the socket.
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
{
// increment the count of pending operations
Interlocked.Increment(ref rwOperationCount);
// allocate a packet buffer // allocate a packet buffer
//WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut(); //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
UDPPacketBuffer buf = new UDPPacketBuffer(); UDPPacketBuffer buf = new UDPPacketBuffer();
if (!m_shutdownFlag)
{
try try
{ {
// kick off an async read // kick off an async read
udpSocket.BeginReceiveFrom( m_udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data, //wrappedBuffer.Instance.Data,
buf.Data, buf.Data,
0, 0,
UDPPacketBuffer.BUFFER_SIZE, UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None, SocketFlags.None,
//ref wrappedBuffer.Instance.RemoteEndPoint,
ref buf.RemoteEndPoint, ref buf.RemoteEndPoint,
new AsyncCallback(AsyncEndReceive), AsyncEndReceive,
//wrappedBuffer); //wrappedBuffer);
buf); buf);
} }
catch (SocketException) catch (SocketException e)
{ {
// something bad happened if (e.SocketErrorCode == SocketError.ConnectionReset)
//Logger.Log( {
// "A SocketException occurred in UDPServer.AsyncBeginReceive()", m_log.Warn("[UDPBASE]: SIO_UDP_CONNRESET was ignored, attempting to salvage the UDP listener on port " + m_udpPort);
// Helpers.LogLevel.Error, se); bool salvaged = false;
while (!salvaged)
// an error occurred, therefore the operation is void. Decrement the reference count. {
Interlocked.Decrement(ref rwOperationCount); try
{
m_udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data,
buf.Data,
0,
UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None,
ref buf.RemoteEndPoint,
AsyncEndReceive,
//wrappedBuffer);
buf);
salvaged = true;
} }
catch (SocketException) { }
catch (ObjectDisposedException) { return; }
} }
// we're done with the socket for now, release the reader lock. m_log.Warn("[UDPBASE]: Salvaged the UDP listener on port " + m_udpPort);
rwLock.ReleaseReaderLock(); }
}
catch (ObjectDisposedException) { }
}
} }
private void AsyncEndReceive(IAsyncResult iar) private void AsyncEndReceive(IAsyncResult iar)
{ {
// Asynchronous receive operations will complete here through the call // Asynchronous receive operations will complete here through the call
// to AsyncBeginReceive // to AsyncBeginReceive
if (!m_shutdownFlag)
// aquire a reader lock
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
{ {
// start another receive - this keeps the server going!
AsyncBeginReceive();
// get the buffer that was created in AsyncBeginReceive // get the buffer that was created in AsyncBeginReceive
// this is the received data // this is the received data
//WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState; //WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
@ -241,100 +211,49 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{ {
// get the length of data actually read from the socket, store it with the // get the length of data actually read from the socket, store it with the
// buffer // buffer
buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint); buffer.DataLength = m_udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// this operation is now complete, decrement the reference count
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket, release the reader lock
rwLock.ReleaseReaderLock();
// call the abstract method PacketReceived(), passing the buffer that // call the abstract method PacketReceived(), passing the buffer that
// has just been filled from the socket read. // has just been filled from the socket read.
PacketReceived(buffer); PacketReceived(buffer);
} }
catch (SocketException) catch (SocketException) { }
{ catch (ObjectDisposedException) { }
// an error occurred, therefore the operation is void. Decrement the reference count. //finally { wrappedBuffer.Dispose(); }
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket for now, release the reader lock.
rwLock.ReleaseReaderLock();
}
finally
{
// start another receive - this keeps the server going!
AsyncBeginReceive();
//wrappedBuffer.Dispose();
}
}
else
{
// nothing bad happened, but we are done with the operation
// decrement the reference count and release the reader lock
Interlocked.Decrement(ref rwOperationCount);
rwLock.ReleaseReaderLock();
} }
} }
public void AsyncBeginSend(UDPPacketBuffer buf) public void AsyncBeginSend(UDPPacketBuffer buf)
{ {
rwLock.AcquireReaderLock(-1); if (!m_shutdownFlag)
if (!shutdownFlag)
{ {
try try
{ {
Interlocked.Increment(ref rwOperationCount); m_udpSocket.BeginSendTo(
udpSocket.BeginSendTo(
buf.Data, buf.Data,
0, 0,
buf.DataLength, buf.DataLength,
SocketFlags.None, SocketFlags.None,
buf.RemoteEndPoint, buf.RemoteEndPoint,
new AsyncCallback(AsyncEndSend), AsyncEndSend,
buf); buf);
} }
catch (SocketException) catch (SocketException) { }
{ catch (ObjectDisposedException) { }
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncBeginSend()",
// Helpers.LogLevel.Error, se);
} }
} }
rwLock.ReleaseReaderLock(); void AsyncEndSend(IAsyncResult result)
}
private void AsyncEndSend(IAsyncResult iar)
{ {
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
{
UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
try try
{ {
int bytesSent = udpSocket.EndSendTo(iar); UDPPacketBuffer buf = (UDPPacketBuffer)result.AsyncState;
int bytesSent = m_udpSocket.EndSendTo(result);
// note that call to the abstract PacketSent() method - we are passing the number PacketSent(buf, bytesSent);
// of bytes sent in a separate parameter, since we can't use buffer.DataLength which
// is the number of bytes to send (or bytes received depending upon whether this
// buffer was part of a send or a receive).
PacketSent(buffer, bytesSent);
} }
catch (SocketException) catch (SocketException) { }
{ catch (ObjectDisposedException) { }
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncEndSend()",
// Helpers.LogLevel.Error, se);
}
}
Interlocked.Decrement(ref rwOperationCount);
rwLock.ReleaseReaderLock();
} }
} }
} }