OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* Added a lock object for the write functions in LLUDPClientCollection (immutable != concurrent write safety) 

* Allow the UDP server to bind to a user-specified port again
* Updated to a newer version of OpenSimUDPBase that streamlines the code even more. This also reintroduces the highly concurrent packet handling which needs more testing
prioritization
John Hurliman 2009-10-09 01:53:06 -07:00
parent 56a27c37d3
commit a5b9971fd7
3 changed files with 169 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
OpenSim/Region/ClientStack/LindenUDP/LLUDPClientCollection.cs
View File

@ -36,6 +36,9 @@ using ReaderWriterLockImpl = OpenMetaverse.ReaderWriterLockSlim;
namespace OpenSim.Region.ClientStack.LindenUDP
{
/// <summary>
/// A thread safe mapping from endpoints to client references
/// </summary>
public sealed class UDPClientCollection
{
#region IComparers
@ -52,43 +55,80 @@ namespace OpenSim.Region.ClientStack.LindenUDP
#endregion IComparers
/// <summary>An immutable dictionary mapping from <seealso cref="IPEndPoint"/>
/// to <seealso cref="LLUDPClient"/> references</summary>
private ImmutableMap<IPEndPoint, LLUDPClient> m_dict;
/// <summary>Immutability grants thread safety for concurrent reads and
/// read-writes, but not concurrent writes</summary>
private object m_writeLock;
/// <summary>Number of clients in the collection</summary>
public int Count { get { return m_dict.Count; } }
/// <summary>
/// Default constructor
/// </summary>
public UDPClientCollection()
{
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
}
/// <summary>
/// Add a client reference to the collection
/// </summary>
/// <param name="key">Remote endpoint of the client</param>
/// <param name="value">Reference to the client object</param>
public void Add(IPEndPoint key, LLUDPClient value)
{
m_dict = m_dict.Add(key, value);
lock (m_writeLock)
m_dict = m_dict.Add(key, value);
}
/// <summary>
/// Remove a client from the collection
/// </summary>
/// <param name="key">Remote endpoint of the client</param>
public void Remove(IPEndPoint key)
{
m_dict = m_dict.Delete(key);
lock (m_writeLock)
m_dict = m_dict.Delete(key);
}
/// <summary>
/// Resets the client collection
/// </summary>
public void Clear()
{
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
}
public int Count
{
get { return m_dict.Count; }
lock (m_writeLock)
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
}
/// <summary>
/// Checks if an endpoint is in the collection
/// </summary>
/// <param name="key">Endpoint to check for</param>
/// <returns>True if the endpoint was found in the collection, otherwise false</returns>
public bool ContainsKey(IPEndPoint key)
{
return m_dict.ContainsKey(key);
}
/// <summary>
/// Attempts to fetch a value out of the collection
/// </summary>
/// <param name="key">Endpoint of the client to retrieve</param>
/// <param name="value">Retrieved client, or null on lookup failure</param>
/// <returns>True if the lookup succeeded, otherwise false</returns>
public bool TryGetValue(IPEndPoint key, out LLUDPClient value)
{
return m_dict.TryGetValue(key, out value);
}
/// <summary>
/// Performs a given task in parallel for each of the elements in the
/// collection
/// </summary>
/// <param name="action">Action to perform on each element</param>
public void ForEach(Action<LLUDPClient> action)
{
Parallel.ForEach<LLUDPClient>(m_dict.Values, action);

20
OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
View File

@ -96,7 +96,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// <summary>Incoming packets that are awaiting handling</summary>
private OpenMetaverse.BlockingQueue<IncomingPacket> packetInbox = new OpenMetaverse.BlockingQueue<IncomingPacket>();
/// <summary></summary>
private UDPClientCollection clients = new UDPClientCollection();
private UDPClientCollection m_clients = new UDPClientCollection();
/// <summary>Bandwidth throttle for this UDP server</summary>
private TokenBucket m_throttle;
/// <summary>Bandwidth throttle rates for this UDP server</summary>
@ -115,7 +115,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public Socket Server { get { return null; } }
public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
: base((int)port)
: base(listenIP, (int)port)
{
#region Environment.TickCount Measurement
@ -143,7 +143,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public new void Start()
{
if (m_scene == null)
throw new InvalidOperationException("Cannot LLUDPServer.Start() without an IScene reference");
throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
base.Start();
@ -188,7 +188,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
m_scene.ClientManager.Remove(udpClient.CircuitCode);
udpClient.ClientAPI.Close(false);
udpClient.Shutdown();
clients.Remove(udpClient.RemoteEndPoint);
m_clients.Remove(udpClient.RemoteEndPoint);
}
public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting)
@ -208,7 +208,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
for (int i = 0; i < packetCount; i++)
{
byte[] data = datas[i];
clients.ForEach(
m_clients.ForEach(
delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
}
@ -216,7 +216,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
else
{
byte[] data = packet.ToBytes();
clients.ForEach(
m_clients.ForEach(
delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
}
@ -502,7 +502,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
// Determine which agent this packet came from
if (!clients.TryGetValue(address, out client))
if (!m_clients.TryGetValue(address, out client))
{
m_log.Warn("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address);
return;
@ -606,7 +606,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (m_scene.RegionStatus != RegionStatus.SlaveScene)
{
AuthenticateResponse sessionInfo;
bool isNewCircuit = !clients.ContainsKey(remoteEndPoint);
bool isNewCircuit = !m_clients.ContainsKey(remoteEndPoint);
if (!IsClientAuthorized(useCircuitCode, out sessionInfo))
{
@ -648,7 +648,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
udpClient.ClientAPI = clientApi;
// Add the new client to our list of tracked clients
clients.Add(udpClient.RemoteEndPoint, udpClient);
m_clients.Add(udpClient.RemoteEndPoint, udpClient);
}
private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend)
@ -726,7 +726,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
elapsed500MS = 0;
}
clients.ForEach(
m_clients.ForEach(
delegate(LLUDPClient client)
{
if (client.DequeueOutgoing())

303
OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
View File

@ -29,101 +29,90 @@ using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using OpenMetaverse;
using log4net;
namespace OpenSim.Region.ClientStack.LindenUDP
namespace OpenMetaverse
{
/// <summary>
///
/// Base UDP server
/// </summary>
public abstract class OpenSimUDPBase
{
// these abstract methods must be implemented in a derived class to actually do
// something with the packets that are sent and received.
private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
/// <summary>
/// This method is called when an incoming packet is received
/// </summary>
/// <param name="buffer">Incoming packet buffer</param>
protected abstract void PacketReceived(UDPPacketBuffer buffer);
/// <summary>
/// This method is called when an outgoing packet is sent
/// </summary>
/// <param name="buffer">Outgoing packet buffer</param>
/// <param name="bytesSent">Number of bytes written to the wire</param>
protected abstract void PacketSent(UDPPacketBuffer buffer, int bytesSent);
// the port to listen on
internal int udpPort;
/// <summary>UDP port to bind to in server mode</summary>
protected int m_udpPort;
// the UDP socket
private Socket udpSocket;
/// <summary>Local IP address to bind to in server mode</summary>
protected IPAddress m_localBindAddress;
// the ReaderWriterLock is used solely for the purposes of shutdown (Stop()).
// since there are potentially many "reader" threads in the internal .NET IOCP
// thread pool, this is a cheaper synchronization primitive than using
// a Mutex object. This allows many UDP socket "reads" concurrently - when
// Stop() is called, it attempts to obtain a writer lock which will then
// wait until all outstanding operations are completed before shutting down.
// this avoids the problem of closing the socket with outstanding operations
// and trying to catch the inevitable ObjectDisposedException.
private ReaderWriterLock rwLock = new ReaderWriterLock();
/// <summary>UDP socket, used in either client or server mode</summary>
private Socket m_udpSocket;
// number of outstanding operations. This is a reference count
// which we use to ensure that the threads exit cleanly. Note that
// we need this because the threads will potentially still need to process
// data even after the socket is closed.
private int rwOperationCount = 0;
// the all important shutdownFlag. This is synchronized through the ReaderWriterLock.
private volatile bool shutdownFlag = true;
// the remote endpoint to communicate with
protected IPEndPoint remoteEndPoint = null;
/// <summary>The all important shutdown flag</summary>
private volatile bool m_shutdownFlag = true;
/// <summary>Returns true if the server is currently listening, otherwise false</summary>
public bool IsRunning { get { return !m_shutdownFlag; } }
/// <summary>
/// Initialize the UDP packet handler in server mode
/// Default constructor
/// </summary>
/// <param name="bindAddress">Local IP address to bind the server to</param>
/// <param name="port">Port to listening for incoming UDP packets on</param>
public OpenSimUDPBase(int port)
public OpenSimUDPBase(IPAddress bindAddress, int port)
{
udpPort = port;
m_localBindAddress = bindAddress;
m_udpPort = port;
}
/// <summary>
/// Initialize the UDP packet handler in client mode
/// </summary>
/// <param name="endPoint">Remote UDP server to connect to</param>
public OpenSimUDPBase(IPEndPoint endPoint)
{
remoteEndPoint = endPoint;
udpPort = 0;
}
/// <summary>
///
/// Start the UDP server
/// </summary>
/// <remarks>This method will attempt to set the SIO_UDP_CONNRESET flag
/// on the socket to get newer versions of Windows to behave in a sane
/// manner (not throwing an exception when the remote side resets the
/// connection). This call is ignored on Mono where the flag is not
/// necessary</remarks>
public void Start()
{
if (shutdownFlag)
if (m_shutdownFlag)
{
if (remoteEndPoint == null)
{
// Server mode
const int SIO_UDP_CONNRESET = -1744830452;
// create and bind the socket
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
udpSocket = new Socket(
AddressFamily.InterNetwork,
SocketType.Dgram,
ProtocolType.Udp);
udpSocket.Bind(ipep);
}
else
IPEndPoint ipep = new IPEndPoint(m_localBindAddress, m_udpPort);
m_udpSocket = new Socket(
AddressFamily.InterNetwork,
SocketType.Dgram,
ProtocolType.Udp);
try
{
// Client mode
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
udpSocket = new Socket(
AddressFamily.InterNetwork,
SocketType.Dgram,
ProtocolType.Udp);
udpSocket.Bind(ipep);
//udpSocket.Connect(remoteEndPoint);
// this udp socket flag is not supported under mono,
// so we'll catch the exception and continue
m_udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag set");
}
catch (SocketException)
{
m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag not supported on this platform, ignoring");
}
m_udpSocket.Bind(ipep);
// we're not shutting down, we're starting up
shutdownFlag = false;
m_shutdownFlag = false;
// kick off an async receive. The Start() method will return, the
// actual receives will occur asynchronously and will be caught in
@ -133,104 +122,85 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
/// <summary>
///
/// Stops the UDP server
/// </summary>
public void Stop()
{
if (!shutdownFlag)
if (!m_shutdownFlag)
{
// wait indefinitely for a writer lock. Once this is called, the .NET runtime
// will deny any more reader locks, in effect blocking all other send/receive
// threads. Once we have the lock, we set shutdownFlag to inform the other
// threads that the socket is closed.
rwLock.AcquireWriterLock(-1);
shutdownFlag = true;
udpSocket.Close();
rwLock.ReleaseWriterLock();
// wait for any pending operations to complete on other
// threads before exiting.
const int FORCE_STOP = 100;
int i = 0;
while (rwOperationCount > 0 && i < FORCE_STOP)
{
Thread.Sleep(10);
++i;
}
if (i >= FORCE_STOP)
{
Logger.Log("UDPBase.Stop() forced shutdown while waiting on pending operations",
Helpers.LogLevel.Warning);
}
m_shutdownFlag = true;
m_udpSocket.Close();
}
}
/// <summary>
///
/// </summary>
public bool IsRunning
{
get { return !shutdownFlag; }
}
private void AsyncBeginReceive()
{
// this method actually kicks off the async read on the socket.
// we aquire a reader lock here to ensure that no other thread
// is trying to set shutdownFlag and close the socket.
rwLock.AcquireReaderLock(-1);
// allocate a packet buffer
//WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
UDPPacketBuffer buf = new UDPPacketBuffer();
if (!shutdownFlag)
if (!m_shutdownFlag)
{
// increment the count of pending operations
Interlocked.Increment(ref rwOperationCount);
// allocate a packet buffer
//WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
UDPPacketBuffer buf = new UDPPacketBuffer();
try
{
// kick off an async read
udpSocket.BeginReceiveFrom(
m_udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data,
buf.Data,
0,
UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None,
//ref wrappedBuffer.Instance.RemoteEndPoint,
ref buf.RemoteEndPoint,
new AsyncCallback(AsyncEndReceive),
AsyncEndReceive,
//wrappedBuffer);
buf);
}
catch (SocketException)
catch (SocketException e)
{
// something bad happened
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncBeginReceive()",
// Helpers.LogLevel.Error, se);
if (e.SocketErrorCode == SocketError.ConnectionReset)
{
m_log.Warn("[UDPBASE]: SIO_UDP_CONNRESET was ignored, attempting to salvage the UDP listener on port " + m_udpPort);
bool salvaged = false;
while (!salvaged)
{
try
{
m_udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data,
buf.Data,
0,
UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None,
ref buf.RemoteEndPoint,
AsyncEndReceive,
//wrappedBuffer);
buf);
salvaged = true;
}
catch (SocketException) { }
catch (ObjectDisposedException) { return; }
}
// an error occurred, therefore the operation is void. Decrement the reference count.
Interlocked.Decrement(ref rwOperationCount);
m_log.Warn("[UDPBASE]: Salvaged the UDP listener on port " + m_udpPort);
}
}
catch (ObjectDisposedException) { }
}
// we're done with the socket for now, release the reader lock.
rwLock.ReleaseReaderLock();
}
private void AsyncEndReceive(IAsyncResult iar)
{
// Asynchronous receive operations will complete here through the call
// to AsyncBeginReceive
// aquire a reader lock
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
if (!m_shutdownFlag)
{
// start another receive - this keeps the server going!
AsyncBeginReceive();
// get the buffer that was created in AsyncBeginReceive
// this is the received data
//WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
@ -241,100 +211,49 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
// get the length of data actually read from the socket, store it with the
// buffer
buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// this operation is now complete, decrement the reference count
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket, release the reader lock
rwLock.ReleaseReaderLock();
buffer.DataLength = m_udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// call the abstract method PacketReceived(), passing the buffer that
// has just been filled from the socket read.
PacketReceived(buffer);
}
catch (SocketException)
{
// an error occurred, therefore the operation is void. Decrement the reference count.
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket for now, release the reader lock.
rwLock.ReleaseReaderLock();
}
finally
{
// start another receive - this keeps the server going!
AsyncBeginReceive();
//wrappedBuffer.Dispose();
}
}
else
{
// nothing bad happened, but we are done with the operation
// decrement the reference count and release the reader lock
Interlocked.Decrement(ref rwOperationCount);
rwLock.ReleaseReaderLock();
catch (SocketException) { }
catch (ObjectDisposedException) { }
//finally { wrappedBuffer.Dispose(); }
}
}
public void AsyncBeginSend(UDPPacketBuffer buf)
{
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
if (!m_shutdownFlag)
{
try
{
Interlocked.Increment(ref rwOperationCount);
udpSocket.BeginSendTo(
m_udpSocket.BeginSendTo(
buf.Data,
0,
buf.DataLength,
SocketFlags.None,
buf.RemoteEndPoint,
new AsyncCallback(AsyncEndSend),
AsyncEndSend,
buf);
}
catch (SocketException)
{
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncBeginSend()",
// Helpers.LogLevel.Error, se);
}
catch (SocketException) { }
catch (ObjectDisposedException) { }
}
rwLock.ReleaseReaderLock();
}
private void AsyncEndSend(IAsyncResult iar)
void AsyncEndSend(IAsyncResult result)
{
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag)
try
{
UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
UDPPacketBuffer buf = (UDPPacketBuffer)result.AsyncState;
int bytesSent = m_udpSocket.EndSendTo(result);
try
{
int bytesSent = udpSocket.EndSendTo(iar);
// note that call to the abstract PacketSent() method - we are passing the number
// of bytes sent in a separate parameter, since we can't use buffer.DataLength which
// is the number of bytes to send (or bytes received depending upon whether this
// buffer was part of a send or a receive).
PacketSent(buffer, bytesSent);
}
catch (SocketException)
{
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncEndSend()",
// Helpers.LogLevel.Error, se);
}
PacketSent(buf, bytesSent);
}
Interlocked.Decrement(ref rwOperationCount);
rwLock.ReleaseReaderLock();
catch (SocketException) { }
catch (ObjectDisposedException) { }
}
}
}