Putting this to exactly what it was yesterday around this time, so we can do the 100-ppl load test in WP. We need to carefully play with this code in order to understand all the problems.

prioritization
Diva Canto 2009-10-09 11:32:53 -07:00
parent 78f57283f4
commit 8231cb2231
2 changed files with 411 additions and 182 deletions

View File

@ -247,16 +247,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
////return success; ////return success;
lock (m_sync) //lock (m_sync)
return Dictionary2.TryGetValue(key, out value);
//try
//{
// return Dictionary2.TryGetValue(key, out value); // return Dictionary2.TryGetValue(key, out value);
//}
//catch { } try
//value = null; {
//return false; return Dictionary2.TryGetValue(key, out value);
}
catch { }
value = null;
return false;
} }

View File

@ -1,29 +1,30 @@
/* //--------- Modified Version -------------------
* Copyright (c) 2006, Clutch, Inc. ///*
* Original Author: Jeff Cesnik // * Copyright (c) 2006, Clutch, Inc.
* All rights reserved. // * Original Author: Jeff Cesnik
* // * All rights reserved.
* - Redistribution and use in source and binary forms, with or without // *
* modification, are permitted provided that the following conditions are met: // * - Redistribution and use in source and binary forms, with or without
* // * modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this // *
* list of conditions and the following disclaimer. // * - Redistributions of source code must retain the above copyright notice, this
* - Neither the name of the openmetaverse.org nor the names // * list of conditions and the following disclaimer.
* of its contributors may be used to endorse or promote products derived from // * - Neither the name of the openmetaverse.org nor the names
* this software without specific prior written permission. // * of its contributors may be used to endorse or promote products derived from
* // * this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" // *
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE // * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR // * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF // * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS // * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN // * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* POSSIBILITY OF SUCH DAMAGE. // * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
*/ // * POSSIBILITY OF SUCH DAMAGE.
// */
using System; using System;
using System.Net; using System.Net;
@ -31,6 +32,317 @@ using System.Net.Sockets;
using System.Threading; using System.Threading;
using OpenMetaverse; using OpenMetaverse;
//namespace OpenSim.Region.ClientStack.LindenUDP
//{
// /// <summary>
// ///
// /// </summary>
// public abstract class OpenSimUDPBase
// {
// // these abstract methods must be implemented in a derived class to actually do
// // something with the packets that are sent and received.
// protected abstract void PacketReceived(UDPPacketBuffer buffer);
// protected abstract void PacketSent(UDPPacketBuffer buffer, int bytesSent);
// // the port to listen on
// internal int udpPort;
// // the UDP socket
// private Socket udpSocket;
// // the ReaderWriterLock is used solely for the purposes of shutdown (Stop()).
// // since there are potentially many "reader" threads in the internal .NET IOCP
// // thread pool, this is a cheaper synchronization primitive than using
// // a Mutex object. This allows many UDP socket "reads" concurrently - when
// // Stop() is called, it attempts to obtain a writer lock which will then
// // wait until all outstanding operations are completed before shutting down.
// // this avoids the problem of closing the socket with outstanding operations
// // and trying to catch the inevitable ObjectDisposedException.
// private ReaderWriterLock rwLock = new ReaderWriterLock();
// // number of outstanding operations. This is a reference count
// // which we use to ensure that the threads exit cleanly. Note that
// // we need this because the threads will potentially still need to process
// // data even after the socket is closed.
// private int rwOperationCount = 0;
// // the all important shutdownFlag. This is synchronized through the ReaderWriterLock.
// private volatile bool shutdownFlag = true;
// // the remote endpoint to communicate with
// protected IPEndPoint remoteEndPoint = null;
// /// <summary>
// /// Initialize the UDP packet handler in server mode
// /// </summary>
// /// <param name="port">Port to listening for incoming UDP packets on</param>
// public OpenSimUDPBase(int port)
// {
// udpPort = port;
// }
// /// <summary>
// /// Initialize the UDP packet handler in client mode
// /// </summary>
// /// <param name="endPoint">Remote UDP server to connect to</param>
// public OpenSimUDPBase(IPEndPoint endPoint)
// {
// remoteEndPoint = endPoint;
// udpPort = 0;
// }
// /// <summary>
// ///
// /// </summary>
// public void Start()
// {
// if (shutdownFlag)
// {
// if (remoteEndPoint == null)
// {
// // Server mode
// // create and bind the socket
// IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
// udpSocket = new Socket(
// AddressFamily.InterNetwork,
// SocketType.Dgram,
// ProtocolType.Udp);
// udpSocket.Bind(ipep);
// }
// else
// {
// // Client mode
// IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
// udpSocket = new Socket(
// AddressFamily.InterNetwork,
// SocketType.Dgram,
// ProtocolType.Udp);
// udpSocket.Bind(ipep);
// //udpSocket.Connect(remoteEndPoint);
// }
// // we're not shutting down, we're starting up
// shutdownFlag = false;
// // kick off an async receive. The Start() method will return, the
// // actual receives will occur asynchronously and will be caught in
// // AsyncEndRecieve().
// AsyncBeginReceive();
// }
// }
// /// <summary>
// ///
// /// </summary>
// public void Stop()
// {
// if (!shutdownFlag)
// {
// // wait indefinitely for a writer lock. Once this is called, the .NET runtime
// // will deny any more reader locks, in effect blocking all other send/receive
// // threads. Once we have the lock, we set shutdownFlag to inform the other
// // threads that the socket is closed.
// rwLock.AcquireWriterLock(-1);
// shutdownFlag = true;
// udpSocket.Close();
// rwLock.ReleaseWriterLock();
// // wait for any pending operations to complete on other
// // threads before exiting.
// const int FORCE_STOP = 100;
// int i = 0;
// while (rwOperationCount > 0 && i < FORCE_STOP)
// {
// Thread.Sleep(10);
// ++i;
// }
// if (i >= FORCE_STOP)
// {
// Logger.Log("UDPBase.Stop() forced shutdown while waiting on pending operations",
// Helpers.LogLevel.Warning);
// }
// }
// }
// /// <summary>
// ///
// /// </summary>
// public bool IsRunning
// {
// get { return !shutdownFlag; }
// }
// private void AsyncBeginReceive()
// {
// // this method actually kicks off the async read on the socket.
// // we aquire a reader lock here to ensure that no other thread
// // is trying to set shutdownFlag and close the socket.
// rwLock.AcquireReaderLock(-1);
// if (!shutdownFlag)
// {
// // increment the count of pending operations
// Interlocked.Increment(ref rwOperationCount);
// // allocate a packet buffer
// //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
// UDPPacketBuffer buf = new UDPPacketBuffer();
// try
// {
// // kick off an async read
// udpSocket.BeginReceiveFrom(
// //wrappedBuffer.Instance.Data,
// buf.Data,
// 0,
// UDPPacketBuffer.BUFFER_SIZE,
// SocketFlags.None,
// //ref wrappedBuffer.Instance.RemoteEndPoint,
// ref buf.RemoteEndPoint,
// new AsyncCallback(AsyncEndReceive),
// //wrappedBuffer);
// buf);
// }
// catch (SocketException)
// {
// // something bad happened
// //Logger.Log(
// // "A SocketException occurred in UDPServer.AsyncBeginReceive()",
// // Helpers.LogLevel.Error, se);
// // an error occurred, therefore the operation is void. Decrement the reference count.
// Interlocked.Decrement(ref rwOperationCount);
// }
// }
// // we're done with the socket for now, release the reader lock.
// rwLock.ReleaseReaderLock();
// }
// private void AsyncEndReceive(IAsyncResult iar)
// {
// // Asynchronous receive operations will complete here through the call
// // to AsyncBeginReceive
// // aquire a reader lock
// rwLock.AcquireReaderLock(-1);
// if (!shutdownFlag)
// {
// // get the buffer that was created in AsyncBeginReceive
// // this is the received data
// //WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
// //UDPPacketBuffer buffer = wrappedBuffer.Instance;
// UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
// try
// {
// // get the length of data actually read from the socket, store it with the
// // buffer
// buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// // this operation is now complete, decrement the reference count
// Interlocked.Decrement(ref rwOperationCount);
// // we're done with the socket, release the reader lock
// rwLock.ReleaseReaderLock();
// // call the abstract method PacketReceived(), passing the buffer that
// // has just been filled from the socket read.
// PacketReceived(buffer);
// }
// catch (SocketException)
// {
// // an error occurred, therefore the operation is void. Decrement the reference count.
// Interlocked.Decrement(ref rwOperationCount);
// // we're done with the socket for now, release the reader lock.
// rwLock.ReleaseReaderLock();
// }
// finally
// {
// // start another receive - this keeps the server going!
// AsyncBeginReceive();
// //wrappedBuffer.Dispose();
// }
// }
// else
// {
// // nothing bad happened, but we are done with the operation
// // decrement the reference count and release the reader lock
// Interlocked.Decrement(ref rwOperationCount);
// rwLock.ReleaseReaderLock();
// }
// }
// public void AsyncBeginSend(UDPPacketBuffer buf)
// {
// rwLock.AcquireReaderLock(-1);
// if (!shutdownFlag)
// {
// try
// {
// Interlocked.Increment(ref rwOperationCount);
// udpSocket.BeginSendTo(
// buf.Data,
// 0,
// buf.DataLength,
// SocketFlags.None,
// buf.RemoteEndPoint,
// new AsyncCallback(AsyncEndSend),
// buf);
// }
// catch (SocketException)
// {
// //Logger.Log(
// // "A SocketException occurred in UDPServer.AsyncBeginSend()",
// // Helpers.LogLevel.Error, se);
// }
// }
// rwLock.ReleaseReaderLock();
// }
// private void AsyncEndSend(IAsyncResult iar)
// {
// rwLock.AcquireReaderLock(-1);
// if (!shutdownFlag)
// {
// UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
// try
// {
// int bytesSent = udpSocket.EndSendTo(iar);
// // note that call to the abstract PacketSent() method - we are passing the number
// // of bytes sent in a separate parameter, since we can't use buffer.DataLength which
// // is the number of bytes to send (or bytes received depending upon whether this
// // buffer was part of a send or a receive).
// PacketSent(buffer, bytesSent);
// }
// catch (SocketException)
// {
// //Logger.Log(
// // "A SocketException occurred in UDPServer.AsyncEndSend()",
// // Helpers.LogLevel.Error, se);
// }
// }
// Interlocked.Decrement(ref rwOperationCount);
// rwLock.ReleaseReaderLock();
// }
// }
//}
//--------- Original Version -------------------
namespace OpenSim.Region.ClientStack.LindenUDP namespace OpenSim.Region.ClientStack.LindenUDP
{ {
/// <summary> /// <summary>
@ -49,29 +361,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// the UDP socket // the UDP socket
private Socket udpSocket; private Socket udpSocket;
// the ReaderWriterLock is used solely for the purposes of shutdown (Stop()). // the all important shutdownFlag.
// since there are potentially many "reader" threads in the internal .NET IOCP
// thread pool, this is a cheaper synchronization primitive than using
// a Mutex object. This allows many UDP socket "reads" concurrently - when
// Stop() is called, it attempts to obtain a writer lock which will then
// wait until all outstanding operations are completed before shutting down.
// this avoids the problem of closing the socket with outstanding operations
// and trying to catch the inevitable ObjectDisposedException.
private ReaderWriterLock rwLock = new ReaderWriterLock();
// number of outstanding operations. This is a reference count
// which we use to ensure that the threads exit cleanly. Note that
// we need this because the threads will potentially still need to process
// data even after the socket is closed.
private int rwOperationCount = 0;
// the all important shutdownFlag. This is synchronized through the ReaderWriterLock.
private volatile bool shutdownFlag = true; private volatile bool shutdownFlag = true;
// the remote endpoint to communicate with // the remote endpoint to communicate with
protected IPEndPoint remoteEndPoint = null; protected IPEndPoint remoteEndPoint = null;
/// <summary> /// <summary>
/// Initialize the UDP packet handler in server mode /// Initialize the UDP packet handler in server mode
/// </summary> /// </summary>
@ -98,29 +393,24 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{ {
if (shutdownFlag) if (shutdownFlag)
{ {
if (remoteEndPoint == null) const int SIO_UDP_CONNRESET = -1744830452;
{
// Server mode
// create and bind the socket IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort); udpSocket = new Socket(
udpSocket = new Socket( AddressFamily.InterNetwork,
AddressFamily.InterNetwork, SocketType.Dgram,
SocketType.Dgram, ProtocolType.Udp);
ProtocolType.Udp); try
udpSocket.Bind(ipep);
}
else
{ {
// Client mode // this udp socket flag is not supported under mono,
IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort); // so we'll catch the exception and continue
udpSocket = new Socket( udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
AddressFamily.InterNetwork,
SocketType.Dgram,
ProtocolType.Udp);
udpSocket.Bind(ipep);
//udpSocket.Connect(remoteEndPoint);
} }
catch (SocketException)
{
Logger.DebugLog("UDP SIO_UDP_CONNRESET flag not supported on this platform");
}
udpSocket.Bind(ipep);
// we're not shutting down, we're starting up // we're not shutting down, we're starting up
shutdownFlag = false; shutdownFlag = false;
@ -143,26 +433,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// will deny any more reader locks, in effect blocking all other send/receive // will deny any more reader locks, in effect blocking all other send/receive
// threads. Once we have the lock, we set shutdownFlag to inform the other // threads. Once we have the lock, we set shutdownFlag to inform the other
// threads that the socket is closed. // threads that the socket is closed.
rwLock.AcquireWriterLock(-1);
shutdownFlag = true; shutdownFlag = true;
udpSocket.Close(); udpSocket.Close();
rwLock.ReleaseWriterLock();
// wait for any pending operations to complete on other
// threads before exiting.
const int FORCE_STOP = 100;
int i = 0;
while (rwOperationCount > 0 && i < FORCE_STOP)
{
Thread.Sleep(10);
++i;
}
if (i >= FORCE_STOP)
{
Logger.Log("UDPBase.Stop() forced shutdown while waiting on pending operations",
Helpers.LogLevel.Warning);
}
} }
} }
@ -176,20 +448,12 @@ namespace OpenSim.Region.ClientStack.LindenUDP
private void AsyncBeginReceive() private void AsyncBeginReceive()
{ {
// this method actually kicks off the async read on the socket. // allocate a packet buffer
// we aquire a reader lock here to ensure that no other thread //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
// is trying to set shutdownFlag and close the socket. UDPPacketBuffer buf = new UDPPacketBuffer();
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag) if (!shutdownFlag)
{ {
// increment the count of pending operations
Interlocked.Increment(ref rwOperationCount);
// allocate a packet buffer
//WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
UDPPacketBuffer buf = new UDPPacketBuffer();
try try
{ {
// kick off an async read // kick off an async read
@ -199,38 +463,53 @@ namespace OpenSim.Region.ClientStack.LindenUDP
0, 0,
UDPPacketBuffer.BUFFER_SIZE, UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None, SocketFlags.None,
//ref wrappedBuffer.Instance.RemoteEndPoint,
ref buf.RemoteEndPoint, ref buf.RemoteEndPoint,
new AsyncCallback(AsyncEndReceive), AsyncEndReceive,
//wrappedBuffer); //wrappedBuffer);
buf); buf);
} }
catch (SocketException) catch (SocketException e)
{ {
// something bad happened if (e.SocketErrorCode == SocketError.ConnectionReset)
//Logger.Log( {
// "A SocketException occurred in UDPServer.AsyncBeginReceive()", Logger.Log("SIO_UDP_CONNRESET was ignored, attempting to salvage the UDP listener on port " + udpPort, Helpers.LogLevel.Error);
// Helpers.LogLevel.Error, se); bool salvaged = false;
while (!salvaged)
{
try
{
udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data,
buf.Data,
0,
UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None,
ref buf.RemoteEndPoint,
AsyncEndReceive,
//wrappedBuffer);
buf);
salvaged = true;
}
catch (SocketException) { }
catch (ObjectDisposedException) { return; }
}
// an error occurred, therefore the operation is void. Decrement the reference count. Logger.Log("Salvaged the UDP listener on port " + udpPort, Helpers.LogLevel.Info);
Interlocked.Decrement(ref rwOperationCount); }
} }
catch (ObjectDisposedException) { }
} }
// we're done with the socket for now, release the reader lock.
rwLock.ReleaseReaderLock();
} }
private void AsyncEndReceive(IAsyncResult iar) private void AsyncEndReceive(IAsyncResult iar)
{ {
// Asynchronous receive operations will complete here through the call // Asynchronous receive operations will complete here through the call
// to AsyncBeginReceive // to AsyncBeginReceive
// aquire a reader lock
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag) if (!shutdownFlag)
{ {
// start another receive - this keeps the server going!
AsyncBeginReceive();
// get the buffer that was created in AsyncBeginReceive // get the buffer that was created in AsyncBeginReceive
// this is the received data // this is the received data
//WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState; //WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
@ -243,98 +522,48 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// buffer // buffer
buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint); buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// this operation is now complete, decrement the reference count
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket, release the reader lock
rwLock.ReleaseReaderLock();
// call the abstract method PacketReceived(), passing the buffer that // call the abstract method PacketReceived(), passing the buffer that
// has just been filled from the socket read. // has just been filled from the socket read.
PacketReceived(buffer); PacketReceived(buffer);
} }
catch (SocketException) catch (SocketException) { }
{ catch (ObjectDisposedException) { }
// an error occurred, therefore the operation is void. Decrement the reference count. //finally { wrappedBuffer.Dispose(); }
Interlocked.Decrement(ref rwOperationCount);
// we're done with the socket for now, release the reader lock.
rwLock.ReleaseReaderLock();
}
finally
{
// start another receive - this keeps the server going!
AsyncBeginReceive();
//wrappedBuffer.Dispose();
}
}
else
{
// nothing bad happened, but we are done with the operation
// decrement the reference count and release the reader lock
Interlocked.Decrement(ref rwOperationCount);
rwLock.ReleaseReaderLock();
} }
} }
public void AsyncBeginSend(UDPPacketBuffer buf) public void AsyncBeginSend(UDPPacketBuffer buf)
{ {
rwLock.AcquireReaderLock(-1);
if (!shutdownFlag) if (!shutdownFlag)
{ {
try try
{ {
Interlocked.Increment(ref rwOperationCount);
udpSocket.BeginSendTo( udpSocket.BeginSendTo(
buf.Data, buf.Data,
0, 0,
buf.DataLength, buf.DataLength,
SocketFlags.None, SocketFlags.None,
buf.RemoteEndPoint, buf.RemoteEndPoint,
new AsyncCallback(AsyncEndSend), AsyncEndSend,
buf); buf);
} }
catch (SocketException) catch (SocketException) { }
{ catch (ObjectDisposedException) { }
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncBeginSend()",
// Helpers.LogLevel.Error, se);
}
} }
rwLock.ReleaseReaderLock();
} }
private void AsyncEndSend(IAsyncResult iar) void AsyncEndSend(IAsyncResult result)
{ {
rwLock.AcquireReaderLock(-1); try
if (!shutdownFlag)
{ {
UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState; UDPPacketBuffer buf = (UDPPacketBuffer)result.AsyncState;
int bytesSent = udpSocket.EndSendTo(result);
try PacketSent(buf, bytesSent);
{
int bytesSent = udpSocket.EndSendTo(iar);
// note that call to the abstract PacketSent() method - we are passing the number
// of bytes sent in a separate parameter, since we can't use buffer.DataLength which
// is the number of bytes to send (or bytes received depending upon whether this
// buffer was part of a send or a receive).
PacketSent(buffer, bytesSent);
}
catch (SocketException)
{
//Logger.Log(
// "A SocketException occurred in UDPServer.AsyncEndSend()",
// Helpers.LogLevel.Error, se);
}
} }
catch (SocketException) { }
Interlocked.Decrement(ref rwOperationCount); catch (ObjectDisposedException) { }
rwLock.ReleaseReaderLock();
} }
} }
} }