OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

also throttle llhttprequest by owner, options: PrimOwnerRequestsBurst = 5, PrimOwnerRequestsPerSec = 25; increase concurrency to 8

0.9.1.0-post-fixes
UbitUmarov 2019-02-24 09:46:55 +00:00
parent 800f6d6529
commit cd5a6daa84
3 changed files with 87 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
OpenSim/Region/CoreModules/Scripting/HttpRequest/ScriptsHttpRequests.cs
View File

@ -95,16 +95,18 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
{ {
// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); // private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private object HttpListLock = new object(); private object m_httpListLock = new object();
private int httpTimeout = 30000; private int m_httpTimeout = 30000;
private string m_name = "HttpScriptRequests"; private string m_name = "HttpScriptRequests";
private OutboundUrlFilter m_outboundUrlFilter; private OutboundUrlFilter m_outboundUrlFilter;
private string m_proxyurl = ""; private string m_proxyurl = "";
private string m_proxyexcepts = ""; private string m_proxyexcepts = "";
private float m_primpersec = 1.0f; private float m_primPerSec = 1.0f;
private float m_primburst = 3f; private float m_primBurst = 3.0f;
private float m_primOwnerPerSec = 25.0f;
private float m_primOwnerBurst = 5.0f;
private struct ThrottleData private struct ThrottleData
{ {
@ -116,8 +118,8 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
private Dictionary<UUID, HttpRequestClass> m_pendingRequests; private Dictionary<UUID, HttpRequestClass> m_pendingRequests;
private ConcurrentQueue<HttpRequestClass> m_CompletedRequests; private ConcurrentQueue<HttpRequestClass> m_CompletedRequests;
private ConcurrentDictionary<uint, ThrottleData> m_RequestsThrottle; private ConcurrentDictionary<uint, ThrottleData> m_RequestsThrottle;
private ConcurrentDictionary<UUID, ThrottleData> m_OwnerRequestsThrottle;
private Scene m_scene;
public static SmartThreadPool ThreadPool = null; public static SmartThreadPool ThreadPool = null;
public HttpRequestModule() public HttpRequestModule()
@ -131,38 +133,64 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
return UUID.Zero; return UUID.Zero;
} }
public bool CheckThrottle(uint localID) public bool CheckThrottle(uint localID, UUID ownerID)
{ {
ThrottleData th; ThrottleData th;
double now = Util.GetTimeStamp(); double now = Util.GetTimeStamp();
bool ret = false; bool ret;
if (m_RequestsThrottle.TryGetValue(localID, out th)) if (m_RequestsThrottle.TryGetValue(localID, out th))
{ {
double delta = now - th.lastTime; double delta = now - th.lastTime;
th.lastTime = now; th.lastTime = now;
float add = (float)(m_primpersec * delta); float add = (float)(m_primPerSec * delta);
th.count += add; th.count += add;
if (th.count > m_primburst) if (th.count > m_primBurst)
th.count = m_primburst; th.count = m_primBurst;
ret = th.count > 0; ret = th.count > 0;
if (ret)
th.count--;
} }
else else
{ {
th = new ThrottleData() th = new ThrottleData()
{ {
lastTime = now, lastTime = now,
count = m_primburst count = m_primBurst - 1
}; };
ret = true; ret = true;
} }
m_RequestsThrottle[localID] = th;
if(!ret)
return false;
if (m_OwnerRequestsThrottle.TryGetValue(ownerID, out th))
{
double delta = now - th.lastTime;
th.lastTime = now;
float add = (float)(m_primOwnerPerSec * delta);
th.count += add;
if (th.count > m_primOwnerBurst)
th.count = m_primOwnerBurst;
ret = th.count > 0;
if (ret) if (ret)
th.count--; th.count--;
}
else
{
th = new ThrottleData()
{
lastTime = now,
count = m_primOwnerBurst - 1
};
}
m_OwnerRequestsThrottle[ownerID] = th;
m_RequestsThrottle[localID] = th;
return ret; return ret;
} }
@ -170,6 +198,12 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
uint localID, UUID itemID, string url, List<string> parameters, Dictionary<string, string> headers, string body, uint localID, UUID itemID, string url, List<string> parameters, Dictionary<string, string> headers, string body,
out HttpInitialRequestStatus status) out HttpInitialRequestStatus status)
{ {
if (!CheckAllowed(new Uri(url)))
{
status = HttpInitialRequestStatus.DISALLOWED_BY_FILTER;
return UUID.Zero;
}
UUID reqID = UUID.Random(); UUID reqID = UUID.Random();
HttpRequestClass htc = new HttpRequestClass(); HttpRequestClass htc = new HttpRequestClass();
@ -254,7 +288,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
htc.ItemID = itemID; htc.ItemID = itemID;
htc.Url = url; htc.Url = url;
htc.ReqID = reqID; htc.ReqID = reqID;
htc.HttpTimeout = httpTimeout; htc.HttpTimeout = m_httpTimeout;
htc.OutboundBody = body; htc.OutboundBody = body;
htc.ResponseHeaders = headers; htc.ResponseHeaders = headers;
htc.proxyurl = m_proxyurl; htc.proxyurl = m_proxyurl;
@ -263,16 +297,12 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
// Same number as default HttpWebRequest.MaximumAutomaticRedirections // Same number as default HttpWebRequest.MaximumAutomaticRedirections
htc.MaxRedirects = 50; htc.MaxRedirects = 50;
if (StartHttpRequest(htc)) lock (m_httpListLock)
{ m_pendingRequests.Add(reqID, htc);
htc.Process();
status = HttpInitialRequestStatus.OK; status = HttpInitialRequestStatus.OK;
return htc.ReqID; return reqID;
}
else
{
status = HttpInitialRequestStatus.DISALLOWED_BY_FILTER;
return UUID.Zero;
}
} }
/// <summary> /// <summary>
@ -284,25 +314,10 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
return m_outboundUrlFilter.CheckAllowed(url); return m_outboundUrlFilter.CheckAllowed(url);
} }
public bool StartHttpRequest(HttpRequestClass req)
{
if (!CheckAllowed(new Uri(req.Url)))
return false;
lock (HttpListLock)
m_pendingRequests.Add(req.ReqID, req);
req.Process();
return true;
}
public void StopHttpRequest(uint m_localID, UUID m_itemID) public void StopHttpRequest(uint m_localID, UUID m_itemID)
{
if (m_pendingRequests != null)
{ {
List<UUID> toremove = new List<UUID>(); List<UUID> toremove = new List<UUID>();
lock (HttpListLock) lock (m_httpListLock)
{ {
foreach (HttpRequestClass tmpReq in m_pendingRequests.Values) foreach (HttpRequestClass tmpReq in m_pendingRequests.Values)
{ {
@ -316,7 +331,6 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
m_pendingRequests.Remove(id); m_pendingRequests.Remove(id);
} }
} }
}
/* /*
* TODO * TODO
@ -327,11 +341,11 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
* it will need some refactoring and this works 'enough' right now * it will need some refactoring and this works 'enough' right now
*/ */
public void GotCompletedRequest(HttpRequestClass req) public void GotCompletedRequest(HttpRequestClass req)
{
lock (m_httpListLock)
{ {
if (req.Removed) if (req.Removed)
return; return;
lock (HttpListLock)
{
m_pendingRequests.Remove(req.ReqID); m_pendingRequests.Remove(req.ReqID);
m_CompletedRequests.Enqueue(req); m_CompletedRequests.Enqueue(req);
} }
@ -340,17 +354,15 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public IServiceRequest GetNextCompletedRequest() public IServiceRequest GetNextCompletedRequest()
{ {
HttpRequestClass req; HttpRequestClass req;
while(m_CompletedRequests.TryDequeue(out req)) if(m_CompletedRequests.TryDequeue(out req))
{
if(!req.Removed)
return req; return req;
}
return null; return null;
} }
public void RemoveCompletedRequest(UUID reqId) public void RemoveCompletedRequest(UUID reqId)
{ {
lock (HttpListLock) lock (m_httpListLock)
{ {
HttpRequestClass tmpReq; HttpRequestClass tmpReq;
if (m_pendingRequests.TryGetValue(reqId, out tmpReq)) if (m_pendingRequests.TryGetValue(reqId, out tmpReq))
@ -374,18 +386,26 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config); m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config);
int maxThreads = 5; int maxThreads = 8;
IConfig httpConfig = config.Configs["ScriptsHttpRequestModule"]; IConfig httpConfig = config.Configs["ScriptsHttpRequestModule"];
if (httpConfig != null) if (httpConfig != null)
{ {
maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads); maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads);
m_primburst = httpConfig.GetFloat("PrimRequestsBurst", m_primburst); m_primBurst = httpConfig.GetFloat("PrimRequestsBurst", m_primBurst);
m_primpersec = httpConfig.GetFloat("PrimRequestsPerSec", m_primpersec); m_primPerSec = httpConfig.GetFloat("PrimRequestsPerSec", m_primPerSec);
m_primOwnerBurst = httpConfig.GetFloat("PrimOwnerRequestsBurst", m_primOwnerBurst);
m_primOwnerPerSec = httpConfig.GetFloat("PrimOwnerRequestsPerSec", m_primOwnerPerSec);
m_httpTimeout = httpConfig.GetInt("RequestsTimeOut", m_httpTimeout);
if(m_httpTimeout > 60000)
m_httpTimeout = 60000;
else if(m_httpTimeout < 200)
m_httpTimeout = 200;
} }
m_pendingRequests = new Dictionary<UUID, HttpRequestClass>(); m_pendingRequests = new Dictionary<UUID, HttpRequestClass>();
m_CompletedRequests = new ConcurrentQueue<HttpRequestClass>(); m_CompletedRequests = new ConcurrentQueue<HttpRequestClass>();
m_RequestsThrottle = new ConcurrentDictionary<uint, ThrottleData>(); m_RequestsThrottle = new ConcurrentDictionary<uint, ThrottleData>();
m_OwnerRequestsThrottle = new ConcurrentDictionary<UUID, ThrottleData>();
// First instance sets this up for all sims // First instance sets this up for all sims
if (ThreadPool == null) if (ThreadPool == null)
@ -405,16 +425,12 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public void AddRegion(Scene scene) public void AddRegion(Scene scene)
{ {
m_scene = scene; scene.RegisterModuleInterface<IHttpRequestModule>(this);
m_scene.RegisterModuleInterface<IHttpRequestModule>(this);
} }
public void RemoveRegion(Scene scene) public void RemoveRegion(Scene scene)
{ {
scene.UnregisterModuleInterface<IHttpRequestModule>(this); scene.UnregisterModuleInterface<IHttpRequestModule>(this);
if (scene == m_scene)
m_scene = null;
} }
public void PostInitialise() public void PostInitialise()
@ -570,6 +586,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
Request.AllowAutoRedirect = false; Request.AllowAutoRedirect = false;
Request.KeepAlive = false; Request.KeepAlive = false;
Request.Timeout = HttpTimeout;
//This works around some buggy HTTP Servers like Lighttpd //This works around some buggy HTTP Servers like Lighttpd
Request.ServicePoint.Expect100Continue = false; Request.ServicePoint.Expect100Continue = false;
@ -629,7 +646,6 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
bstream.Write(data, 0, data.Length); bstream.Write(data, 0, data.Length);
} }
Request.Timeout = HttpTimeout;
try try
{ {
// execute the request // execute the request

2
OpenSim/Region/Framework/Interfaces/IHttpRequests.cs
View File

@ -87,6 +87,6 @@ namespace OpenSim.Region.Framework.Interfaces
void StopHttpRequest(uint m_localID, UUID m_itemID); void StopHttpRequest(uint m_localID, UUID m_itemID);
IServiceRequest GetNextCompletedRequest(); IServiceRequest GetNextCompletedRequest();
void RemoveCompletedRequest(UUID id); void RemoveCompletedRequest(UUID id);
bool CheckThrottle(uint localID); bool CheckThrottle(uint localID, UUID onerID);
} }
} }

2
OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
View File

@ -13950,7 +13950,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
if(httpScriptMod == null) if(httpScriptMod == null)
return ""; return "";
if(!httpScriptMod.CheckThrottle(m_host.LocalId)) if(!httpScriptMod.CheckThrottle(m_host.LocalId, m_host.OwnerID))
return UUID.Zero.ToString(); return UUID.Zero.ToString();
List<string> param = new List<string>(); List<string> param = new List<string>();