OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

several changes to llHttpRequest processing: options section renamed ScriptsHttpRequestModule; throttle by prim (PrimRequestsBurst = 3, PrimRequestsPerSec = 1) and max concurrent connections per instance (MaxPoolThreads = 5), llhttprequest returns if error, nullkey if throttled, reqid otherwise

0.9.1.0-post-fixes
UbitUmarov 2019-02-24 07:25:50 +00:00
parent 009af3122f
commit 800f6d6529
4 changed files with 117 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
OpenSim/Region/CoreModules/Scripting/HttpRequest/ScriptsHttpRequests.cs
View File

@ -27,6 +27,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.IO;
using System.Net;
using System.Net.Mail;
@ -102,10 +103,21 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
private string m_proxyurl = "";
private string m_proxyexcepts = "";
private float m_primpersec = 1.0f;
private float m_primburst = 3f;
private struct ThrottleData
{
public double lastTime;
public float count;
}
// <request id, HttpRequestClass>
private Dictionary<UUID, HttpRequestClass> m_pendingRequests;
private ConcurrentQueue<HttpRequestClass> m_CompletedRequests;
private ConcurrentDictionary<uint, ThrottleData> m_RequestsThrottle;
private Scene m_scene;
// private Queue<HttpRequestClass> rpcQueue = new Queue<HttpRequestClass>();
public static SmartThreadPool ThreadPool = null;
public HttpRequestModule()
@ -119,6 +131,41 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
return UUID.Zero;
}
public bool CheckThrottle(uint localID)
{
ThrottleData th;
double now = Util.GetTimeStamp();
bool ret = false;
if (m_RequestsThrottle.TryGetValue(localID, out th))
{
double delta = now - th.lastTime;
th.lastTime = now;
float add = (float)(m_primpersec * delta);
th.count += add;
if (th.count > m_primburst)
th.count = m_primburst;
ret = th.count > 0;
}
else
{
th = new ThrottleData()
{
lastTime = now,
count = m_primburst
};
ret = true;
}
if (ret)
th.count--;
m_RequestsThrottle[localID] = th;
return ret;
}
public UUID StartHttpRequest(
uint localID, UUID itemID, string url, List<string> parameters, Dictionary<string, string> headers, string body,
out HttpInitialRequestStatus status)
@ -243,9 +290,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
return false;
lock (HttpListLock)
{
m_pendingRequests.Add(req.ReqID, req);
}
req.Process();
@ -256,14 +301,19 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
{
if (m_pendingRequests != null)
{
List<UUID> toremove = new List<UUID>();
lock (HttpListLock)
{
HttpRequestClass tmpReq;
if (m_pendingRequests.TryGetValue(m_itemID, out tmpReq))
foreach (HttpRequestClass tmpReq in m_pendingRequests.Values)
{
tmpReq.Stop();
m_pendingRequests.Remove(m_itemID);
if(tmpReq.ItemID == m_itemID)
{
tmpReq.Stop();
toremove.Add(tmpReq.ReqID);
}
}
foreach(UUID id in toremove)
m_pendingRequests.Remove(id);
}
}
}
@ -276,37 +326,37 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
* finished. I thought about setting up a queue for this, but
* it will need some refactoring and this works 'enough' right now
*/
public void GotCompletedRequest(HttpRequestClass req)
{
if(req.Removed)
return;
lock (HttpListLock)
{
m_pendingRequests.Remove(req.ReqID);
m_CompletedRequests.Enqueue(req);
}
}
public IServiceRequest GetNextCompletedRequest()
{
lock (HttpListLock)
HttpRequestClass req;
while(m_CompletedRequests.TryDequeue(out req))
{
foreach (UUID luid in m_pendingRequests.Keys)
{
HttpRequestClass tmpReq;
if (m_pendingRequests.TryGetValue(luid, out tmpReq))
{
if (tmpReq.Finished)
{
return tmpReq;
}
}
}
if(!req.Removed)
return req;
}
return null;
}
public void RemoveCompletedRequest(UUID id)
public void RemoveCompletedRequest(UUID reqId)
{
lock (HttpListLock)
{
HttpRequestClass tmpReq;
if (m_pendingRequests.TryGetValue(id, out tmpReq))
if (m_pendingRequests.TryGetValue(reqId, out tmpReq))
{
tmpReq.Stop();
tmpReq = null;
m_pendingRequests.Remove(id);
m_pendingRequests.Remove(reqId);
}
}
}
@ -322,17 +372,20 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
HttpRequestClass.HttpBodyMaxLenMAX = config.Configs["Network"].GetInt("HttpBodyMaxLenMAX", 16384);
m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config);
int maxThreads = 15;
IConfig httpConfig = config.Configs["HttpRequestModule"];
int maxThreads = 5;
IConfig httpConfig = config.Configs["ScriptsHttpRequestModule"];
if (httpConfig != null)
{
maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads);
m_primburst = httpConfig.GetFloat("PrimRequestsBurst", m_primburst);
m_primpersec = httpConfig.GetFloat("PrimRequestsPerSec", m_primpersec);
}
m_pendingRequests = new Dictionary<UUID, HttpRequestClass>();
m_CompletedRequests = new ConcurrentQueue<HttpRequestClass>();
m_RequestsThrottle = new ConcurrentDictionary<uint, ThrottleData>();
// First instance sets this up for all sims
if (ThreadPool == null)
@ -406,11 +459,8 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
/// </summary>
public HttpRequestModule RequestModule { get; set; }
private bool _finished;
public bool Finished
{
get { return _finished; }
}
public bool Finished { get; private set;}
public bool Removed{ get; set;}
public static int HttpBodyMaxLenMAX = 16384;
@ -427,19 +477,10 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public bool HttpPragmaNoCache = true;
// Request info
private UUID _itemID;
public UUID ItemID
{
get { return _itemID; }
set { _itemID = value; }
}
private uint _localID;
public uint LocalID
{
get { return _localID; }
set { _localID = value; }
}
public DateTime Next;
public UUID ReqID { get; set; }
public UUID ItemID { get; set;}
public uint LocalID { get; set;}
public string proxyurl;
public string proxyexcepts;
@ -454,12 +495,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public int MaxRedirects { get; set; }
public string OutboundBody;
private UUID _reqID;
public UUID ReqID
{
get { return _reqID; }
set { _reqID = value; }
}
public HttpWebRequest Request;
public string ResponseBody;
public List<string> ResponseMetadata;
@ -469,10 +505,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public void Process()
{
_finished = false;
lock (HttpRequestModule.ThreadPool)
WorkItem = HttpRequestModule.ThreadPool.QueueWorkItem(new WorkItemCallback(StpSendWrapper), null);
WorkItem = HttpRequestModule.ThreadPool.QueueWorkItem(new WorkItemCallback(StpSendWrapper), null);
}
private object StpSendWrapper(object o)
@ -521,6 +554,9 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
public void SendRequest()
{
if(Removed)
return;
HttpWebResponse response = null;
Stream resStream = null;
byte[] buf = new byte[HttpBodyMaxLenMAX + 16];
@ -672,7 +708,6 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
if (response != null)
response.Close();
// We need to resubmit
if (
(Status == (int)HttpStatusCode.MovedPermanently
@ -684,7 +719,8 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
{
Status = (int)OSHttpStatusCode.ClientErrorJoker;
ResponseBody = "Number of redirects exceeded max redirects";
_finished = true;
WorkItem = null;
RequestModule.GotCompletedRequest(this);
}
else
{
@ -694,13 +730,15 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
{
Status = (int)OSHttpStatusCode.ClientErrorJoker;
ResponseBody = "HTTP redirect code but no location header";
_finished = true;
WorkItem = null;
RequestModule.GotCompletedRequest(this);
}
else if (!RequestModule.CheckAllowed(new Uri(location)))
{
Status = (int)OSHttpStatusCode.ClientErrorJoker;
ResponseBody = "URL from HTTP redirect blocked: " + location;
_finished = true;
WorkItem = null;
RequestModule.GotCompletedRequest(this);
}
else
{
@ -717,9 +755,10 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
}
else
{
_finished = true;
WorkItem = null;
if (ResponseBody == null)
ResponseBody = String.Empty;
RequestModule.GotCompletedRequest(this);
}
}
}
@ -728,10 +767,12 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
{
try
{
Removed = true;
if(WorkItem == null)
return;
if (!WorkItem.Cancel())
{
WorkItem.Cancel(true);
}
}
catch (Exception)
{

1
OpenSim/Region/Framework/Interfaces/IHttpRequests.cs
View File

@ -87,5 +87,6 @@ namespace OpenSim.Region.Framework.Interfaces
void StopHttpRequest(uint m_localID, UUID m_itemID);
IServiceRequest GetNextCompletedRequest();
void RemoveCompletedRequest(UUID id);
bool CheckThrottle(uint localID);
}
}

19
OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs
View File

@ -13945,14 +13945,14 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
public LSL_Key llHTTPRequest(string url, LSL_List parameters, string body)
{
// Partial implementation: support for parameter flags needed
// see http://wiki.secondlife.com/wiki/LlHTTPRequest
// parameter flags support are implemented in ScriptsHttpRequests.cs
// in StartHttpRequest
m_host.AddScriptLPS(1);
IHttpRequestModule httpScriptMod =
m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
IHttpRequestModule httpScriptMod = m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
if(httpScriptMod == null)
return "";
if(!httpScriptMod.CheckThrottle(m_host.LocalId))
return UUID.Zero.ToString();
List<string> param = new List<string>();
bool ok;
Int32 flag;
@ -14123,8 +14123,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
}
HttpInitialRequestStatus status;
UUID reqID
= httpScriptMod.StartHttpRequest(m_host.LocalId, m_item.ItemID, url, param, httpHeaders, body, out status);
UUID reqID = httpScriptMod.StartHttpRequest(m_host.LocalId, m_item.ItemID, url, param, httpHeaders, body, out status);
if (status == HttpInitialRequestStatus.DISALLOWED_BY_FILTER)
Error("llHttpRequest", string.Format("Request to {0} disallowed by filter", url));
@ -14132,7 +14131,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
if (reqID != UUID.Zero)
return reqID.ToString();
else
return null;
return "";
}

13
OpenSim/Region/ScriptEngine/Shared/Api/Implementation/Plugins/HttpRequest.cs
View File

@ -48,14 +48,11 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api.Plugins
if (m_CmdManager.m_ScriptEngine.World == null)
return;
IHttpRequestModule iHttpReq =
m_CmdManager.m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
HttpRequestClass httpInfo = null;
if (iHttpReq != null)
httpInfo = (HttpRequestClass)iHttpReq.GetNextCompletedRequest();
IHttpRequestModule iHttpReq = m_CmdManager.m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
if(iHttpReq == null)
return;
HttpRequestClass httpInfo = (HttpRequestClass)iHttpReq.GetNextCompletedRequest();
while (httpInfo != null)
{
//m_log.Debug("[AsyncLSL]:" + httpInfo.response_body + httpInfo.status);
@ -67,8 +64,6 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api.Plugins
// implemented here yet anyway. Should be fixed if/when maxsize
// is supported
iHttpReq.RemoveCompletedRequest(httpInfo.ReqID);
object[] resobj = new object[]
{
new LSL_Types.LSLString(httpInfo.ReqID.ToString()),