OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The other half of the asset fix. Implement an exponentially incrementing retry timer for asset upload failures. Total queue time in the ballpark of 24 hours, which should be a reasonable time for any grid admin to get their asset service back online. This should stop lost assets.

avinationmerge
Tom Grimshaw 2010-06-29 23:18:48 -07:00
parent 06fa038875
commit 8dc7c0849e
2 changed files with 96 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
OpenSim/Framework/AssetBase.cs
View File

@ -60,6 +60,8 @@ namespace OpenSim.Framework
/// </summary>
private AssetMetadata m_metadata;
private int m_uploadAttempts;
// This is needed for .NET serialization!!!
// Do NOT "Optimize" away!
public AssetBase()
@ -197,6 +199,12 @@ namespace OpenSim.Framework
set { m_metadata.Type = value; }
}
public int UploadAttempts
{
get { return m_uploadAttempts; }
set { m_uploadAttempts = value; }
}
/// <summary>
/// Is this a region only asset, or does this exist on the asset server also
/// </summary>

99
OpenSim/Services/Connectors/Asset/AssetServiceConnector.cs
View File

@ -30,6 +30,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Timers;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
@ -48,7 +49,9 @@ namespace OpenSim.Services.Connectors
private string m_ServerURI = String.Empty;
private IImprovedAssetCache m_Cache = null;
private int m_retryCounter;
private Dictionary<int, List<AssetBase>> m_retryQueue = new Dictionary<int, List<AssetBase>>();
private Timer m_retryTimer;
public AssetServicesConnector()
{
}
@ -85,6 +88,55 @@ namespace OpenSim.Services.Connectors
MainConsole.Instance.Commands.AddCommand("asset", false, "dump asset",
"dump asset <id> <file>",
"dump one cached asset", HandleDumpAsset);
m_retryTimer = new Timer();
m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
m_retryTimer.Interval = 60000;
}
protected void retryCheck(object source, ElapsedEventArgs e)
{
m_retryCounter++;
if (m_retryCounter > 60) m_retryCounter -= 60;
List<int> keys = new List<int>();
foreach (int a in m_retryQueue.Keys)
{
keys.Add(a);
}
foreach (int a in keys)
{
//We exponentially fall back on frequency until we reach one attempt per hour
//The net result is that we end up in the queue for roughly 24 hours..
//24 hours worth of assets could be a lot, so the hope is that the region admin
//will have gotten the asset connector back online quickly!
int timefactor = a ^ 2;
if (timefactor > 60)
{
timefactor = 60;
}
//First, find out if we care about this timefactor
if (timefactor % a == 0)
{
//Yes, we do!
List<AssetBase> retrylist = m_retryQueue[a];
m_retryQueue.Remove(a);
foreach(AssetBase ass in retrylist)
{
Store(ass); //Store my ass. This function will put it back in the dictionary if it fails
}
}
}
if (m_retryQueue.Count == 0)
{
//It might only be one tick per minute, but I have
//repented and abandoned my wasteful ways
m_retryCounter = 0;
m_retryTimer.Stop();
}
}
protected void SetCache(IImprovedAssetCache cache)
@ -222,20 +274,45 @@ namespace OpenSim.Services.Connectors
}
catch (Exception e)
{
m_log.WarnFormat("[ASSET CONNECTOR]: Unable to send asset {0} to asset server. Reason: {1}", asset.ID, e.Message);
newID = UUID.Zero.ToString();
}
if (newID != String.Empty)
if (newID == UUID.Zero.ToString())
{
// Placing this here, so that this work with old asset servers that don't send any reply back
// SynchronousRestObjectRequester returns somethins that is not an empty string
if (newID != null)
asset.ID = newID;
if (m_Cache != null)
m_Cache.Cache(asset);
//The asset upload failed, put it in a queue for later
asset.UploadAttempts++;
if (asset.UploadAttempts > 30)
{
//By this stage we've been in the queue for a good few hours;
//We're going to drop the asset.
m_log.ErrorFormat("[Assets] Dropping asset {0} - Upload has been in the queue for too long.", asset.ID.ToString());
}
else
{
if (!m_retryQueue.ContainsKey(asset.UploadAttempts))
{
m_retryQueue.Add(asset.UploadAttempts, new List<AssetBase>());
}
List<AssetBase> m_queue = m_retryQueue[asset.UploadAttempts];
m_queue.Add(asset);
m_log.WarnFormat("[Assets] Upload failed: {0} - Requeuing asset for another run.", asset.ID.ToString());
m_retryTimer.Start();
}
}
return newID;
else
{
if (newID != String.Empty)
{
// Placing this here, so that this work with old asset servers that don't send any reply back
// SynchronousRestObjectRequester returns somethins that is not an empty string
if (newID != null)
asset.ID = newID;
if (m_Cache != null)
m_Cache.Cache(asset);
}
}
return asset.ID;
}
public bool UpdateContent(string id, byte[] data)