2009-06-09 17:48:22 +00:00
/ *
2009-08-28 17:12:30 +00:00
* Copyright ( c ) Contributors , http : //opensimulator.org/
* See CONTRIBUTORS . TXT for a full list of copyright holders .
*
* Redistribution and use in source and binary forms , with or without
* modification , are permitted provided that the following conditions are met :
* * Redistributions of source code must retain the above copyright
* notice , this list of conditions and the following disclaimer .
* * Redistributions in binary form must reproduce the above copyright
* notice , this list of conditions and the following disclaimer in the
* documentation and / or other materials provided with the distribution .
* * Neither the name of the OpenSimulator Project nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission .
*
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ` ` AS IS ' ' AND ANY
* EXPRESS OR IMPLIED WARRANTIES , INCLUDING , BUT NOT LIMITED TO , THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED . IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT , INDIRECT , INCIDENTAL , SPECIAL , EXEMPLARY , OR CONSEQUENTIAL DAMAGES
* ( INCLUDING , BUT NOT LIMITED TO , PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES ;
* LOSS OF USE , DATA , OR PROFITS ; OR BUSINESS INTERRUPTION ) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY , WHETHER IN CONTRACT , STRICT LIABILITY , OR TORT
* ( INCLUDING NEGLIGENCE OR OTHERWISE ) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE , EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE .
* /
2009-06-09 17:48:22 +00:00
// Uncomment to make asset Get requests for existing
// #define WAIT_ON_INPROGRESS_REQUESTS
using System ;
using System.IO ;
using System.Collections.Generic ;
2013-10-30 18:57:51 +00:00
using System.Linq ;
2009-06-09 17:48:22 +00:00
using System.Reflection ;
using System.Runtime.Serialization ;
using System.Runtime.Serialization.Formatters.Binary ;
using System.Threading ;
using System.Timers ;
using log4net ;
using Nini.Config ;
using Mono.Addins ;
using OpenMetaverse ;
using OpenSim.Framework ;
2009-10-12 22:21:32 +00:00
using OpenSim.Framework.Console ;
2009-06-09 17:48:22 +00:00
using OpenSim.Region.Framework.Interfaces ;
using OpenSim.Region.Framework.Scenes ;
using OpenSim.Services.Interfaces ;
2012-11-10 23:54:11 +00:00
//[assembly: Addin("FlotsamAssetCache", "1.1")]
//[assembly: AddinDependency("OpenSim", "0.5")]
2012-09-09 17:50:44 +00:00
2012-09-09 18:30:59 +00:00
namespace OpenSim.Region.CoreModules.Asset
2009-06-09 17:48:22 +00:00
{
2012-11-13 02:08:02 +00:00
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")]
2009-10-12 22:21:32 +00:00
public class FlotsamAssetCache : ISharedRegionModule , IImprovedAssetCache , IAssetService
2009-06-09 17:48:22 +00:00
{
private static readonly ILog m_log =
LogManager . GetLogger (
MethodBase . GetCurrentMethod ( ) . DeclaringType ) ;
2009-10-20 17:33:23 +00:00
private bool m_Enabled ;
2009-06-09 17:48:22 +00:00
private const string m_ModuleName = "FlotsamAssetCache" ;
2011-06-10 23:04:21 +00:00
private const string m_DefaultCacheDirectory = "./assetcache" ;
2009-06-09 17:48:22 +00:00
private string m_CacheDirectory = m_DefaultCacheDirectory ;
2009-10-20 17:33:23 +00:00
private readonly List < char > m_InvalidChars = new List < char > ( ) ;
2009-06-09 17:48:22 +00:00
2010-02-19 21:18:03 +00:00
private int m_LogLevel = 0 ;
2011-06-10 23:04:21 +00:00
private ulong m_HitRateDisplay = 100 ; // How often to display hit statistics, given in requests
2009-06-09 17:48:22 +00:00
2009-10-20 17:33:23 +00:00
private static ulong m_Requests ;
private static ulong m_RequestsForInprogress ;
private static ulong m_DiskHits ;
private static ulong m_MemoryHits ;
2009-06-09 17:48:22 +00:00
#if WAIT_ON_INPROGRESS_REQUESTS
private Dictionary < string , ManualResetEvent > m_CurrentlyWriting = new Dictionary < string , ManualResetEvent > ( ) ;
private int m_WaitOnInprogressTimeout = 3000 ;
# else
2012-05-04 22:03:33 +00:00
private HashSet < string > m_CurrentlyWriting = new HashSet < string > ( ) ;
2009-06-09 17:48:22 +00:00
# endif
2011-07-04 21:51:47 +00:00
private bool m_FileCacheEnabled = true ;
2009-10-20 17:33:23 +00:00
private ExpiringCache < string , AssetBase > m_MemoryCache ;
2011-06-10 23:04:21 +00:00
private bool m_MemoryCacheEnabled = false ;
2009-06-09 17:48:22 +00:00
// Expiration is expressed in hours.
2011-06-10 23:04:21 +00:00
private const double m_DefaultMemoryExpiration = 2 ;
2009-06-09 17:48:22 +00:00
private const double m_DefaultFileExpiration = 48 ;
2011-02-25 01:25:38 +00:00
private TimeSpan m_MemoryExpiration = TimeSpan . FromHours ( m_DefaultMemoryExpiration ) ;
2011-02-25 01:31:38 +00:00
private TimeSpan m_FileExpiration = TimeSpan . FromHours ( m_DefaultFileExpiration ) ;
2011-06-10 23:04:21 +00:00
private TimeSpan m_FileExpirationCleanupTimer = TimeSpan . FromHours ( 0.166 ) ;
2009-06-09 17:48:22 +00:00
2009-07-28 17:20:51 +00:00
private static int m_CacheDirectoryTiers = 1 ;
2009-07-29 17:21:36 +00:00
private static int m_CacheDirectoryTierLen = 3 ;
2009-07-28 17:20:51 +00:00
private static int m_CacheWarnAt = 30000 ;
2009-10-20 17:33:23 +00:00
private System . Timers . Timer m_CacheCleanTimer ;
2009-06-09 17:48:22 +00:00
2009-10-20 17:33:23 +00:00
private IAssetService m_AssetService ;
2009-10-12 22:21:32 +00:00
private List < Scene > m_Scenes = new List < Scene > ( ) ;
2009-06-09 17:48:22 +00:00
public FlotsamAssetCache ( )
{
m_InvalidChars . AddRange ( Path . GetInvalidPathChars ( ) ) ;
m_InvalidChars . AddRange ( Path . GetInvalidFileNameChars ( ) ) ;
}
2009-08-10 22:08:22 +00:00
public Type ReplaceableInterface
2009-07-10 20:17:13 +00:00
{
get { return null ; }
}
2009-06-09 17:48:22 +00:00
public string Name
{
get { return m_ModuleName ; }
}
public void Initialise ( IConfigSource source )
{
IConfig moduleConfig = source . Configs [ "Modules" ] ;
2009-10-12 22:21:32 +00:00
2009-06-09 17:48:22 +00:00
if ( moduleConfig ! = null )
{
2009-10-20 17:33:23 +00:00
string name = moduleConfig . GetString ( "AssetCaching" , String . Empty ) ;
2009-06-09 17:48:22 +00:00
if ( name = = Name )
{
2009-10-20 17:33:23 +00:00
m_MemoryCache = new ExpiringCache < string , AssetBase > ( ) ;
2009-06-09 20:20:30 +00:00
m_Enabled = true ;
2009-10-20 17:33:23 +00:00
2009-09-08 20:31:09 +00:00
m_log . InfoFormat ( "[FLOTSAM ASSET CACHE]: {0} enabled" , this . Name ) ;
2009-06-09 20:20:30 +00:00
2009-06-09 17:48:22 +00:00
IConfig assetConfig = source . Configs [ "AssetCache" ] ;
if ( assetConfig = = null )
{
2012-06-20 00:28:55 +00:00
m_log . Debug (
2011-06-10 23:04:21 +00:00
"[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults." ) ;
2009-06-09 17:48:22 +00:00
}
2011-06-10 23:04:21 +00:00
else
{
2011-07-04 21:51:47 +00:00
m_FileCacheEnabled = assetConfig . GetBoolean ( "FileCacheEnabled" , m_FileCacheEnabled ) ;
2011-06-10 23:04:21 +00:00
m_CacheDirectory = assetConfig . GetString ( "CacheDirectory" , m_DefaultCacheDirectory ) ;
2009-06-09 17:48:22 +00:00
2011-06-10 23:04:21 +00:00
m_MemoryCacheEnabled = assetConfig . GetBoolean ( "MemoryCacheEnabled" , m_MemoryCacheEnabled ) ;
m_MemoryExpiration = TimeSpan . FromHours ( assetConfig . GetDouble ( "MemoryCacheTimeout" , m_DefaultMemoryExpiration ) ) ;
#if WAIT_ON_INPROGRESS_REQUESTS
m_WaitOnInprogressTimeout = assetConfig . GetInt ( "WaitOnInprogressTimeout" , 3000 ) ;
# endif
m_LogLevel = assetConfig . GetInt ( "LogLevel" , m_LogLevel ) ;
m_HitRateDisplay = ( ulong ) assetConfig . GetLong ( "HitRateDisplay" , ( long ) m_HitRateDisplay ) ;
2009-06-09 17:48:22 +00:00
2011-06-10 23:04:21 +00:00
m_FileExpiration = TimeSpan . FromHours ( assetConfig . GetDouble ( "FileCacheTimeout" , m_DefaultFileExpiration ) ) ;
m_FileExpirationCleanupTimer
= TimeSpan . FromHours (
assetConfig . GetDouble ( "FileCleanupTimer" , m_FileExpirationCleanupTimer . TotalHours ) ) ;
2009-06-09 17:48:22 +00:00
2011-06-10 23:04:21 +00:00
m_CacheDirectoryTiers = assetConfig . GetInt ( "CacheDirectoryTiers" , m_CacheDirectoryTiers ) ;
m_CacheDirectoryTierLen = assetConfig . GetInt ( "CacheDirectoryTierLength" , m_CacheDirectoryTierLen ) ;
m_CacheWarnAt = assetConfig . GetInt ( "CacheWarnAt" , m_CacheWarnAt ) ;
}
m_log . InfoFormat ( "[FLOTSAM ASSET CACHE]: Cache Directory {0}" , m_CacheDirectory ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled & & ( m_FileExpiration > TimeSpan . Zero ) & & ( m_FileExpirationCleanupTimer > TimeSpan . Zero ) )
2009-06-09 17:48:22 +00:00
{
2009-10-20 17:33:23 +00:00
m_CacheCleanTimer = new System . Timers . Timer ( m_FileExpirationCleanupTimer . TotalMilliseconds ) ;
m_CacheCleanTimer . AutoReset = true ;
m_CacheCleanTimer . Elapsed + = CleanupExpiredFiles ;
lock ( m_CacheCleanTimer )
m_CacheCleanTimer . Start ( ) ;
2009-06-09 17:48:22 +00:00
}
2009-07-28 17:20:51 +00:00
if ( m_CacheDirectoryTiers < 1 )
{
m_CacheDirectoryTiers = 1 ;
}
else if ( m_CacheDirectoryTiers > 3 )
{
m_CacheDirectoryTiers = 3 ;
}
if ( m_CacheDirectoryTierLen < 1 )
{
m_CacheDirectoryTierLen = 1 ;
}
else if ( m_CacheDirectoryTierLen > 4 )
{
m_CacheDirectoryTierLen = 4 ;
}
2012-03-08 01:51:37 +00:00
MainConsole . Instance . Commands . AddCommand ( "Assets" , true , "fcache status" , "fcache status" , "Display cache status" , HandleConsoleCommand ) ;
MainConsole . Instance . Commands . AddCommand ( "Assets" , true , "fcache clear" , "fcache clear [file] [memory]" , "Remove all assets in the cache. If file or memory is specified then only this cache is cleared." , HandleConsoleCommand ) ;
MainConsole . Instance . Commands . AddCommand ( "Assets" , true , "fcache assets" , "fcache assets" , "Attempt a deep scan and cache of all assets in all scenes" , HandleConsoleCommand ) ;
MainConsole . Instance . Commands . AddCommand ( "Assets" , true , "fcache expire" , "fcache expire <datetime>" , "Purge cached assets older then the specified date/time" , HandleConsoleCommand ) ;
2009-06-09 17:48:22 +00:00
}
}
}
public void PostInitialise ( )
{
}
public void Close ( )
{
}
public void AddRegion ( Scene scene )
{
if ( m_Enabled )
2009-09-08 20:31:09 +00:00
{
2009-06-09 17:48:22 +00:00
scene . RegisterModuleInterface < IImprovedAssetCache > ( this ) ;
2009-10-12 22:21:32 +00:00
m_Scenes . Add ( scene ) ;
2009-09-08 20:31:09 +00:00
}
2009-06-09 17:48:22 +00:00
}
public void RemoveRegion ( Scene scene )
{
2009-10-12 22:21:32 +00:00
if ( m_Enabled )
{
scene . UnregisterModuleInterface < IImprovedAssetCache > ( this ) ;
m_Scenes . Remove ( scene ) ;
}
2009-06-09 17:48:22 +00:00
}
public void RegionLoaded ( Scene scene )
{
2012-11-14 15:50:19 +00:00
if ( m_Enabled & & m_AssetService = = null )
m_AssetService = scene . RequestModuleInterface < IAssetService > ( ) ;
2009-06-09 17:48:22 +00:00
}
////////////////////////////////////////////////////////////
// IImprovedAssetCache
/ /
private void UpdateMemoryCache ( string key , AssetBase asset )
{
2011-07-04 21:30:18 +00:00
m_MemoryCache . AddOrUpdate ( key , asset , m_MemoryExpiration ) ;
2009-06-09 17:48:22 +00:00
}
2011-07-04 21:30:18 +00:00
private void UpdateFileCache ( string key , AssetBase asset )
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:30:18 +00:00
string filename = GetFileName ( asset . ID ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
try
{
2012-05-04 21:57:33 +00:00
// If the file is already cached just update access time.
2011-07-04 21:30:18 +00:00
if ( File . Exists ( filename ) )
{
2012-05-04 21:57:33 +00:00
lock ( m_CurrentlyWriting )
{
if ( ! m_CurrentlyWriting . Contains ( filename ) )
File . SetLastAccessTime ( filename , DateTime . Now ) ;
}
2011-07-04 21:30:18 +00:00
}
else
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:30:18 +00:00
// Once we start writing, make sure we flag that we're writing
// that object to the cache so that we don't try to write the
// same file multiple times.
lock ( m_CurrentlyWriting )
2009-06-09 17:48:22 +00:00
{
#if WAIT_ON_INPROGRESS_REQUESTS
2011-07-04 21:30:18 +00:00
if ( m_CurrentlyWriting . ContainsKey ( filename ) )
{
return ;
}
else
{
m_CurrentlyWriting . Add ( filename , new ManualResetEvent ( false ) ) ;
}
2009-06-09 17:48:22 +00:00
# else
2011-07-04 21:30:18 +00:00
if ( m_CurrentlyWriting . Contains ( filename ) )
{
return ;
2009-06-09 17:48:22 +00:00
}
2011-07-04 21:30:18 +00:00
else
{
m_CurrentlyWriting . Add ( filename ) ;
}
# endif
2009-06-09 17:48:22 +00:00
}
2011-07-04 21:30:18 +00:00
Util . FireAndForget (
delegate { WriteFileCache ( filename , asset ) ; } ) ;
2009-06-09 17:48:22 +00:00
}
2011-07-04 21:30:18 +00:00
}
catch ( Exception e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}" ,
asset . ID , e . Message , e . StackTrace ) ;
2009-06-09 17:48:22 +00:00
}
}
2011-07-04 21:30:18 +00:00
public void Cache ( AssetBase asset )
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:30:18 +00:00
// TODO: Spawn this off to some seperate thread to do the actual writing
if ( asset ! = null )
{
//m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID);
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
if ( m_MemoryCacheEnabled )
UpdateMemoryCache ( asset . ID , asset ) ;
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
UpdateFileCache ( asset . ID , asset ) ;
2011-07-04 21:30:18 +00:00
}
}
/// <summary>
/// Try to get an asset from the in-memory cache.
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
private AssetBase GetFromMemoryCache ( string id )
{
2009-06-09 17:48:22 +00:00
AssetBase asset = null ;
2011-07-04 21:30:18 +00:00
if ( m_MemoryCache . TryGetValue ( id , out asset ) )
2009-06-09 17:48:22 +00:00
m_MemoryHits + + ;
2011-07-04 21:30:18 +00:00
return asset ;
}
/// <summary>
/// Try to get an asset from the file cache.
/// </summary>
/// <param name="id"></param>
2013-02-08 21:21:20 +00:00
/// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns>
2011-07-04 21:30:18 +00:00
private AssetBase GetFromFileCache ( string id )
2013-03-30 01:21:16 +00:00
{
2011-07-04 21:30:18 +00:00
string filename = GetFileName ( id ) ;
2013-02-08 21:21:20 +00:00
2013-03-30 01:21:16 +00:00
#if WAIT_ON_INPROGRESS_REQUESTS
// Check if we're already downloading this asset. If so, try to wait for it to
// download.
if ( m_WaitOnInprogressTimeout > 0 )
{
m_RequestsForInprogress + + ;
ManualResetEvent waitEvent ;
if ( m_CurrentlyWriting . TryGetValue ( filename , out waitEvent ) )
{
waitEvent . WaitOne ( m_WaitOnInprogressTimeout ) ;
return Get ( id ) ;
}
}
# else
// Track how often we have the problem that an asset is requested while
// it is still being downloaded by a previous request.
if ( m_CurrentlyWriting . Contains ( filename ) )
{
m_RequestsForInprogress + + ;
return null ;
}
# endif
AssetBase asset = null ;
2011-07-04 21:30:18 +00:00
if ( File . Exists ( filename ) )
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:30:18 +00:00
FileStream stream = null ;
try
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:30:18 +00:00
stream = File . Open ( filename , FileMode . Open , FileAccess . Read , FileShare . Read ) ;
BinaryFormatter bformatter = new BinaryFormatter ( ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
asset = ( AssetBase ) bformatter . Deserialize ( stream ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
m_DiskHits + + ;
}
catch ( System . Runtime . Serialization . SerializationException e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}" ,
filename , id , e . Message , e . StackTrace ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
// If there was a problem deserializing the asset, the asset may
// either be corrupted OR was serialized under an old format
// {different version of AssetBase} -- we should attempt to
// delete it and re-cache
File . Delete ( filename ) ;
}
catch ( Exception e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}" ,
filename , id , e . Message , e . StackTrace ) ;
2009-06-09 17:48:22 +00:00
}
2011-07-04 21:30:18 +00:00
finally
{
if ( stream ! = null )
stream . Close ( ) ;
}
}
2009-06-09 17:48:22 +00:00
2011-07-04 21:30:18 +00:00
return asset ;
}
public AssetBase Get ( string id )
{
m_Requests + + ;
AssetBase asset = null ;
if ( m_MemoryCacheEnabled )
asset = GetFromMemoryCache ( id ) ;
2011-08-13 14:16:43 +00:00
2011-08-02 06:14:20 +00:00
if ( asset = = null & & m_FileCacheEnabled )
2011-08-13 14:16:43 +00:00
{
2011-07-04 21:30:18 +00:00
asset = GetFromFileCache ( id ) ;
2009-06-09 17:48:22 +00:00
2011-08-13 14:16:43 +00:00
if ( m_MemoryCacheEnabled & & asset ! = null )
UpdateMemoryCache ( id , asset ) ;
}
2009-06-09 17:48:22 +00:00
if ( ( ( m_LogLevel > = 1 ) ) & & ( m_HitRateDisplay ! = 0 ) & & ( m_Requests % m_HitRateDisplay = = 0 ) )
{
2009-09-08 20:31:09 +00:00
m_log . InfoFormat ( "[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}" , id , asset = = null ? "Miss" : "Hit" ) ;
2009-06-09 17:48:22 +00:00
2013-10-30 18:57:51 +00:00
GenerateCacheHitReport ( ) . ForEach ( l = > m_log . InfoFormat ( "[FLOTSAM ASSET CACHE]: {0}" , l ) ) ;
2009-06-09 17:48:22 +00:00
}
return asset ;
}
2010-04-08 19:31:44 +00:00
public AssetBase GetCached ( string id )
{
return Get ( id ) ;
}
2009-06-09 17:48:22 +00:00
public void Expire ( string id )
{
if ( m_LogLevel > = 2 )
2011-07-30 00:35:22 +00:00
m_log . DebugFormat ( "[FLOTSAM ASSET CACHE]: Expiring Asset {0}" , id ) ;
2009-06-09 17:48:22 +00:00
try
{
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:51:47 +00:00
string filename = GetFileName ( id ) ;
if ( File . Exists ( filename ) )
{
File . Delete ( filename ) ;
}
2009-06-09 17:48:22 +00:00
}
2009-06-10 04:28:56 +00:00
if ( m_MemoryCacheEnabled )
2009-06-09 17:48:22 +00:00
m_MemoryCache . Remove ( id ) ;
}
catch ( Exception e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}" ,
id , e . Message , e . StackTrace ) ;
2009-06-09 17:48:22 +00:00
}
}
public void Clear ( )
{
if ( m_LogLevel > = 2 )
2011-07-04 21:51:47 +00:00
m_log . Debug ( "[FLOTSAM ASSET CACHE]: Clearing caches." ) ;
2009-06-09 17:48:22 +00:00
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
2009-06-09 17:48:22 +00:00
{
2011-07-04 21:51:47 +00:00
foreach ( string dir in Directory . GetDirectories ( m_CacheDirectory ) )
{
Directory . Delete ( dir ) ;
}
2009-06-09 17:48:22 +00:00
}
2009-06-10 04:28:56 +00:00
if ( m_MemoryCacheEnabled )
2009-06-09 17:48:22 +00:00
m_MemoryCache . Clear ( ) ;
}
private void CleanupExpiredFiles ( object source , ElapsedEventArgs e )
{
if ( m_LogLevel > = 2 )
2011-02-25 01:31:38 +00:00
m_log . DebugFormat ( "[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}." , m_FileExpiration ) ;
2009-06-09 17:48:22 +00:00
2009-10-12 22:21:32 +00:00
// Purge all files last accessed prior to this point
DateTime purgeLine = DateTime . Now - m_FileExpiration ;
2012-09-15 00:08:15 +00:00
// An asset cache may contain local non-temporary assets that are not in the asset service. Therefore,
// before cleaning up expired files we must scan the objects in the scene to make sure that we retain
// such local assets if they have not been recently accessed.
TouchAllSceneAssets ( false ) ;
2009-10-12 22:21:32 +00:00
2009-06-09 17:48:22 +00:00
foreach ( string dir in Directory . GetDirectories ( m_CacheDirectory ) )
{
2009-10-12 22:21:32 +00:00
CleanExpiredFiles ( dir , purgeLine ) ;
2009-09-08 20:31:09 +00:00
}
}
2009-07-28 17:20:51 +00:00
2009-09-08 20:31:09 +00:00
/// <summary>
2009-10-12 22:21:32 +00:00
/// Recurses through specified directory checking for asset files last
/// accessed prior to the specified purge line and deletes them. Also
/// removes empty tier directories.
2009-09-08 20:31:09 +00:00
/// </summary>
/// <param name="dir"></param>
2011-07-04 21:30:18 +00:00
/// <param name="purgeLine"></param>
2009-10-12 22:21:32 +00:00
private void CleanExpiredFiles ( string dir , DateTime purgeLine )
2009-09-08 20:31:09 +00:00
{
2013-02-08 21:21:20 +00:00
try
2009-09-08 20:31:09 +00:00
{
2013-02-08 21:21:20 +00:00
foreach ( string file in Directory . GetFiles ( dir ) )
2009-07-28 17:20:51 +00:00
{
2013-02-08 21:21:20 +00:00
if ( File . GetLastAccessTime ( file ) < purgeLine )
{
File . Delete ( file ) ;
}
2009-07-28 17:20:51 +00:00
}
2009-09-08 20:31:09 +00:00
2013-02-08 21:21:20 +00:00
// Recurse into lower tiers
foreach ( string subdir in Directory . GetDirectories ( dir ) )
{
CleanExpiredFiles ( subdir , purgeLine ) ;
}
2009-09-08 20:31:09 +00:00
2013-02-08 21:21:20 +00:00
// Check if a tier directory is empty, if so, delete it
int dirSize = Directory . GetFiles ( dir ) . Length + Directory . GetDirectories ( dir ) . Length ;
if ( dirSize = = 0 )
{
Directory . Delete ( dir ) ;
}
else if ( dirSize > = m_CacheWarnAt )
{
m_log . WarnFormat (
"[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration" ,
dir , dirSize ) ;
}
2009-09-08 20:31:09 +00:00
}
2013-02-08 21:21:20 +00:00
catch ( Exception e )
2009-09-08 20:31:09 +00:00
{
2013-02-08 21:21:20 +00:00
m_log . Warn (
string . Format ( "[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception " , dir ) , e ) ;
2009-09-30 16:00:09 +00:00
}
2009-06-09 17:48:22 +00:00
}
2009-10-12 22:21:32 +00:00
/// <summary>
/// Determines the filename for an AssetID stored in the file cache
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
2009-06-09 17:48:22 +00:00
private string GetFileName ( string id )
{
// Would it be faster to just hash the darn thing?
foreach ( char c in m_InvalidChars )
{
id = id . Replace ( c , '_' ) ;
}
2009-07-28 17:20:51 +00:00
string path = m_CacheDirectory ;
for ( int p = 1 ; p < = m_CacheDirectoryTiers ; p + + )
{
2009-07-29 17:21:36 +00:00
string pathPart = id . Substring ( ( p - 1 ) * m_CacheDirectoryTierLen , m_CacheDirectoryTierLen ) ;
2009-07-28 17:20:51 +00:00
path = Path . Combine ( path , pathPart ) ;
}
return Path . Combine ( path , id ) ;
2009-06-09 17:48:22 +00:00
}
2009-10-12 22:21:32 +00:00
/// <summary>
/// Writes a file to the file cache, creating any nessesary
/// tier directories along the way
/// </summary>
/// <param name="filename"></param>
/// <param name="asset"></param>
2009-06-09 17:48:22 +00:00
private void WriteFileCache ( string filename , AssetBase asset )
{
2009-10-04 16:44:04 +00:00
Stream stream = null ;
2009-10-12 22:21:32 +00:00
2009-10-04 16:56:54 +00:00
// Make sure the target cache directory exists
string directory = Path . GetDirectoryName ( filename ) ;
2009-10-12 22:21:32 +00:00
2009-10-04 16:56:54 +00:00
// Write file first to a temp name, so that it doesn't look
// like it's already cached while it's still writing.
string tempname = Path . Combine ( directory , Path . GetRandomFileName ( ) ) ;
2009-10-12 22:21:32 +00:00
2009-06-09 17:48:22 +00:00
try
{
2011-08-16 20:03:43 +00:00
try
2009-06-09 17:48:22 +00:00
{
2011-08-16 20:03:43 +00:00
if ( ! Directory . Exists ( directory ) )
{
Directory . CreateDirectory ( directory ) ;
}
stream = File . Open ( tempname , FileMode . Create ) ;
BinaryFormatter bformatter = new BinaryFormatter ( ) ;
bformatter . Serialize ( stream , asset ) ;
2009-06-09 17:48:22 +00:00
}
2011-08-16 20:03:43 +00:00
catch ( IOException e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-08-16 20:03:43 +00:00
"[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}." ,
asset . ID , tempname , filename , directory , e . Message , e . StackTrace ) ;
2009-06-09 17:48:22 +00:00
2011-08-16 20:03:43 +00:00
return ;
}
finally
{
if ( stream ! = null )
stream . Close ( ) ;
}
try
{
// Now that it's written, rename it so that it can be found.
/ /
// File.Copy(tempname, filename, true);
// File.Delete(tempname);
/ /
2011-08-16 20:31:08 +00:00
// For a brief period, this was done as a separate copy and then temporary file delete operation to
// avoid an IOException caused by move if some competing thread had already written the file.
2011-08-16 20:03:43 +00:00
// However, this causes exceptions on Windows when other threads attempt to read a file
2011-08-16 20:31:08 +00:00
// which is still being copied. So instead, go back to moving the file and swallow any IOException.
2011-08-16 20:03:43 +00:00
/ /
// This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the
// filesystem.
File . Move ( tempname , filename ) ;
if ( m_LogLevel > = 2 )
m_log . DebugFormat ( "[FLOTSAM ASSET CACHE]: Cache Stored :: {0}" , asset . ID ) ;
}
catch ( IOException )
{
// If we see an IOException here it's likely that some other competing thread has written the
// cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be
// signally by the earlier temporary file writing code.
}
2009-06-09 17:48:22 +00:00
}
finally
{
// Even if the write fails with an exception, we need to make sure
// that we release the lock on that file, otherwise it'll never get
// cached
lock ( m_CurrentlyWriting )
{
#if WAIT_ON_INPROGRESS_REQUESTS
ManualResetEvent waitEvent ;
if ( m_CurrentlyWriting . TryGetValue ( filename , out waitEvent ) )
{
m_CurrentlyWriting . Remove ( filename ) ;
waitEvent . Set ( ) ;
}
# else
2011-08-05 21:45:42 +00:00
m_CurrentlyWriting . Remove ( filename ) ;
2009-06-09 17:48:22 +00:00
# endif
}
}
}
2009-10-12 22:21:32 +00:00
/// <summary>
/// Scan through the file cache, and return number of assets currently cached.
/// </summary>
/// <param name="dir"></param>
/// <returns></returns>
2009-09-08 20:31:09 +00:00
private int GetFileCacheCount ( string dir )
{
int count = Directory . GetFiles ( dir ) . Length ;
foreach ( string subdir in Directory . GetDirectories ( dir ) )
{
count + = GetFileCacheCount ( subdir ) ;
2009-06-09 17:48:22 +00:00
}
2009-09-08 20:31:09 +00:00
return count ;
2009-06-09 17:48:22 +00:00
}
2009-09-08 20:31:09 +00:00
2009-10-12 22:21:32 +00:00
/// <summary>
/// This notes the last time the Region had a deep asset scan performed on it.
/// </summary>
2013-02-08 21:21:20 +00:00
/// <param name="regionID"></param>
private void StampRegionStatusFile ( UUID regionID )
2009-10-12 22:21:32 +00:00
{
2013-02-08 21:21:20 +00:00
string RegionCacheStatusFile = Path . Combine ( m_CacheDirectory , "RegionStatus_" + regionID . ToString ( ) + ".fac" ) ;
try
2009-10-12 22:21:32 +00:00
{
2013-02-08 21:21:20 +00:00
if ( File . Exists ( RegionCacheStatusFile ) )
{
File . SetLastWriteTime ( RegionCacheStatusFile , DateTime . Now ) ;
}
else
{
File . WriteAllText (
RegionCacheStatusFile ,
"Please do not delete this file unless you are manually clearing your Flotsam Asset Cache." ) ;
}
2009-10-12 22:21:32 +00:00
}
2013-02-08 21:21:20 +00:00
catch ( Exception e )
2009-10-12 22:21:32 +00:00
{
2013-02-08 21:21:20 +00:00
m_log . Warn (
string . Format (
"[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception " ,
regionID ) ,
e ) ;
2009-10-12 22:21:32 +00:00
}
}
/// <summary>
/// Iterates through all Scenes, doing a deep scan through assets
2012-09-15 00:08:15 +00:00
/// to update the access time of all assets present in the scene or referenced by assets
/// in the scene.
2009-10-12 22:21:32 +00:00
/// </summary>
2012-09-15 00:08:15 +00:00
/// <param name="storeUncached">
/// If true, then assets scanned which are not found in cache are added to the cache.
/// </param>
/// <returns>Number of distinct asset references found in the scene.</returns>
private int TouchAllSceneAssets ( bool storeUncached )
2009-10-12 22:21:32 +00:00
{
UuidGatherer gatherer = new UuidGatherer ( m_AssetService ) ;
2013-02-02 02:57:38 +00:00
HashSet < UUID > uniqueUuids = new HashSet < UUID > ( ) ;
2010-03-15 21:17:17 +00:00
Dictionary < UUID , AssetType > assets = new Dictionary < UUID , AssetType > ( ) ;
2013-02-02 02:57:38 +00:00
2009-10-12 22:21:32 +00:00
foreach ( Scene s in m_Scenes )
{
StampRegionStatusFile ( s . RegionInfo . RegionID ) ;
s . ForEachSOG ( delegate ( SceneObjectGroup e )
2013-02-02 02:57:38 +00:00
{
2009-10-12 22:21:32 +00:00
gatherer . GatherAssetUuids ( e , assets ) ;
2013-02-02 02:57:38 +00:00
foreach ( UUID assetID in assets . Keys )
{
uniqueUuids . Add ( assetID ) ;
2009-10-12 22:21:32 +00:00
2013-02-02 02:57:38 +00:00
string filename = GetFileName ( assetID . ToString ( ) ) ;
if ( File . Exists ( filename ) )
{
File . SetLastAccessTime ( filename , DateTime . Now ) ;
}
else if ( storeUncached )
{
AssetBase cachedAsset = m_AssetService . Get ( assetID . ToString ( ) ) ;
if ( cachedAsset = = null & & assets [ assetID ] ! = AssetType . Unknown )
m_log . DebugFormat (
"[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets" ,
assetID , assets [ assetID ] , e . Name , e . AbsolutePosition , s . Name ) ;
}
}
assets . Clear ( ) ;
} ) ;
2009-10-12 22:21:32 +00:00
}
2013-02-02 02:57:38 +00:00
return uniqueUuids . Count ;
2009-10-12 22:21:32 +00:00
}
/// <summary>
/// Deletes all cache contents
/// </summary>
private void ClearFileCache ( )
{
foreach ( string dir in Directory . GetDirectories ( m_CacheDirectory ) )
{
try
{
Directory . Delete ( dir , true ) ;
}
catch ( Exception e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}" ,
dir , m_CacheDirectory , e . Message , e . StackTrace ) ;
2009-10-12 22:21:32 +00:00
}
}
foreach ( string file in Directory . GetFiles ( m_CacheDirectory ) )
{
try
{
File . Delete ( file ) ;
}
catch ( Exception e )
{
2013-02-08 21:21:20 +00:00
m_log . WarnFormat (
2011-07-30 00:35:22 +00:00
"[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}" ,
file , m_CacheDirectory , e . Message , e . StackTrace ) ;
2009-10-12 22:21:32 +00:00
}
}
}
2013-10-30 18:57:51 +00:00
private List < string > GenerateCacheHitReport ( )
{
List < string > outputLines = new List < string > ( ) ;
double fileHitRate = ( double ) m_DiskHits / m_Requests * 100.0 ;
outputLines . Add (
string . Format ( "File Hit Rate: {0}% for {1} requests" , fileHitRate . ToString ( "0.00" ) , m_Requests ) ) ;
if ( m_MemoryCacheEnabled )
{
double memHitRate = ( double ) m_MemoryHits / m_Requests * 100.0 ;
outputLines . Add (
string . Format ( "Memory Hit Rate: {0}% for {1} requests" , memHitRate . ToString ( "0.00" ) , m_Requests ) ) ;
}
outputLines . Add (
string . Format (
"Unnecessary requests due to requests for assets that are currently downloading: {0}" ,
m_RequestsForInprogress ) ) ;
return outputLines ;
}
2009-09-08 20:31:09 +00:00
#region Console Commands
private void HandleConsoleCommand ( string module , string [ ] cmdparams )
{
2013-10-30 18:32:53 +00:00
ICommandConsole con = MainConsole . Instance ;
2009-10-12 22:21:32 +00:00
if ( cmdparams . Length > = 2 )
2009-09-08 20:31:09 +00:00
{
string cmd = cmdparams [ 1 ] ;
2013-10-30 18:32:53 +00:00
2009-09-08 20:31:09 +00:00
switch ( cmd )
{
2009-10-12 22:21:32 +00:00
case "status" :
2011-07-04 21:51:47 +00:00
if ( m_MemoryCacheEnabled )
2013-10-30 18:57:51 +00:00
con . OutputFormat ( "Memory Cache: {0} assets" , m_MemoryCache . Count ) ;
2011-07-04 21:51:47 +00:00
else
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "Memory cache disabled" ) ;
2009-09-08 20:31:09 +00:00
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
2009-10-12 22:21:32 +00:00
{
2011-07-04 21:51:47 +00:00
int fileCount = GetFileCacheCount ( m_CacheDirectory ) ;
2013-10-30 18:57:51 +00:00
con . OutputFormat ( "File Cache: {0} assets" , fileCount ) ;
}
else
{
con . Output ( "File cache disabled" ) ;
}
GenerateCacheHitReport ( ) . ForEach ( l = > con . Output ( l ) ) ;
if ( m_FileCacheEnabled )
{
2013-10-30 18:32:53 +00:00
con . Output ( "Deep scans have previously been performed on the following regions:" ) ;
2011-07-04 21:51:47 +00:00
foreach ( string s in Directory . GetFiles ( m_CacheDirectory , "*.fac" ) )
2013-10-30 18:32:53 +00:00
{
2011-07-04 21:51:47 +00:00
string RegionID = s . Remove ( 0 , s . IndexOf ( "_" ) ) . Replace ( ".fac" , "" ) ;
DateTime RegionDeepScanTMStamp = File . GetLastWriteTime ( s ) ;
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "Region: {0}, {1}" , RegionID , RegionDeepScanTMStamp . ToString ( "MM/dd/yyyy hh:mm:ss" ) ) ;
2011-07-04 21:51:47 +00:00
}
}
2009-09-08 20:31:09 +00:00
break ;
2009-10-12 22:21:32 +00:00
case "clear" :
2011-06-10 23:50:20 +00:00
if ( cmdparams . Length < 2 )
2009-09-08 20:31:09 +00:00
{
2013-10-30 18:32:53 +00:00
con . Output ( "Usage is fcache clear [file] [memory]" ) ;
2009-10-12 22:21:32 +00:00
break ;
2009-09-08 20:31:09 +00:00
}
2011-06-10 23:50:20 +00:00
bool clearMemory = false , clearFile = false ;
if ( cmdparams . Length = = 2 )
{
clearMemory = true ;
clearFile = true ;
}
2009-10-12 22:21:32 +00:00
foreach ( string s in cmdparams )
2009-09-08 20:31:09 +00:00
{
2009-10-12 22:21:32 +00:00
if ( s . ToLower ( ) = = "memory" )
2011-06-10 23:50:20 +00:00
clearMemory = true ;
2009-10-12 22:21:32 +00:00
else if ( s . ToLower ( ) = = "file" )
2011-06-10 23:50:20 +00:00
clearFile = true ;
}
if ( clearMemory )
{
2011-07-04 21:51:47 +00:00
if ( m_MemoryCacheEnabled )
{
m_MemoryCache . Clear ( ) ;
2013-10-30 18:32:53 +00:00
con . Output ( "Memory cache cleared." ) ;
2011-07-04 21:51:47 +00:00
}
else
{
2013-10-30 18:32:53 +00:00
con . Output ( "Memory cache not enabled." ) ;
2011-07-04 21:51:47 +00:00
}
2009-09-08 20:31:09 +00:00
}
2011-06-10 23:50:20 +00:00
if ( clearFile )
{
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
{
ClearFileCache ( ) ;
2013-10-30 18:32:53 +00:00
con . Output ( "File cache cleared." ) ;
2011-07-04 21:51:47 +00:00
}
else
{
2013-10-30 18:32:53 +00:00
con . Output ( "File cache not enabled." ) ;
2011-07-04 21:51:47 +00:00
}
2011-06-10 23:50:20 +00:00
}
2009-10-12 22:21:32 +00:00
break ;
case "assets" :
2013-10-30 18:32:53 +00:00
con . Output ( "Ensuring assets are cached for all scenes." ) ;
2009-10-12 22:21:32 +00:00
Util . FireAndForget ( delegate {
2012-09-15 00:08:15 +00:00
int assetReferenceTotal = TouchAllSceneAssets ( true ) ;
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "Completed check with {0} assets." , assetReferenceTotal ) ;
2009-10-18 23:58:03 +00:00
} ) ;
2009-09-08 20:31:09 +00:00
break ;
2009-10-12 22:21:32 +00:00
case "expire" :
2010-01-29 21:52:13 +00:00
if ( cmdparams . Length < 3 )
2009-10-12 22:21:32 +00:00
{
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "Invalid parameters for Expire, please specify a valid date & time" , cmd ) ;
2009-10-12 22:21:32 +00:00
break ;
}
string s_expirationDate = "" ;
DateTime expirationDate ;
if ( cmdparams . Length > 3 )
{
s_expirationDate = string . Join ( " " , cmdparams , 2 , cmdparams . Length - 2 ) ;
}
else
{
s_expirationDate = cmdparams [ 2 ] ;
}
if ( ! DateTime . TryParse ( s_expirationDate , out expirationDate ) )
{
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "{0} is not a valid date & time" , cmd ) ;
2009-10-12 22:21:32 +00:00
break ;
}
2011-07-04 21:51:47 +00:00
if ( m_FileCacheEnabled )
CleanExpiredFiles ( m_CacheDirectory , expirationDate ) ;
else
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "File cache not active, not clearing." ) ;
2009-10-12 22:21:32 +00:00
break ;
2009-09-08 20:31:09 +00:00
default :
2013-10-30 18:32:53 +00:00
con . OutputFormat ( "Unknown command {0}" , cmd ) ;
2009-09-08 20:31:09 +00:00
break ;
}
}
else if ( cmdparams . Length = = 1 )
{
2013-10-30 18:32:53 +00:00
con . Output ( "fcache assets - Attempt a deep cache of all assets in all scenes" ) ;
con . Output ( "fcache expire <datetime> - Purge assets older then the specified date & time" ) ;
con . Output ( "fcache clear [file] [memory] - Remove cached assets" ) ;
con . Output ( "fcache status - Display cache status" ) ;
2009-09-08 20:31:09 +00:00
}
}
# endregion
2009-10-12 22:21:32 +00:00
#region IAssetService Members
public AssetMetadata GetMetadata ( string id )
{
AssetBase asset = Get ( id ) ;
return asset . Metadata ;
}
public byte [ ] GetData ( string id )
{
AssetBase asset = Get ( id ) ;
return asset . Data ;
}
public bool Get ( string id , object sender , AssetRetrieved handler )
{
AssetBase asset = Get ( id ) ;
handler ( id , sender , asset ) ;
return true ;
}
public string Store ( AssetBase asset )
{
2009-11-09 16:03:15 +00:00
if ( asset . FullID = = UUID . Zero )
2009-10-12 22:21:32 +00:00
{
asset . FullID = UUID . Random ( ) ;
}
Cache ( asset ) ;
return asset . ID ;
}
public bool UpdateContent ( string id , byte [ ] data )
{
AssetBase asset = Get ( id ) ;
asset . Data = data ;
Cache ( asset ) ;
return true ;
}
public bool Delete ( string id )
{
Expire ( id ) ;
return true ;
}
# endregion
2009-06-09 17:48:22 +00:00
}
2011-08-02 06:14:20 +00:00
}