OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bringing FSAssets more inline with current OpenSim standards. * If no connection string found in assets config, fallback to using default database config * Create database storage interface to allow other db connectors to be written at some point * Add MySQL migrations file to create the initial db table * Added new config option named DaysBetweenAccessTimeUpdates to reduce db load by only updating access times when fetching assets if the last access time was over the threshold. This idea was taken from XAssets service. * Change log message headers to indicate FS assets is the source not just assets 

Signed-off-by: Melanie Thielker <melanie@t-data.com>
fsassets
AliciaRaven 2015-07-12 01:35:57 +01:00 committed by Melanie Thielker
parent c59144ff13
commit f3f748ed11
4 changed files with 199 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
OpenSim/Data/IFSAssetData.cs Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) Contributors, http://opensimulator.org/
* See CONTRIBUTORS.TXT for a full list of copyright holders.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the OpenSimulator Project nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
using System.Collections.Generic;
using OpenMetaverse;
using OpenSim.Framework;
namespace OpenSim.Data
{
public delegate string FSStoreDelegate(AssetBase asset, bool force);
public interface IFSAssetDataPlugin : IPlugin
{
bool[] AssetsExist(UUID[] uuids);
void Initialise(string connect, string realm, int SkipAccessTimeDays);
bool Delete(string id);
AssetMetadata Get(string id, out string hash);
bool Store(AssetMetadata metadata, string hash);
void Import(string conn, string table, int start, int count, bool force, FSStoreDelegate store);
int Count();
}
}

109
OpenSim/Data/MySQL/MySQLFSAssetData.cs
View File

@ -29,36 +29,76 @@ using System;
using System.Reflection;
using System.Collections.Generic;
using System.Data;
using OpenSim.Data;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using log4net;
using MySql.Data.MySqlClient;
using System.Data;
using OpenMetaverse;
namespace OpenSim.Data.MySQL
{
public delegate string StoreDelegate(AssetBase asset, bool force);
public class FSAssetConnectorData
public class MySQLFSAssetData : IFSAssetDataPlugin
{
private static readonly ILog m_log =
LogManager.GetLogger(
MethodBase.GetCurrentMethod().DeclaringType);
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected MySqlConnection m_Connection = null;
protected string m_ConnectionString;
protected string m_Table;
protected Object m_connLock = new Object();
public FSAssetConnectorData(string connectionString, string table)
{
m_ConnectionString = connectionString;
m_Table = table;
/// <summary>
/// Number of days that must pass before we update the access time on an asset when it has been fetched
/// Config option to change this is "DaysBetweenAccessTimeUpdates"
/// </summary>
private int DaysBetweenAccessTimeUpdates = 0;
OpenDatabase();
protected virtual Assembly Assembly
{
get { return GetType().Assembly; }
}
public MySQLFSAssetData()
{
}
#region IPlugin Members
public string Version { get { return "1.0.0.0"; } }
// Loads and initialises the MySQL storage plugin and checks for migrations
public void Initialise(string connect, string realm, int UpdateAccessTime)
{
m_ConnectionString = connect;
m_Table = realm;
DaysBetweenAccessTimeUpdates = UpdateAccessTime;
try
{
OpenDatabase();
Migration m = new Migration(m_Connection, Assembly, "FSAssetStore");
m.Update();
}
catch (MySqlException e)
{
m_log.ErrorFormat("[FSASSETS]: Can't connect to database: {0}", e.Message.ToString());
}
}
public void Initialise()
{
throw new NotImplementedException();
}
public void Dispose() { }
public string Name
{
get { return "MySQL FSAsset storage engine"; }
}
#endregion
private bool OpenDatabase()
{
@ -126,13 +166,15 @@ namespace OpenSim.Data.MySQL
}
}
#region IFSAssetDataPlugin Members
public AssetMetadata Get(string id, out string hash)
{
hash = String.Empty;
MySqlCommand cmd = new MySqlCommand();
cmd.CommandText = String.Format("select id, name, description, type, hash, create_time, asset_flags from {0} where id = ?id", m_Table);
cmd.CommandText = String.Format("select id, name, description, type, hash, create_time, access_time, asset_flags from {0} where id = ?id", m_Table);
cmd.Parameters.AddWithValue("?id", id);
IDataReader reader = ExecuteReader(cmd);
@ -158,17 +200,29 @@ namespace OpenSim.Data.MySQL
meta.CreationDate = Util.ToDateTime(Convert.ToInt32(reader["create_time"]));
meta.Flags = (AssetFlags)Convert.ToInt32(reader["asset_flags"]);
int AccessTime = Convert.ToInt32(reader["access_time"]);
reader.Close();
cmd.CommandText = String.Format("update {0} set access_time = UNIX_TIMESTAMP() where id = ?id", m_Table);
cmd.ExecuteNonQuery();
UpdateAccessTime(AccessTime, cmd);
FreeCommand(cmd);
return meta;
}
private void UpdateAccessTime(int AccessTime, MySqlCommand cmd)
{
// Reduce DB work by only updating access time if asset hasn't recently been accessed
// 0 By Default, Config option is "DaysBetweenAccessTimeUpdates"
if (DaysBetweenAccessTimeUpdates > 0 && (DateTime.UtcNow - Utils.UnixTimeToDateTime(AccessTime)).TotalDays < DaysBetweenAccessTimeUpdates)
return;
cmd.CommandText = String.Format("UPDATE {0} SET `access_time` = UNIX_TIMESTAMP() WHERE `id` = ?id", m_Table);
cmd.ExecuteNonQuery();
}
protected void FreeCommand(MySqlCommand cmd)
{
MySqlConnection c = cmd.Connection;
@ -214,7 +268,7 @@ namespace OpenSim.Data.MySQL
catch(Exception e)
{
m_log.Error("[FSAssets] Failed to store asset with ID " + meta.ID);
m_log.Error(e.ToString());
m_log.Error(e.ToString());
return false;
}
}
@ -272,20 +326,21 @@ namespace OpenSim.Data.MySQL
return count;
}
public void Delete(string id)
public bool Delete(string id)
{
MySqlCommand cmd = m_Connection.CreateCommand();
using (MySqlCommand cmd = m_Connection.CreateCommand())
{
cmd.CommandText = String.Format("delete from {0} where id = ?id", m_Table);
cmd.CommandText = String.Format("delete from {0} where id = ?id", m_Table);
cmd.Parameters.AddWithValue("?id", id);
cmd.Parameters.AddWithValue("?id", id);
ExecuteNonQuery(cmd);
}
ExecuteNonQuery(cmd);
cmd.Dispose();
return true;
}
public void Import(string conn, string table, int start, int count, bool force, StoreDelegate store)
public void Import(string conn, string table, int start, int count, bool force, FSStoreDelegate store)
{
MySqlConnection importConn;
@ -353,5 +408,7 @@ namespace OpenSim.Data.MySQL
MainConsole.Instance.Output(String.Format("Import done, {0} assets imported", imported));
}
#endregion
}
}

18
OpenSim/Data/MySQL/Resources/FSAssetStore.migrations Normal file
View File

@ -0,0 +1,18 @@
# -----------------
:VERSION 1
BEGIN;
CREATE TABLE `fsassets` (
`id` char(36) NOT NULL,
`name` varchar(64) NOT NULL DEFAULT '',
`description` varchar(64) NOT NULL DEFAULT '',
`type` int(11) NOT NULL,
`hash` char(80) NOT NULL,
`create_time` int(11) NOT NULL DEFAULT '0',
`access_time` int(11) NOT NULL DEFAULT '0',
`asset_flags` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
COMMIT;

80
OpenSim/Services/FSAssetService/FSAssetService.cs
View File

@ -33,6 +33,7 @@ using System.IO.Compression;
using System.Text;
using System.Threading;
using System.Reflection;
using OpenSim.Data;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Server.Base;
@ -47,9 +48,7 @@ namespace OpenSim.Services.FSAssetService
{
public class FSAssetConnector : ServiceBase, IAssetService
{
private static readonly ILog m_log =
LogManager.GetLogger(
MethodBase.GetCurrentMethod().DeclaringType);
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
static System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
static SHA256CryptoServiceProvider SHA256 = new SHA256CryptoServiceProvider();
@ -64,9 +63,7 @@ namespace OpenSim.Services.FSAssetService
}
protected IAssetLoader m_AssetLoader = null;
protected string m_ConnectionString;
protected FSAssetConnectorData m_DataConnector = null;
protected string m_FsckProgram;
protected IFSAssetDataPlugin m_DataConnector = null;
protected IAssetService m_FallbackService;
protected Thread m_WriterThread;
protected Thread m_StatsThread;
@ -78,7 +75,6 @@ namespace OpenSim.Services.FSAssetService
protected int m_missingAssets = 0;
protected int m_missingAssetsFS = 0;
protected string m_FSBase;
protected string m_Realm;
public FSAssetConnector(IConfigSource config)
: this(config, "AssetService")
@ -87,8 +83,6 @@ namespace OpenSim.Services.FSAssetService
public FSAssetConnector(IConfigSource config, string configName) : base(config)
{
m_FsckProgram = string.Empty;
MainConsole.Instance.Commands.AddCommand("fs", false,
"show assets", "show assets", "Show asset stats",
HandleShowAssets);
@ -109,35 +103,63 @@ namespace OpenSim.Services.FSAssetService
HandleImportAssets);
IConfig assetConfig = config.Configs[configName];
if (assetConfig == null)
{
throw new Exception("No AssetService configuration");
}
m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty);
if (m_ConnectionString == string.Empty)
// Get Database Connector from Asset Config (If present)
string dllName = assetConfig.GetString("StorageProvider", string.Empty);
string m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty);
string m_Realm = assetConfig.GetString("Realm", "fsassets");
int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0);
// If not found above, fallback to Database defaults
IConfig dbConfig = config.Configs["DatabaseService"];
if (dbConfig != null)
{
throw new Exception("Missing database connection string");
if (dllName == String.Empty)
dllName = dbConfig.GetString("StorageProvider", String.Empty);
if (m_ConnectionString == String.Empty)
m_ConnectionString = dbConfig.GetString("ConnectionString", String.Empty);
}
m_Realm = assetConfig.GetString("Realm", "fsassets");
// No databse connection found in either config
if (dllName.Equals(String.Empty))
throw new Exception("No StorageProvider configured");
m_DataConnector = new FSAssetConnectorData(m_ConnectionString, m_Realm);
if (m_ConnectionString.Equals(String.Empty))
throw new Exception("Missing database connection string");
// Create Storage Provider
m_DataConnector = LoadPlugin<IFSAssetDataPlugin>(dllName);
if (m_DataConnector == null)
throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName));
// Initialize DB And perform any migrations required
m_DataConnector.Initialise(m_ConnectionString, m_Realm, SkipAccessTimeDays);
// Setup Fallback Service
string str = assetConfig.GetString("FallbackService", string.Empty);
if (str != string.Empty)
{
object[] args = new object[] { config };
m_FallbackService = LoadPlugin<IAssetService>(str, args);
if (m_FallbackService != null)
{
m_log.Info("[FALLBACK]: Fallback service loaded");
m_log.Info("[FSASSETS]: Fallback service loaded");
}
else
{
m_log.Error("[FALLBACK]: Failed to load fallback service");
m_log.Error("[FSASSETS]: Failed to load fallback service");
}
}
// Setup directory structure including temp directory
m_SpoolDirectory = assetConfig.GetString("SpoolDirectory", "/tmp");
string spoolTmp = Path.Combine(m_SpoolDirectory, "spool");
@ -147,7 +169,7 @@ namespace OpenSim.Services.FSAssetService
m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty);
if (m_FSBase == String.Empty)
{
m_log.ErrorFormat("[ASSET]: BaseDirectory not specified");
m_log.ErrorFormat("[FSASSETS]: BaseDirectory not specified");
throw new Exception("Configuration error");
}
@ -156,14 +178,14 @@ namespace OpenSim.Services.FSAssetService
{
m_AssetLoader = LoadPlugin<IAssetLoader>(loader);
string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty);
m_log.InfoFormat("[ASSET]: Loading default asset set from {0}", loaderArgs);
m_log.InfoFormat("[FSASSETS]: Loading default asset set from {0}", loaderArgs);
m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs,
delegate(AssetBase a)
{
Store(a, false);
});
}
m_log.Info("[ASSET]: FS asset service enabled");
m_log.Info("[FSASSETS]: FS asset service enabled");
m_WriterThread = new Thread(Writer);
m_WriterThread.Start();
@ -184,7 +206,7 @@ namespace OpenSim.Services.FSAssetService
double avg = (double)m_readTicks / (double)m_readCount;
// if (avg > 10000)
// Environment.Exit(0);
m_log.InfoFormat("[ASSET]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS);
m_log.InfoFormat("[FSASSETS]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS);
}
m_readCount = 0;
m_readTicks = 0;
@ -196,7 +218,7 @@ namespace OpenSim.Services.FSAssetService
private void Writer()
{
m_log.Info("[ASSET]: Writer started");
m_log.Info("[FSASSETS]: Writer started");
while (true)
{
@ -236,7 +258,7 @@ namespace OpenSim.Services.FSAssetService
int totalTicks = System.Environment.TickCount - tickCount;
if (totalTicks > 0) // Wrap?
{
m_log.InfoFormat("[ASSET]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
m_log.InfoFormat("[FSASSETS]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
}
}
@ -326,13 +348,13 @@ namespace OpenSim.Services.FSAssetService
asset.Metadata.ContentType =
SLUtil.SLAssetTypeToContentType((int)asset.Type);
sha = GetSHA256Hash(asset.Data);
m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id);
m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id);
Store(asset);
}
}
if (asset == null)
{
// m_log.InfoFormat("[ASSET]: Asset {0} not found", id);
// m_log.InfoFormat("[FSASSETS]: Asset {0} not found", id);
m_missingAssets++;
}
return asset;
@ -353,13 +375,13 @@ namespace OpenSim.Services.FSAssetService
asset.Metadata.ContentType =
SLUtil.SLAssetTypeToContentType((int)asset.Type);
sha = GetSHA256Hash(asset.Data);
m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id);
m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id);
Store(asset);
}
}
if (asset == null)
m_missingAssetsFS++;
// m_log.InfoFormat("[ASSET]: Asset {0}, hash {1} not found in FS", id, hash);
// m_log.InfoFormat("[FSASSETS]: Asset {0}, hash {1} not found in FS", id, hash);
else
return asset;
}
@ -649,7 +671,7 @@ namespace OpenSim.Services.FSAssetService
{
count = Convert.ToInt32(args[4]);
}
m_DataConnector.Import(conn, table, start, count, force, new StoreDelegate(Store));
m_DataConnector.Import(conn, table, start, count, force, new FSStoreDelegate(Store));
}
}