OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* When saving an oar, save assets when immediately received rather than storing them all up in memory 

* Hopefully this will remove out of memory problems when saving large oars on machines without much memory
* It may also speed up saving of large oars
0.6.5-rc1
Justin Clarke Casey 2009-05-14 20:37:54 +00:00
parent 02bac7fea4
commit 6277156044
6 changed files with 156 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
OpenSim/Framework/Serialization/TarArchiveWriter.cs
View File

@ -109,10 +109,14 @@ namespace OpenSim.Framework.Serialization
// Write two consecutive 0 blocks to end the archive
byte[] finalZeroPadding = new byte[1024];
m_bw.Write(finalZeroPadding);
m_bw.Flush();
m_bw.Close();
lock (m_bw)
{
m_bw.Write(finalZeroPadding);
m_bw.Flush();
m_bw.Close();
}
}
public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
@ -197,20 +201,23 @@ namespace OpenSim.Framework.Serialization
header[154] = 0;
// Write out header
m_bw.Write(header);
// Write out data
m_bw.Write(data);
if (data.Length % 512 != 0)
lock (m_bw)
{
int paddingRequired = 512 - (data.Length % 512);
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
byte[] padding = new byte[paddingRequired];
m_bw.Write(padding);
// Write out header
m_bw.Write(header);
// Write out data
m_bw.Write(data);
if (data.Length % 512 != 0)
{
int paddingRequired = 512 - (data.Length % 512);
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
byte[] padding = new byte[paddingRequired];
m_bw.Write(padding);
}
}
}
}

21
OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
View File

@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
private InventoryArchiverModule m_module;
private CachedUserInfo m_userInfo;
private string m_invPath;
protected TarArchiveWriter m_archive;
protected TarArchiveWriter m_archiveWriter;
protected UuidGatherer m_assetGatherer;
/// <value>
@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache);
}
protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
protected void ReceivedAllAssets(ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
{
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(m_archive);
Exception reportedException = null;
bool succeeded = true;
try
{
m_archive.Close();
m_archiveWriter.Close();
}
catch (IOException e)
{
@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager);
string serialization = UserInventoryItemSerializer.Serialize(saveItem);
m_archive.WriteFile(filename, serialization);
m_archiveWriter.WriteFile(filename, serialization);
m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids);
}
@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryFolder.ID);
// We need to make sure that we record empty folders
m_archive.WriteDir(path);
m_archiveWriter.WriteDir(path);
}
List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
}
m_archive = new TarArchiveWriter(m_saveStream);
m_archiveWriter = new TarArchiveWriter(m_saveStream);
if (null == inventoryFolder)
{
@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
}
SaveUsers();
new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
new AssetsRequest(
new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys,
m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
}
/// <summary>
@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
if (creator != null)
{
m_archive.WriteFile(
m_archiveWriter.WriteFile(
ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml",
UserProfileSerializer.Serialize(creator.UserProfile));
}

29
OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
View File

@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
/// <summary>
/// Method called when all the necessary assets for an archive request have been received.
/// </summary>
public delegate void AssetsRequestCallback(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids);
public delegate void AssetsRequestCallback(
ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids);
/// <summary>
/// Execute the write of an archive once we have received all the necessary data
@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
protected IRegionSerialiserModule m_serialiser;
protected List<SceneObjectGroup> m_sceneObjects;
protected Scene m_scene;
protected Stream m_saveStream;
protected TarArchiveWriter m_archiveWriter;
protected Guid m_requestId;
public ArchiveWriteRequestExecution(
@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
ITerrainModule terrainModule,
IRegionSerialiserModule serialiser,
Scene scene,
Stream saveStream,
TarArchiveWriter archiveWriter,
Guid requestId)
{
m_sceneObjects = sceneObjects;
m_terrainModule = terrainModule;
m_serialiser = serialiser;
m_scene = scene;
m_saveStream = saveStream;
m_archiveWriter = archiveWriter;
m_requestId = requestId;
}
protected internal void ReceivedAllAssets(
IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
{
foreach (UUID uuid in assetsNotFoundUuids)
{
@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_log.InfoFormat(
"[ARCHIVER]: Received {0} of {1} assets requested",
assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count);
assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count);
m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
// Write out control file
archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
// Write out region settings
string settingsPath
= String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
MemoryStream ms = new MemoryStream();
m_terrainModule.SaveToStream(terrainPath, ms);
archive.WriteFile(terrainPath, ms.ToArray());
m_archiveWriter.WriteFile(terrainPath, ms.ToArray());
ms.Close();
m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
sceneObject.UUID);
archive.WriteFile(filename, serializedObject);
m_archiveWriter.WriteFile(filename, serializedObject);
}
m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
// Write out assets
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(archive);
archive.Close();
m_archiveWriter.Close();
m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);

11
OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
View File

@ -35,6 +35,7 @@ using System.Threading;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Serialization;
using OpenSim.Region.CoreModules.World.Terrain;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
assetUuids[regionSettings.TerrainTexture4] = 1;
TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream);
// Asynchronously request all the assets required to perform this archive operation
ArchiveWriteRequestExecution awre
= new ArchiveWriteRequestExecution(
@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_scene.RequestModuleInterface<ITerrainModule>(),
m_scene.RequestModuleInterface<IRegionSerialiserModule>(),
m_scene,
m_saveStream,
m_requestId);
archiveWriter,
m_requestId);
new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
new AssetsRequest(
new AssetsArchiver(archiveWriter), assetUuids.Keys,
m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
}
}
}

152
OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
View File

@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver
/// <value>
/// Post a message to the log every x assets as a progress bar
/// </value>
private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
/// <summary>
/// Archive assets
/// </summary>
protected IDictionary<UUID, AssetBase> m_assets;
/// <value>
/// Keep a count of the number of assets written so that we can provide status updates
/// </value>
protected int m_assetsWritten;
protected TarArchiveWriter m_archiveWriter;
public AssetsArchiver(IDictionary<UUID, AssetBase> assets)
public AssetsArchiver(TarArchiveWriter archiveWriter)
{
m_assets = assets;
m_archiveWriter = archiveWriter;
}
/// <summary>
/// Archive the assets given to this archiver to the given archive.
/// </summary>
/// <param name="archive"></param>
public void Archive(TarArchiveWriter archive)
{
public void WriteAsset(AssetBase asset)
{
//WriteMetadata(archive);
WriteData(archive);
WriteData(asset);
}
/// <summary>
/// Write an assets metadata file to the given archive
/// </summary>
/// <param name="archive"></param>
protected void WriteMetadata(TarArchiveWriter archive)
{
StringWriter sw = new StringWriter();
XmlTextWriter xtw = new XmlTextWriter(sw);
xtw.Formatting = Formatting.Indented;
xtw.WriteStartDocument();
xtw.WriteStartElement("assets");
foreach (UUID uuid in m_assets.Keys)
{
AssetBase asset = m_assets[uuid];
if (asset != null)
{
xtw.WriteStartElement("asset");
string extension = string.Empty;
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
{
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
}
xtw.WriteElementString("filename", uuid.ToString() + extension);
xtw.WriteElementString("name", asset.Name);
xtw.WriteElementString("description", asset.Description);
xtw.WriteElementString("asset-type", asset.Type.ToString());
xtw.WriteEndElement();
}
}
xtw.WriteEndElement();
xtw.WriteEndDocument();
archive.WriteFile("assets.xml", sw.ToString());
}
// protected void WriteMetadata(TarArchiveWriter archive)
// {
// StringWriter sw = new StringWriter();
// XmlTextWriter xtw = new XmlTextWriter(sw);
//
// xtw.Formatting = Formatting.Indented;
// xtw.WriteStartDocument();
//
// xtw.WriteStartElement("assets");
//
// foreach (UUID uuid in m_assets.Keys)
// {
// AssetBase asset = m_assets[uuid];
//
// if (asset != null)
// {
// xtw.WriteStartElement("asset");
//
// string extension = string.Empty;
//
// if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
// {
// extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
// }
//
// xtw.WriteElementString("filename", uuid.ToString() + extension);
//
// xtw.WriteElementString("name", asset.Name);
// xtw.WriteElementString("description", asset.Description);
// xtw.WriteElementString("asset-type", asset.Type.ToString());
//
// xtw.WriteEndElement();
// }
// }
//
// xtw.WriteEndElement();
//
// xtw.WriteEndDocument();
//
// archive.WriteFile("assets.xml", sw.ToString());
// }
/// <summary>
/// Write asset data files to the given archive
/// </summary>
/// <param name="archive"></param>
protected void WriteData(TarArchiveWriter archive)
/// <param name="asset"></param>
protected void WriteData(AssetBase asset)
{
// It appears that gtar, at least, doesn't need the intermediate directory entries in the tar
//archive.AddDir("assets");
int assetsAdded = 0;
string extension = string.Empty;
foreach (UUID uuid in m_assets.Keys)
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
{
AssetBase asset = m_assets[uuid];
string extension = string.Empty;
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
{
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
}
else
{
m_log.ErrorFormat(
"[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
asset.Type, asset.ID);
}
archive.WriteFile(
ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
asset.Data);
assetsAdded++;
if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
}
else
{
m_log.ErrorFormat(
"[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
asset.Type, asset.ID);
}
if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0)
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
m_archiveWriter.WriteFile(
ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension,
asset.Data);
m_assetsWritten++;
if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten);
}
}
}

53
OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
View File

@ -32,6 +32,7 @@ using System.Threading;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Serialization;
namespace OpenSim.Region.CoreModules.World.Archiver
{
@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
/// <summary>
/// <value>
/// uuids to request
/// </summary>
/// </value>
protected ICollection<UUID> m_uuids;
/// <summary>
/// <value>
/// Callback used when all the assets requested have been received.
/// </summary>
/// </value>
protected AssetsRequestCallback m_assetsRequestCallback;
/// <summary>
/// Assets retrieved in this request
/// </summary>
protected Dictionary<UUID, AssetBase> m_assets = new Dictionary<UUID, AssetBase>();
/// <summary>
/// <value>
/// List of assets that were found. This will be passed back to the requester.
/// </value>
protected List<UUID> m_foundAssetUuids = new List<UUID>();
/// <value>
/// Maintain a list of assets that could not be found. This will be passed back to the requester.
/// </summary>
/// </value>
protected List<UUID> m_notFoundAssetUuids = new List<UUID>();
/// <summary>
/// <value>
/// Record the number of asset replies required so we know when we've finished
/// </summary>
/// </value>
private int m_repliesRequired;
/// <summary>
/// <value>
/// Asset cache used to request the assets
/// </summary>
/// </value>
protected IAssetCache m_assetCache;
protected AssetsArchiver m_assetsArchiver;
protected internal AssetsRequest(
ICollection<UUID> uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
AssetsArchiver assetsArchiver, ICollection<UUID> uuids,
IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
{
m_assetsArchiver = assetsArchiver;
m_uuids = uuids;
m_assetsRequestCallback = assetsRequestCallback;
m_assetCache = assetCache;
@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// We can stop here if there are no assets to fetch
if (m_repliesRequired == 0)
m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
foreach (UUID uuid in m_uuids)
{
@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver
if (asset != null)
{
// Make sure that we don't run out of memory by hogging assets in the cache
m_assetCache.ExpireAsset(assetID);
m_assets[assetID] = asset;
m_foundAssetUuids.Add(assetID);
m_assetsArchiver.WriteAsset(asset);
}
else
{
m_notFoundAssetUuids.Add(assetID);
}
if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
{
m_log.DebugFormat(
"[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets",
m_assets.Count, m_notFoundAssetUuids.Count);
m_foundAssetUuids.Count, m_notFoundAssetUuids.Count);
// We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive
// We want to stop using the asset cache thread asap
// as we now need to do the work of producing the rest of the archive
Thread newThread = new Thread(PerformAssetsRequestCallback);
newThread.Name = "OpenSimulator archiving thread post assets receipt";
newThread.Start();
@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{
try
{
m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
}
catch (Exception e)
{