* When saving an oar, save assets when immediately received rather than storing them all up in memory
* Hopefully this will remove out of memory problems when saving large oars on machines without much memory * It may also speed up saving of large oars0.6.5-rc1
parent
02bac7fea4
commit
6277156044
|
@ -109,10 +109,14 @@ namespace OpenSim.Framework.Serialization
|
||||||
|
|
||||||
// Write two consecutive 0 blocks to end the archive
|
// Write two consecutive 0 blocks to end the archive
|
||||||
byte[] finalZeroPadding = new byte[1024];
|
byte[] finalZeroPadding = new byte[1024];
|
||||||
m_bw.Write(finalZeroPadding);
|
|
||||||
|
|
||||||
m_bw.Flush();
|
lock (m_bw)
|
||||||
m_bw.Close();
|
{
|
||||||
|
m_bw.Write(finalZeroPadding);
|
||||||
|
|
||||||
|
m_bw.Flush();
|
||||||
|
m_bw.Close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
|
public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
|
||||||
|
@ -197,20 +201,23 @@ namespace OpenSim.Framework.Serialization
|
||||||
|
|
||||||
header[154] = 0;
|
header[154] = 0;
|
||||||
|
|
||||||
// Write out header
|
lock (m_bw)
|
||||||
m_bw.Write(header);
|
|
||||||
|
|
||||||
// Write out data
|
|
||||||
m_bw.Write(data);
|
|
||||||
|
|
||||||
if (data.Length % 512 != 0)
|
|
||||||
{
|
{
|
||||||
int paddingRequired = 512 - (data.Length % 512);
|
// Write out header
|
||||||
|
m_bw.Write(header);
|
||||||
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
|
|
||||||
|
// Write out data
|
||||||
byte[] padding = new byte[paddingRequired];
|
m_bw.Write(data);
|
||||||
m_bw.Write(padding);
|
|
||||||
|
if (data.Length % 512 != 0)
|
||||||
|
{
|
||||||
|
int paddingRequired = 512 - (data.Length % 512);
|
||||||
|
|
||||||
|
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
|
||||||
|
|
||||||
|
byte[] padding = new byte[paddingRequired];
|
||||||
|
m_bw.Write(padding);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
private InventoryArchiverModule m_module;
|
private InventoryArchiverModule m_module;
|
||||||
private CachedUserInfo m_userInfo;
|
private CachedUserInfo m_userInfo;
|
||||||
private string m_invPath;
|
private string m_invPath;
|
||||||
protected TarArchiveWriter m_archive;
|
protected TarArchiveWriter m_archiveWriter;
|
||||||
protected UuidGatherer m_assetGatherer;
|
protected UuidGatherer m_assetGatherer;
|
||||||
|
|
||||||
/// <value>
|
/// <value>
|
||||||
|
@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache);
|
m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
|
protected void ReceivedAllAssets(ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
|
||||||
{
|
{
|
||||||
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
|
|
||||||
assetsArchiver.Archive(m_archive);
|
|
||||||
|
|
||||||
Exception reportedException = null;
|
Exception reportedException = null;
|
||||||
bool succeeded = true;
|
bool succeeded = true;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
m_archive.Close();
|
m_archiveWriter.Close();
|
||||||
}
|
}
|
||||||
catch (IOException e)
|
catch (IOException e)
|
||||||
{
|
{
|
||||||
|
@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager);
|
saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager);
|
||||||
|
|
||||||
string serialization = UserInventoryItemSerializer.Serialize(saveItem);
|
string serialization = UserInventoryItemSerializer.Serialize(saveItem);
|
||||||
m_archive.WriteFile(filename, serialization);
|
m_archiveWriter.WriteFile(filename, serialization);
|
||||||
|
|
||||||
m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids);
|
m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids);
|
||||||
}
|
}
|
||||||
|
@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
inventoryFolder.ID);
|
inventoryFolder.ID);
|
||||||
|
|
||||||
// We need to make sure that we record empty folders
|
// We need to make sure that we record empty folders
|
||||||
m_archive.WriteDir(path);
|
m_archiveWriter.WriteDir(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
|
List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
|
||||||
|
@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
|
inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
m_archive = new TarArchiveWriter(m_saveStream);
|
m_archiveWriter = new TarArchiveWriter(m_saveStream);
|
||||||
|
|
||||||
if (null == inventoryFolder)
|
if (null == inventoryFolder)
|
||||||
{
|
{
|
||||||
|
@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
}
|
}
|
||||||
|
|
||||||
SaveUsers();
|
SaveUsers();
|
||||||
new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
|
new AssetsRequest(
|
||||||
|
new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys,
|
||||||
|
m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
|
||||||
|
|
||||||
if (creator != null)
|
if (creator != null)
|
||||||
{
|
{
|
||||||
m_archive.WriteFile(
|
m_archiveWriter.WriteFile(
|
||||||
ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml",
|
ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml",
|
||||||
UserProfileSerializer.Serialize(creator.UserProfile));
|
UserProfileSerializer.Serialize(creator.UserProfile));
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Method called when all the necessary assets for an archive request have been received.
|
/// Method called when all the necessary assets for an archive request have been received.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public delegate void AssetsRequestCallback(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids);
|
public delegate void AssetsRequestCallback(
|
||||||
|
ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Execute the write of an archive once we have received all the necessary data
|
/// Execute the write of an archive once we have received all the necessary data
|
||||||
|
@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
protected IRegionSerialiserModule m_serialiser;
|
protected IRegionSerialiserModule m_serialiser;
|
||||||
protected List<SceneObjectGroup> m_sceneObjects;
|
protected List<SceneObjectGroup> m_sceneObjects;
|
||||||
protected Scene m_scene;
|
protected Scene m_scene;
|
||||||
protected Stream m_saveStream;
|
protected TarArchiveWriter m_archiveWriter;
|
||||||
protected Guid m_requestId;
|
protected Guid m_requestId;
|
||||||
|
|
||||||
public ArchiveWriteRequestExecution(
|
public ArchiveWriteRequestExecution(
|
||||||
|
@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
ITerrainModule terrainModule,
|
ITerrainModule terrainModule,
|
||||||
IRegionSerialiserModule serialiser,
|
IRegionSerialiserModule serialiser,
|
||||||
Scene scene,
|
Scene scene,
|
||||||
Stream saveStream,
|
TarArchiveWriter archiveWriter,
|
||||||
Guid requestId)
|
Guid requestId)
|
||||||
{
|
{
|
||||||
m_sceneObjects = sceneObjects;
|
m_sceneObjects = sceneObjects;
|
||||||
m_terrainModule = terrainModule;
|
m_terrainModule = terrainModule;
|
||||||
m_serialiser = serialiser;
|
m_serialiser = serialiser;
|
||||||
m_scene = scene;
|
m_scene = scene;
|
||||||
m_saveStream = saveStream;
|
m_archiveWriter = archiveWriter;
|
||||||
m_requestId = requestId;
|
m_requestId = requestId;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected internal void ReceivedAllAssets(
|
protected internal void ReceivedAllAssets(
|
||||||
IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
|
ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
|
||||||
{
|
{
|
||||||
foreach (UUID uuid in assetsNotFoundUuids)
|
foreach (UUID uuid in assetsNotFoundUuids)
|
||||||
{
|
{
|
||||||
|
@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
|
|
||||||
m_log.InfoFormat(
|
m_log.InfoFormat(
|
||||||
"[ARCHIVER]: Received {0} of {1} assets requested",
|
"[ARCHIVER]: Received {0} of {1} assets requested",
|
||||||
assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count);
|
assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count);
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
|
m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
|
||||||
|
|
||||||
TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
|
|
||||||
|
|
||||||
// Write out control file
|
// Write out control file
|
||||||
archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
|
m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
|
m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
|
||||||
|
|
||||||
// Write out region settings
|
// Write out region settings
|
||||||
string settingsPath
|
string settingsPath
|
||||||
= String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
|
= String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
|
||||||
archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
|
m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
|
m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
|
||||||
|
|
||||||
|
@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
|
|
||||||
MemoryStream ms = new MemoryStream();
|
MemoryStream ms = new MemoryStream();
|
||||||
m_terrainModule.SaveToStream(terrainPath, ms);
|
m_terrainModule.SaveToStream(terrainPath, ms);
|
||||||
archive.WriteFile(terrainPath, ms.ToArray());
|
m_archiveWriter.WriteFile(terrainPath, ms.ToArray());
|
||||||
ms.Close();
|
ms.Close();
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
|
m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
|
||||||
|
@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
|
Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
|
||||||
sceneObject.UUID);
|
sceneObject.UUID);
|
||||||
|
|
||||||
archive.WriteFile(filename, serializedObject);
|
m_archiveWriter.WriteFile(filename, serializedObject);
|
||||||
}
|
}
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
|
m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
|
||||||
|
|
||||||
// Write out assets
|
m_archiveWriter.Close();
|
||||||
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
|
|
||||||
assetsArchiver.Archive(archive);
|
|
||||||
|
|
||||||
archive.Close();
|
|
||||||
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);
|
m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,7 @@ using System.Threading;
|
||||||
using log4net;
|
using log4net;
|
||||||
using OpenMetaverse;
|
using OpenMetaverse;
|
||||||
using OpenSim.Framework;
|
using OpenSim.Framework;
|
||||||
|
using OpenSim.Framework.Serialization;
|
||||||
using OpenSim.Region.CoreModules.World.Terrain;
|
using OpenSim.Region.CoreModules.World.Terrain;
|
||||||
using OpenSim.Region.Framework.Interfaces;
|
using OpenSim.Region.Framework.Interfaces;
|
||||||
using OpenSim.Region.Framework.Scenes;
|
using OpenSim.Region.Framework.Scenes;
|
||||||
|
@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
|
if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
|
||||||
assetUuids[regionSettings.TerrainTexture4] = 1;
|
assetUuids[regionSettings.TerrainTexture4] = 1;
|
||||||
|
|
||||||
|
TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream);
|
||||||
|
|
||||||
// Asynchronously request all the assets required to perform this archive operation
|
// Asynchronously request all the assets required to perform this archive operation
|
||||||
ArchiveWriteRequestExecution awre
|
ArchiveWriteRequestExecution awre
|
||||||
= new ArchiveWriteRequestExecution(
|
= new ArchiveWriteRequestExecution(
|
||||||
|
@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
m_scene.RequestModuleInterface<ITerrainModule>(),
|
m_scene.RequestModuleInterface<ITerrainModule>(),
|
||||||
m_scene.RequestModuleInterface<IRegionSerialiserModule>(),
|
m_scene.RequestModuleInterface<IRegionSerialiserModule>(),
|
||||||
m_scene,
|
m_scene,
|
||||||
m_saveStream,
|
archiveWriter,
|
||||||
m_requestId);
|
m_requestId);
|
||||||
|
|
||||||
new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
|
new AssetsRequest(
|
||||||
|
new AssetsArchiver(archiveWriter), assetUuids.Keys,
|
||||||
|
m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
/// <value>
|
/// <value>
|
||||||
/// Post a message to the log every x assets as a progress bar
|
/// Post a message to the log every x assets as a progress bar
|
||||||
/// </value>
|
/// </value>
|
||||||
private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
|
protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Archive assets
|
/// Keep a count of the number of assets written so that we can provide status updates
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected IDictionary<UUID, AssetBase> m_assets;
|
protected int m_assetsWritten;
|
||||||
|
|
||||||
|
protected TarArchiveWriter m_archiveWriter;
|
||||||
|
|
||||||
public AssetsArchiver(IDictionary<UUID, AssetBase> assets)
|
public AssetsArchiver(TarArchiveWriter archiveWriter)
|
||||||
{
|
{
|
||||||
m_assets = assets;
|
m_archiveWriter = archiveWriter;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Archive the assets given to this archiver to the given archive.
|
/// Archive the assets given to this archiver to the given archive.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="archive"></param>
|
/// <param name="archive"></param>
|
||||||
public void Archive(TarArchiveWriter archive)
|
public void WriteAsset(AssetBase asset)
|
||||||
{
|
{
|
||||||
//WriteMetadata(archive);
|
//WriteMetadata(archive);
|
||||||
WriteData(archive);
|
WriteData(asset);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Write an assets metadata file to the given archive
|
/// Write an assets metadata file to the given archive
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="archive"></param>
|
/// <param name="archive"></param>
|
||||||
protected void WriteMetadata(TarArchiveWriter archive)
|
// protected void WriteMetadata(TarArchiveWriter archive)
|
||||||
{
|
// {
|
||||||
StringWriter sw = new StringWriter();
|
// StringWriter sw = new StringWriter();
|
||||||
XmlTextWriter xtw = new XmlTextWriter(sw);
|
// XmlTextWriter xtw = new XmlTextWriter(sw);
|
||||||
|
//
|
||||||
xtw.Formatting = Formatting.Indented;
|
// xtw.Formatting = Formatting.Indented;
|
||||||
xtw.WriteStartDocument();
|
// xtw.WriteStartDocument();
|
||||||
|
//
|
||||||
xtw.WriteStartElement("assets");
|
// xtw.WriteStartElement("assets");
|
||||||
|
//
|
||||||
foreach (UUID uuid in m_assets.Keys)
|
// foreach (UUID uuid in m_assets.Keys)
|
||||||
{
|
// {
|
||||||
AssetBase asset = m_assets[uuid];
|
// AssetBase asset = m_assets[uuid];
|
||||||
|
//
|
||||||
if (asset != null)
|
// if (asset != null)
|
||||||
{
|
// {
|
||||||
xtw.WriteStartElement("asset");
|
// xtw.WriteStartElement("asset");
|
||||||
|
//
|
||||||
string extension = string.Empty;
|
// string extension = string.Empty;
|
||||||
|
//
|
||||||
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
|
// if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
|
||||||
{
|
// {
|
||||||
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
|
// extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
xtw.WriteElementString("filename", uuid.ToString() + extension);
|
// xtw.WriteElementString("filename", uuid.ToString() + extension);
|
||||||
|
//
|
||||||
xtw.WriteElementString("name", asset.Name);
|
// xtw.WriteElementString("name", asset.Name);
|
||||||
xtw.WriteElementString("description", asset.Description);
|
// xtw.WriteElementString("description", asset.Description);
|
||||||
xtw.WriteElementString("asset-type", asset.Type.ToString());
|
// xtw.WriteElementString("asset-type", asset.Type.ToString());
|
||||||
|
//
|
||||||
xtw.WriteEndElement();
|
// xtw.WriteEndElement();
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
xtw.WriteEndElement();
|
// xtw.WriteEndElement();
|
||||||
|
//
|
||||||
xtw.WriteEndDocument();
|
// xtw.WriteEndDocument();
|
||||||
|
//
|
||||||
archive.WriteFile("assets.xml", sw.ToString());
|
// archive.WriteFile("assets.xml", sw.ToString());
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Write asset data files to the given archive
|
/// Write asset data files to the given archive
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="archive"></param>
|
/// <param name="asset"></param>
|
||||||
protected void WriteData(TarArchiveWriter archive)
|
protected void WriteData(AssetBase asset)
|
||||||
{
|
{
|
||||||
// It appears that gtar, at least, doesn't need the intermediate directory entries in the tar
|
// It appears that gtar, at least, doesn't need the intermediate directory entries in the tar
|
||||||
//archive.AddDir("assets");
|
//archive.AddDir("assets");
|
||||||
|
|
||||||
int assetsAdded = 0;
|
string extension = string.Empty;
|
||||||
|
|
||||||
foreach (UUID uuid in m_assets.Keys)
|
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
|
||||||
{
|
{
|
||||||
AssetBase asset = m_assets[uuid];
|
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
|
||||||
|
}
|
||||||
string extension = string.Empty;
|
else
|
||||||
|
{
|
||||||
if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
|
m_log.ErrorFormat(
|
||||||
{
|
"[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
|
||||||
extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
|
asset.Type, asset.ID);
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
m_log.ErrorFormat(
|
|
||||||
"[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
|
|
||||||
asset.Type, asset.ID);
|
|
||||||
}
|
|
||||||
|
|
||||||
archive.WriteFile(
|
|
||||||
ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
|
|
||||||
asset.Data);
|
|
||||||
|
|
||||||
assetsAdded++;
|
|
||||||
|
|
||||||
if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
|
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0)
|
m_archiveWriter.WriteFile(
|
||||||
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
|
ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension,
|
||||||
|
asset.Data);
|
||||||
|
|
||||||
|
m_assetsWritten++;
|
||||||
|
|
||||||
|
if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
|
||||||
|
m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,6 +32,7 @@ using System.Threading;
|
||||||
using log4net;
|
using log4net;
|
||||||
using OpenMetaverse;
|
using OpenMetaverse;
|
||||||
using OpenSim.Framework;
|
using OpenSim.Framework;
|
||||||
|
using OpenSim.Framework.Serialization;
|
||||||
|
|
||||||
namespace OpenSim.Region.CoreModules.World.Archiver
|
namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
{
|
{
|
||||||
|
@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
{
|
{
|
||||||
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// uuids to request
|
/// uuids to request
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected ICollection<UUID> m_uuids;
|
protected ICollection<UUID> m_uuids;
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Callback used when all the assets requested have been received.
|
/// Callback used when all the assets requested have been received.
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected AssetsRequestCallback m_assetsRequestCallback;
|
protected AssetsRequestCallback m_assetsRequestCallback;
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Assets retrieved in this request
|
/// List of assets that were found. This will be passed back to the requester.
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected Dictionary<UUID, AssetBase> m_assets = new Dictionary<UUID, AssetBase>();
|
protected List<UUID> m_foundAssetUuids = new List<UUID>();
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Maintain a list of assets that could not be found. This will be passed back to the requester.
|
/// Maintain a list of assets that could not be found. This will be passed back to the requester.
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected List<UUID> m_notFoundAssetUuids = new List<UUID>();
|
protected List<UUID> m_notFoundAssetUuids = new List<UUID>();
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Record the number of asset replies required so we know when we've finished
|
/// Record the number of asset replies required so we know when we've finished
|
||||||
/// </summary>
|
/// </value>
|
||||||
private int m_repliesRequired;
|
private int m_repliesRequired;
|
||||||
|
|
||||||
/// <summary>
|
/// <value>
|
||||||
/// Asset cache used to request the assets
|
/// Asset cache used to request the assets
|
||||||
/// </summary>
|
/// </value>
|
||||||
protected IAssetCache m_assetCache;
|
protected IAssetCache m_assetCache;
|
||||||
|
|
||||||
|
protected AssetsArchiver m_assetsArchiver;
|
||||||
|
|
||||||
protected internal AssetsRequest(
|
protected internal AssetsRequest(
|
||||||
ICollection<UUID> uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
|
AssetsArchiver assetsArchiver, ICollection<UUID> uuids,
|
||||||
|
IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
|
||||||
{
|
{
|
||||||
|
m_assetsArchiver = assetsArchiver;
|
||||||
m_uuids = uuids;
|
m_uuids = uuids;
|
||||||
m_assetsRequestCallback = assetsRequestCallback;
|
m_assetsRequestCallback = assetsRequestCallback;
|
||||||
m_assetCache = assetCache;
|
m_assetCache = assetCache;
|
||||||
|
@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
|
|
||||||
// We can stop here if there are no assets to fetch
|
// We can stop here if there are no assets to fetch
|
||||||
if (m_repliesRequired == 0)
|
if (m_repliesRequired == 0)
|
||||||
m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
|
m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
|
||||||
|
|
||||||
foreach (UUID uuid in m_uuids)
|
foreach (UUID uuid in m_uuids)
|
||||||
{
|
{
|
||||||
|
@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
|
|
||||||
if (asset != null)
|
if (asset != null)
|
||||||
{
|
{
|
||||||
|
// Make sure that we don't run out of memory by hogging assets in the cache
|
||||||
m_assetCache.ExpireAsset(assetID);
|
m_assetCache.ExpireAsset(assetID);
|
||||||
m_assets[assetID] = asset;
|
|
||||||
|
m_foundAssetUuids.Add(assetID);
|
||||||
|
m_assetsArchiver.WriteAsset(asset);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
m_notFoundAssetUuids.Add(assetID);
|
m_notFoundAssetUuids.Add(assetID);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
|
if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
|
||||||
{
|
{
|
||||||
m_log.DebugFormat(
|
m_log.DebugFormat(
|
||||||
"[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets",
|
"[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets",
|
||||||
m_assets.Count, m_notFoundAssetUuids.Count);
|
m_foundAssetUuids.Count, m_notFoundAssetUuids.Count);
|
||||||
|
|
||||||
// We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive
|
// We want to stop using the asset cache thread asap
|
||||||
|
// as we now need to do the work of producing the rest of the archive
|
||||||
Thread newThread = new Thread(PerformAssetsRequestCallback);
|
Thread newThread = new Thread(PerformAssetsRequestCallback);
|
||||||
newThread.Name = "OpenSimulator archiving thread post assets receipt";
|
newThread.Name = "OpenSimulator archiving thread post assets receipt";
|
||||||
newThread.Start();
|
newThread.Start();
|
||||||
|
@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
m_assetsRequestCallback(m_assets, m_notFoundAssetUuids);
|
m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue