diff --git a/OpenSim/Framework/Serialization/TarArchiveWriter.cs b/OpenSim/Framework/Serialization/TarArchiveWriter.cs index d319b0b725..316a8dec61 100644 --- a/OpenSim/Framework/Serialization/TarArchiveWriter.cs +++ b/OpenSim/Framework/Serialization/TarArchiveWriter.cs @@ -109,10 +109,14 @@ namespace OpenSim.Framework.Serialization // Write two consecutive 0 blocks to end the archive byte[] finalZeroPadding = new byte[1024]; - m_bw.Write(finalZeroPadding); - m_bw.Flush(); - m_bw.Close(); + lock (m_bw) + { + m_bw.Write(finalZeroPadding); + + m_bw.Flush(); + m_bw.Close(); + } } public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) @@ -197,20 +201,23 @@ namespace OpenSim.Framework.Serialization header[154] = 0; - // Write out header - m_bw.Write(header); - - // Write out data - m_bw.Write(data); - - if (data.Length % 512 != 0) + lock (m_bw) { - int paddingRequired = 512 - (data.Length % 512); - - //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); - - byte[] padding = new byte[paddingRequired]; - m_bw.Write(padding); + // Write out header + m_bw.Write(header); + + // Write out data + m_bw.Write(data); + + if (data.Length % 512 != 0) + { + int paddingRequired = 512 - (data.Length % 512); + + //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); + + byte[] padding = new byte[paddingRequired]; + m_bw.Write(padding); + } } } } diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs index b5c0d86ae2..fec24258fb 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs @@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver private InventoryArchiverModule m_module; private CachedUserInfo m_userInfo; private string m_invPath; - protected TarArchiveWriter m_archive; + protected TarArchiveWriter m_archiveWriter; protected UuidGatherer m_assetGatherer; /// @@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache); } - protected void ReceivedAllAssets(IDictionary assetsFound, ICollection assetsNotFoundUuids) + protected void ReceivedAllAssets(ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) { - AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); - assetsArchiver.Archive(m_archive); - Exception reportedException = null; bool succeeded = true; try { - m_archive.Close(); + m_archiveWriter.Close(); } catch (IOException e) { @@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager); string serialization = UserInventoryItemSerializer.Serialize(saveItem); - m_archive.WriteFile(filename, serialization); + m_archiveWriter.WriteFile(filename, serialization); m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids); } @@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver inventoryFolder.ID); // We need to make sure that we record empty folders - m_archive.WriteDir(path); + m_archiveWriter.WriteDir(path); } List childFolders = inventoryFolder.RequestListOfFolderImpls(); @@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); } - m_archive = new TarArchiveWriter(m_saveStream); + m_archiveWriter = new TarArchiveWriter(m_saveStream); if (null == inventoryFolder) { @@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver } SaveUsers(); - new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute(); + new AssetsRequest( + new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys, + m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute(); } /// @@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver if (creator != null) { - m_archive.WriteFile( + m_archiveWriter.WriteFile( ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml", UserProfileSerializer.Serialize(creator.UserProfile)); } diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs index 943d9d1488..b167ce2d64 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs @@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Method called when all the necessary assets for an archive request have been received. /// - public delegate void AssetsRequestCallback(IDictionary assetsFound, ICollection assetsNotFoundUuids); + public delegate void AssetsRequestCallback( + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids); /// /// Execute the write of an archive once we have received all the necessary data @@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver protected IRegionSerialiserModule m_serialiser; protected List m_sceneObjects; protected Scene m_scene; - protected Stream m_saveStream; + protected TarArchiveWriter m_archiveWriter; protected Guid m_requestId; public ArchiveWriteRequestExecution( @@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver ITerrainModule terrainModule, IRegionSerialiserModule serialiser, Scene scene, - Stream saveStream, + TarArchiveWriter archiveWriter, Guid requestId) { m_sceneObjects = sceneObjects; m_terrainModule = terrainModule; m_serialiser = serialiser; m_scene = scene; - m_saveStream = saveStream; + m_archiveWriter = archiveWriter; m_requestId = requestId; } protected internal void ReceivedAllAssets( - IDictionary assetsFound, ICollection assetsNotFoundUuids) + ICollection assetsFoundUuids, ICollection assetsNotFoundUuids) { foreach (UUID uuid in assetsNotFoundUuids) { @@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_log.InfoFormat( "[ARCHIVER]: Received {0} of {1} assets requested", - assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count); + assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); - TarArchiveWriter archive = new TarArchiveWriter(m_saveStream); - // Write out control file - archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); + m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); // Write out region settings string settingsPath = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); - archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); + m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); @@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver MemoryStream ms = new MemoryStream(); m_terrainModule.SaveToStream(terrainPath, ms); - archive.WriteFile(terrainPath, ms.ToArray()); + m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); ms.Close(); m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); @@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), sceneObject.UUID); - archive.WriteFile(filename, serializedObject); + m_archiveWriter.WriteFile(filename, serializedObject); } m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); - // Write out assets - AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); - assetsArchiver.Archive(archive); - - archive.Close(); + m_archiveWriter.Close(); m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index a6ad24c700..0a882eb3bb 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs @@ -35,6 +35,7 @@ using System.Threading; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Serialization; using OpenSim.Region.CoreModules.World.Terrain; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; @@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) assetUuids[regionSettings.TerrainTexture4] = 1; + TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); + // Asynchronously request all the assets required to perform this archive operation ArchiveWriteRequestExecution awre = new ArchiveWriteRequestExecution( @@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_scene.RequestModuleInterface(), m_scene.RequestModuleInterface(), m_scene, - m_saveStream, - m_requestId); + archiveWriter, + m_requestId); - new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); + new AssetsRequest( + new AssetsArchiver(archiveWriter), assetUuids.Keys, + m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); } } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs index 29d7c5e69f..0c01cae772 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs @@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver /// /// Post a message to the log every x assets as a progress bar /// - private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; + protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; - /// - /// Archive assets - /// - protected IDictionary m_assets; + /// + /// Keep a count of the number of assets written so that we can provide status updates + /// + protected int m_assetsWritten; + + protected TarArchiveWriter m_archiveWriter; - public AssetsArchiver(IDictionary assets) + public AssetsArchiver(TarArchiveWriter archiveWriter) { - m_assets = assets; + m_archiveWriter = archiveWriter; } /// /// Archive the assets given to this archiver to the given archive. /// /// - public void Archive(TarArchiveWriter archive) - { + public void WriteAsset(AssetBase asset) + { //WriteMetadata(archive); - WriteData(archive); + WriteData(asset); } /// /// Write an assets metadata file to the given archive /// /// - protected void WriteMetadata(TarArchiveWriter archive) - { - StringWriter sw = new StringWriter(); - XmlTextWriter xtw = new XmlTextWriter(sw); - - xtw.Formatting = Formatting.Indented; - xtw.WriteStartDocument(); - - xtw.WriteStartElement("assets"); - - foreach (UUID uuid in m_assets.Keys) - { - AssetBase asset = m_assets[uuid]; - - if (asset != null) - { - xtw.WriteStartElement("asset"); - - string extension = string.Empty; - - if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) - { - extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; - } - - xtw.WriteElementString("filename", uuid.ToString() + extension); - - xtw.WriteElementString("name", asset.Name); - xtw.WriteElementString("description", asset.Description); - xtw.WriteElementString("asset-type", asset.Type.ToString()); - - xtw.WriteEndElement(); - } - } - - xtw.WriteEndElement(); - - xtw.WriteEndDocument(); - - archive.WriteFile("assets.xml", sw.ToString()); - } +// protected void WriteMetadata(TarArchiveWriter archive) +// { +// StringWriter sw = new StringWriter(); +// XmlTextWriter xtw = new XmlTextWriter(sw); +// +// xtw.Formatting = Formatting.Indented; +// xtw.WriteStartDocument(); +// +// xtw.WriteStartElement("assets"); +// +// foreach (UUID uuid in m_assets.Keys) +// { +// AssetBase asset = m_assets[uuid]; +// +// if (asset != null) +// { +// xtw.WriteStartElement("asset"); +// +// string extension = string.Empty; +// +// if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) +// { +// extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; +// } +// +// xtw.WriteElementString("filename", uuid.ToString() + extension); +// +// xtw.WriteElementString("name", asset.Name); +// xtw.WriteElementString("description", asset.Description); +// xtw.WriteElementString("asset-type", asset.Type.ToString()); +// +// xtw.WriteEndElement(); +// } +// } +// +// xtw.WriteEndElement(); +// +// xtw.WriteEndDocument(); +// +// archive.WriteFile("assets.xml", sw.ToString()); +// } /// /// Write asset data files to the given archive /// - /// - protected void WriteData(TarArchiveWriter archive) + /// + protected void WriteData(AssetBase asset) { // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar //archive.AddDir("assets"); - int assetsAdded = 0; + string extension = string.Empty; - foreach (UUID uuid in m_assets.Keys) + if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) { - AssetBase asset = m_assets[uuid]; - - string extension = string.Empty; - - if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) - { - extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; - } - else - { - m_log.ErrorFormat( - "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", - asset.Type, asset.ID); - } - - archive.WriteFile( - ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension, - asset.Data); - - assetsAdded++; - - if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) - m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); + extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; + } + else + { + m_log.ErrorFormat( + "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", + asset.Type, asset.ID); } - if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0) - m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); + m_archiveWriter.WriteFile( + ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension, + asset.Data); + + m_assetsWritten++; + + if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) + m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten); } } } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index 0a0bb4c524..d806a9cb64 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -32,6 +32,7 @@ using System.Threading; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Serialization; namespace OpenSim.Region.CoreModules.World.Archiver { @@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - /// + /// /// uuids to request - /// + /// protected ICollection m_uuids; - /// + /// /// Callback used when all the assets requested have been received. - /// + /// protected AssetsRequestCallback m_assetsRequestCallback; - /// - /// Assets retrieved in this request - /// - protected Dictionary m_assets = new Dictionary(); - - /// + /// + /// List of assets that were found. This will be passed back to the requester. + /// + protected List m_foundAssetUuids = new List(); + + /// /// Maintain a list of assets that could not be found. This will be passed back to the requester. - /// + /// protected List m_notFoundAssetUuids = new List(); - /// + /// /// Record the number of asset replies required so we know when we've finished - /// + /// private int m_repliesRequired; - /// + /// /// Asset cache used to request the assets - /// + /// protected IAssetCache m_assetCache; + protected AssetsArchiver m_assetsArchiver; + protected internal AssetsRequest( - ICollection uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) + AssetsArchiver assetsArchiver, ICollection uuids, + IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) { + m_assetsArchiver = assetsArchiver; m_uuids = uuids; m_assetsRequestCallback = assetsRequestCallback; m_assetCache = assetCache; @@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We can stop here if there are no assets to fetch if (m_repliesRequired == 0) - m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); + m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); foreach (UUID uuid in m_uuids) { @@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (asset != null) { + // Make sure that we don't run out of memory by hogging assets in the cache m_assetCache.ExpireAsset(assetID); - m_assets[assetID] = asset; + + m_foundAssetUuids.Add(assetID); + m_assetsArchiver.WriteAsset(asset); } else { m_notFoundAssetUuids.Add(assetID); } - if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired) + if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired) { m_log.DebugFormat( "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", - m_assets.Count, m_notFoundAssetUuids.Count); + m_foundAssetUuids.Count, m_notFoundAssetUuids.Count); - // We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive + // We want to stop using the asset cache thread asap + // as we now need to do the work of producing the rest of the archive Thread newThread = new Thread(PerformAssetsRequestCallback); newThread.Name = "OpenSimulator archiving thread post assets receipt"; newThread.Start(); @@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver { try { - m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); + m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); } catch (Exception e) {