OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* Improve memory usage when writing OARs 

* This should make saving large OARs a somewhat better experience
* However, the problem where saving an archive pulls large numbers of assets into the asset cache isn't yet resolved
* This patch also removes lots of archive writing spam that crept in
0.6.4-rc1
Justin Clarke Casey 2009-03-06 20:12:08 +00:00
parent 08509d5cf2
commit 85774de231
6 changed files with 81 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
View File

@ -45,7 +45,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
{ {
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected TarArchiveWriter archive = new TarArchiveWriter(); protected TarArchiveWriter m_archive;
protected UuidGatherer m_assetGatherer; protected UuidGatherer m_assetGatherer;
protected Dictionary<UUID, int> assetUuids = new Dictionary<UUID, int>(); protected Dictionary<UUID, int> assetUuids = new Dictionary<UUID, int>();
@ -87,14 +87,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids) protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
{ {
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(archive); assetsArchiver.Archive(m_archive);
Exception reportedException = null; Exception reportedException = null;
bool succeeded = true; bool succeeded = true;
try try
{ {
archive.WriteTar(m_saveStream); m_archive.Close();
} }
catch (IOException e) catch (IOException e)
{ {
@ -172,7 +172,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
writer.WriteEndElement(); writer.WriteEndElement();
archive.AddFile(filename, sw.ToString()); m_archive.WriteFile(filename, sw.ToString());
m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (AssetType)inventoryItem.AssetType, assetUuids); m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (AssetType)inventoryItem.AssetType, assetUuids);
} }
@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryFolder.Name, inventoryFolder.Name,
InventoryArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR, InventoryArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR,
inventoryFolder.ID); inventoryFolder.ID);
archive.AddDir(path); m_archive.WriteDir(path);
List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls(); List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
List<InventoryItemBase> items = inventoryFolder.RequestListOfItems(); List<InventoryItemBase> items = inventoryFolder.RequestListOfItems();
@ -280,6 +280,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
} }
m_archive = new TarArchiveWriter(m_saveStream);
if (null == inventoryFolder) if (null == inventoryFolder)
{ {
if (null == inventoryItem) if (null == inventoryItem)

12
OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
View File

@ -85,17 +85,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
TarArchiveWriter archive = new TarArchiveWriter(); TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
// Write out control file // Write out control file
archive.AddFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
// Write out region settings // Write out region settings
string settingsPath string settingsPath
= String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
archive.AddFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
@ -105,7 +105,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
m_terrainModule.SaveToStream(terrainPath, ms); m_terrainModule.SaveToStream(terrainPath, ms);
archive.AddFile(terrainPath, ms.ToArray()); archive.WriteFile(terrainPath, ms.ToArray());
ms.Close(); ms.Close();
m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@ -125,7 +125,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
sceneObject.UUID); sceneObject.UUID);
archive.AddFile(filename, serializedObject); archive.WriteFile(filename, serializedObject);
} }
m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
@ -134,7 +134,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(archive); assetsArchiver.Archive(archive);
archive.WriteTar(m_saveStream); archive.Close();
m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);

4
OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
View File

@ -110,7 +110,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
xtw.WriteEndDocument(); xtw.WriteEndDocument();
archive.AddFile("assets.xml", sw.ToString()); archive.WriteFile("assets.xml", sw.ToString());
} }
/// <summary> /// <summary>
@ -141,7 +141,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
asset.Type, asset.ID); asset.Type, asset.ID);
} }
archive.AddFile( archive.WriteFile(
ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension, ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
asset.Data); asset.Data);

10
OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs
View File

@ -38,7 +38,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
/// </summary> /// </summary>
public class TarArchiveReader public class TarArchiveReader
{ {
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public enum TarEntryType public enum TarEntryType
{ {
@ -113,14 +113,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{ {
int longNameLength = ConvertOctalBytesToDecimal(header, 124, 11); int longNameLength = ConvertOctalBytesToDecimal(header, 124, 11);
tarHeader.FilePath = m_asciiEncoding.GetString(ReadData(longNameLength)); tarHeader.FilePath = m_asciiEncoding.GetString(ReadData(longNameLength));
m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath); //m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath);
header = m_br.ReadBytes(512); header = m_br.ReadBytes(512);
} }
else else
{ {
tarHeader.FilePath = m_asciiEncoding.GetString(header, 0, 100); tarHeader.FilePath = m_asciiEncoding.GetString(header, 0, 100);
tarHeader.FilePath = tarHeader.FilePath.Trim(m_nullCharArray); tarHeader.FilePath = tarHeader.FilePath.Trim(m_nullCharArray);
m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath); //m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath);
} }
tarHeader.FileSize = ConvertOctalBytesToDecimal(header, 124, 11); tarHeader.FileSize = ConvertOctalBytesToDecimal(header, 124, 11);
@ -168,14 +168,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{ {
byte[] data = m_br.ReadBytes(fileSize); byte[] data = m_br.ReadBytes(fileSize);
m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize); //m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize);
// Read the rest of the empty padding in the 512 byte block // Read the rest of the empty padding in the 512 byte block
if (fileSize % 512 != 0) if (fileSize % 512 != 0)
{ {
int paddingLeft = 512 - (fileSize % 512); int paddingLeft = 512 - (fileSize % 512);
m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft); //m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft);
m_br.ReadBytes(paddingLeft); m_br.ReadBytes(paddingLeft);
} }

96
OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs
View File

@ -39,65 +39,80 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{ {
//private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected Dictionary<string, byte[]> m_files = new Dictionary<string, byte[]>();
protected static ASCIIEncoding m_asciiEncoding = new ASCIIEncoding(); protected static ASCIIEncoding m_asciiEncoding = new ASCIIEncoding();
/// <summary> /// <summary>
/// Add a directory to the tar archive. We can only handle one path level right now! /// Binary writer for the underlying stream
/// </summary>
protected BinaryWriter m_bw;
public TarArchiveWriter(Stream s)
{
m_bw = new BinaryWriter(s);
}
/// <summary>
/// Write a directory entry to the tar archive. We can only handle one path level right now!
/// </summary> /// </summary>
/// <param name="dirName"></param> /// <param name="dirName"></param>
public void AddDir(string dirName) public void WriteDir(string dirName)
{ {
// Directories are signalled by a final / // Directories are signalled by a final /
if (!dirName.EndsWith("/")) if (!dirName.EndsWith("/"))
dirName += "/"; dirName += "/";
AddFile(dirName, new byte[0]); WriteFile(dirName, new byte[0]);
} }
/// <summary> /// <summary>
/// Add a file to the tar archive /// Write a file to the tar archive
/// </summary> /// </summary>
/// <param name="filePath"></param> /// <param name="filePath"></param>
/// <param name="data"></param> /// <param name="data"></param>
public void AddFile(string filePath, string data) public void WriteFile(string filePath, string data)
{ {
AddFile(filePath, m_asciiEncoding.GetBytes(data)); WriteFile(filePath, m_asciiEncoding.GetBytes(data));
} }
/// <summary> /// <summary>
/// Add a file to the tar archive /// Write a file to the tar archive
/// </summary> /// </summary>
/// <param name="filePath"></param> /// <param name="filePath"></param>
/// <param name="data"></param> /// <param name="data"></param>
public void AddFile(string filePath, byte[] data) public void WriteFile(string filePath, byte[] data)
{ {
m_files[filePath] = data; if (filePath.Length > 100)
WriteEntry("././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
char fileType;
if (filePath.EndsWith("/"))
{
fileType = '5';
}
else
{
fileType = '0';
}
WriteEntry(filePath, data, fileType);
} }
/// <summary> /// <summary>
/// Write the raw tar archive data to a stream. The stream will be closed on completion. /// Finish writing the raw tar archive data to a stream. The stream will be closed on completion.
/// </summary> /// </summary>
/// <param name="s">Stream to which to write the data</param> /// <param name="s">Stream to which to write the data</param>
/// <returns></returns> /// <returns></returns>
public void WriteTar(Stream s) public void Close()
{ {
BinaryWriter bw = new BinaryWriter(s);
foreach (string filePath in m_files.Keys)
{
WriteFile(bw, filePath, m_files[filePath]);
}
//m_log.Debug("[TAR ARCHIVE WRITER]: Writing final consecutive 0 blocks"); //m_log.Debug("[TAR ARCHIVE WRITER]: Writing final consecutive 0 blocks");
// Write two consecutive 0 blocks to end the archive // Write two consecutive 0 blocks to end the archive
byte[] finalZeroPadding = new byte[1024]; byte[] finalZeroPadding = new byte[1024];
bw.Write(finalZeroPadding); m_bw.Write(finalZeroPadding);
bw.Flush(); m_bw.Flush();
bw.Close(); m_bw.Close();
} }
public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
@ -121,37 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
} }
/// <summary> /// <summary>
/// Write a particular file of data /// Write a particular entry
/// </summary> /// </summary>
/// <param name="filePath"></param> /// <param name="filePath"></param>
/// <param name="data"></param> /// <param name="data"></param>
protected void WriteFile(BinaryWriter bw, string filePath, byte[] data)
{
if (filePath.Length > 100)
WriteEntry(bw, "././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
char fileType;
if (filePath.EndsWith("/"))
{
fileType = '5';
}
else
{
fileType = '0';
}
WriteEntry(bw, filePath, data, fileType);
}
/// <summary>
/// Write a particular file of data
/// </summary>
/// <param name="bw"></param>
/// <param name="filePath"></param>
/// <param name="data"></param>
/// <param name="fileType"></param> /// <param name="fileType"></param>
protected void WriteEntry(BinaryWriter bw, string filePath, byte[] data, char fileType) protected void WriteEntry(string filePath, byte[] data, char fileType)
{ {
byte[] header = new byte[512]; byte[] header = new byte[512];
@ -208,10 +198,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver
header[154] = 0; header[154] = 0;
// Write out header // Write out header
bw.Write(header); m_bw.Write(header);
// Write out data // Write out data
bw.Write(data); m_bw.Write(data);
if (data.Length % 512 != 0) if (data.Length % 512 != 0)
{ {
@ -220,7 +210,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
byte[] padding = new byte[paddingRequired]; byte[] padding = new byte[paddingRequired];
bw.Write(padding); m_bw.Write(padding);
} }
} }
} }

8
OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs
View File

@ -173,9 +173,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
//log4net.Config.XmlConfigurator.Configure(); //log4net.Config.XmlConfigurator.Configure();
MemoryStream archiveWriteStream = new MemoryStream(); MemoryStream archiveWriteStream = new MemoryStream();
TarArchiveWriter tar = new TarArchiveWriter(); TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream);
tar.AddFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile()); tar.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile());
string part1Name = "object1"; string part1Name = "object1";
PrimitiveBaseShape shape = PrimitiveBaseShape.CreateCylinder(); PrimitiveBaseShape shape = PrimitiveBaseShape.CreateCylinder();
@ -194,9 +194,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
part1Name, part1Name,
Math.Round(groupPosition.X), Math.Round(groupPosition.Y), Math.Round(groupPosition.Z), Math.Round(groupPosition.X), Math.Round(groupPosition.Y), Math.Round(groupPosition.Z),
part1.UUID); part1.UUID);
tar.AddFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2()); tar.WriteFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2());
tar.WriteTar(archiveWriteStream); tar.Close();
MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray()); MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray());