diff --git a/ValvePak/ValvePak.Test/ValvePak.Test.csproj b/ValvePak/ValvePak.Test/ValvePak.Test.csproj index 5f853f4..eda7817 100644 --- a/ValvePak/ValvePak.Test/ValvePak.Test.csproj +++ b/ValvePak/ValvePak.Test/ValvePak.Test.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 0567043..6fbf37d 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -1,13 +1,25 @@ +using Microsoft.VisualBasic.FileIO; using System; using System.Buffers; +using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Hashing; +using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Text.RegularExpressions; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace SteamDatabase.ValvePak { + internal sealed class WriteEntry(ushort archiveIndex, uint fileOffset, PackageEntry entry) + { + internal ushort ArchiveIndex { get; set; } = archiveIndex; + internal uint FileOffset { get; set; } = fileOffset; + internal PackageEntry Entry { get; set; } = entry; + } public partial class Package { /// @@ -101,37 +113,38 @@ public PackageEntry AddFile(string filePath, byte[] fileData) /// Opens and writes the given filename. /// /// The file to open and write. - public void Write(string filename) + public void Write(string filename, int maxFileBytes = int.MaxValue) { - using var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + ArgumentOutOfRangeException.ThrowIfNegative(maxFileBytes); + + using var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None); fs.SetLength(0); - Write(fs); + Write(fs, maxFileBytes); } /// /// Writes to the given . /// /// The input to write to. - public void Write(Stream stream) + public void Write(FileStream stream, int maxFileBytes) { - if (IsDirVPK) - { - throw new InvalidOperationException("This package was opened from a _dir.vpk, writing back is currently unsupported."); - } ArgumentNullException.ThrowIfNull(stream); if (!stream.CanSeek || !stream.CanRead) - { throw new InvalidOperationException("Stream must be seekable and readable."); - } using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); // TODO: input.SetLength() var streamOffset = stream.Position; - ulong fileDataSectionSize = 0; + + List entries = Entries.SelectMany(e => e.Value).ToList(); + + if (entries.Any(e => e.TotalLength > maxFileBytes)) + throw new InvalidOperationException("There are files exceeding max file bytes"); + var tree = new Dictionary>>(); @@ -152,13 +165,6 @@ public void Write(Stream stream) } directoryEntries.Add(entry); - - fileDataSectionSize += entry.TotalLength; - - if (fileDataSectionSize > int.MaxValue) - { - throw new InvalidOperationException("Package contents exceed 2GiB, and splitting packages is currently unsupported."); - } } } @@ -168,15 +174,20 @@ public void Write(Stream stream) writer.Write(0); // TreeSize, to be updated later writer.Write(0); // FileDataSectionSize, to be updated later writer.Write(0); // ArchiveMD5SectionSize - writer.Write(48); // OtherMD5SectionSize + writer.Write(48); //OtherMD5SectionSize writer.Write(0); // SignatureSectionSize - var headerSize = (int)(stream.Position - streamOffset); - uint fileOffset = 0; - const byte NullByte = 0; - // File tree data + bool isSingleFile = entries.Sum(s => s.TotalLength) + headerSize + 64 <= maxFileBytes; + + var groups = CreatePacketsGroup(entries, maxFileBytes, isSingleFile); + + if (groups.Count >= 0x7FFF) + throw new InvalidOperationException("The number of packages exceeds 32766"); + + + uint fileOffset = 0; foreach (var typeEntries in tree) { writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); @@ -191,12 +202,24 @@ public void Write(Stream stream) { var fileLength = entry.TotalLength; + var fullPath = entry.GetFullPath(); + WriteEntry writeEntry = null; + + foreach (var group in groups) + { + if (group.TryGetValue(fullPath, out writeEntry)) + break; + } + if (writeEntry is null) + throw new InvalidOperationException("No need write entry found"); + + writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); writer.Write(NullByte); writer.Write(entry.CRC32); writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(entry.ArchiveIndex); - writer.Write(fileOffset); + writer.Write(writeEntry.ArchiveIndex); + writer.Write(writeEntry.FileOffset); writer.Write(fileLength); writer.Write(ushort.MaxValue); // terminator, 0xFFFF @@ -211,22 +234,47 @@ public void Write(Stream stream) writer.Write(NullByte); - var fileTreeSize = stream.Position - headerSize; + //clear sub file + for (ushort i = 0; i < 999; i++) + { + string sub_FilePath = GetSubFilePath(stream.Name, i); + if (File.Exists(sub_FilePath)) + File.Delete(sub_FilePath); + } - // File data - foreach (var typeEntries in tree) + if (isSingleFile) { - foreach (var directoryEntries in typeEntries.Value) + //Write file data + foreach (var writeEntry in groups[0].Values) { - foreach (var entry in directoryEntries.Value) - { - ReadEntry(entry, out var fileData, validateCrc: false); + ReadEntry(writeEntry.Entry, out var fileData, validateCrc: false); + writer.Write(fileData); + } + } + else + { + //Create and write sub file data + for (ushort i = 0; i < groups.Count; i++) + { + string sub_FilePath = GetSubFilePath(stream.Name, i); - writer.Write(fileData); + using var fs = new FileStream(sub_FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + using var writer_sub = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true); + + var group = groups[i]; + foreach (var writeEntry in group.Values) + { + ReadEntry(writeEntry.Entry, out var fileData, validateCrc: false); + writer_sub.Write(fileData); } } + } + + long fileTreeSize = stream.Position - headerSize; + + var afterFileData = stream.Position; var fileDataSize = afterFileData - fileTreeSize - headerSize; @@ -293,5 +341,76 @@ public void Write(Stream stream) ArrayPool.Shared.Return(buffer); } } + + /// + /// Get the sub file name + /// + /// Index file path + /// Index number + /// + static string GetSubFilePath(string indexFilePath, ushort indexNumber) + { + FileInfo sub_FileInfo = new FileInfo(indexFilePath); + + string sub_FileName = Path.GetFileNameWithoutExtension(sub_FileInfo.FullName); + if (sub_FileName.EndsWith("_dir", StringComparison.OrdinalIgnoreCase)) + sub_FileName = $"{sub_FileName[..^4]}"; + + sub_FileName = $"{sub_FileName}_{indexNumber:D3}"; + return $"{sub_FileInfo.Directory}\\{sub_FileName}{sub_FileInfo.Extension}"; + } + + /// + /// Split the current tree into multiple trees based on packet size + /// + /// Tree of data sources + /// Maximum file byte count + /// List of Trees + + static List> CreatePacketsGroup(List entries, int maxFileBytes, bool isSingleFile) + { + List> groups = new List>(); + uint totalLength = 0; + ushort archiveIndex = 0; + Dictionary group = new Dictionary(); + groups.Add(group); + + if (isSingleFile) + { + foreach (var entry in entries) + { + group.Add(entry.GetFullPath(), new(0x7FFF, totalLength, entry)); + totalLength += entry.TotalLength; + } + } + else + { + group.Add(entries[0].GetFullPath(), new(archiveIndex, totalLength, entries[0])); + totalLength += entries[0].TotalLength; + + entries.RemoveAt(0); + do + { + PackageEntry entry = entries.Find(e => e.TotalLength < (ulong)maxFileBytes - totalLength); + if (entry is not null) + { + group.Add(entry.GetFullPath(), new(archiveIndex, totalLength, entry)); + totalLength += entry.TotalLength; + entries.Remove(entry); + } + else + { + group = new Dictionary(); + groups.Add(group); + totalLength = 0; + archiveIndex++; + } + + } while (entries.Count != 0); + } + + + return groups; + } } }