diff --git a/src/Ramstack.FileSystem.Abstractions/Null/NotFoundDirectory.cs b/src/Ramstack.FileSystem.Abstractions/Null/NotFoundDirectory.cs index 4ffef93..5d05bab 100644 --- a/src/Ramstack.FileSystem.Abstractions/Null/NotFoundDirectory.cs +++ b/src/Ramstack.FileSystem.Abstractions/Null/NotFoundDirectory.cs @@ -5,7 +5,7 @@ namespace Ramstack.FileSystem.Null; /// /// Represents a non-existing directory. /// -public class NotFoundDirectory : VirtualDirectory +public sealed class NotFoundDirectory : VirtualDirectory { /// public override IVirtualFileSystem FileSystem { get; } @@ -27,8 +27,11 @@ protected override ValueTask ExistsCoreAsync(CancellationToken cancellatio default; /// - protected override ValueTask CreateCoreAsync(CancellationToken cancellationToken) => - default; + protected override ValueTask CreateCoreAsync(CancellationToken cancellationToken) + { + Error_UnauthorizedAccess(FullName); + return default; + } /// protected override ValueTask DeleteCoreAsync(CancellationToken cancellationToken) => @@ -37,4 +40,8 @@ protected override ValueTask DeleteCoreAsync(CancellationToken cancellationToken /// protected override IAsyncEnumerable GetFileNodesCoreAsync(CancellationToken cancellationToken) => Array.Empty().ToAsyncEnumerable(); + + [DoesNotReturn] + private static void Error_UnauthorizedAccess(string path) => + throw new UnauthorizedAccessException($"Access to the path '{path}' is denied."); } diff --git a/src/Ramstack.FileSystem.Abstractions/Null/NotFoundFile.cs b/src/Ramstack.FileSystem.Abstractions/Null/NotFoundFile.cs index 8755d92..241a8f4 100644 --- a/src/Ramstack.FileSystem.Abstractions/Null/NotFoundFile.cs +++ b/src/Ramstack.FileSystem.Abstractions/Null/NotFoundFile.cs @@ -1,5 +1,3 @@ -using System.Diagnostics.CodeAnalysis; - namespace Ramstack.FileSystem.Null; /// @@ -43,7 +41,7 @@ protected override ValueTask OpenWriteCoreAsync(CancellationToken cancel /// protected override ValueTask WriteCoreAsync(Stream stream, bool overwrite, CancellationToken cancellationToken) { - Error_FileNotFound(FullName); + Error_UnauthorizedAccess(FullName); return default; } @@ -54,4 +52,8 @@ protected override ValueTask DeleteCoreAsync(CancellationToken cancellationToken [DoesNotReturn] private static void Error_FileNotFound(string path) => throw new FileNotFoundException($"Unable to find file '{path}'.", path); + + [DoesNotReturn] + private static void Error_UnauthorizedAccess(string path) => + throw new UnauthorizedAccessException($"Access to the path '{path}' is denied."); } diff --git a/src/Ramstack.FileSystem.Abstractions/Ramstack.FileSystem.Abstractions.csproj b/src/Ramstack.FileSystem.Abstractions/Ramstack.FileSystem.Abstractions.csproj index 2471127..742f17a 100644 --- a/src/Ramstack.FileSystem.Abstractions/Ramstack.FileSystem.Abstractions.csproj +++ b/src/Ramstack.FileSystem.Abstractions/Ramstack.FileSystem.Abstractions.csproj @@ -35,6 +35,12 @@ true + + + + + + all diff --git a/src/Ramstack.FileSystem.Abstractions/VirtualDirectory.cs b/src/Ramstack.FileSystem.Abstractions/VirtualDirectory.cs index 21c50fc..c665afb 100644 --- a/src/Ramstack.FileSystem.Abstractions/VirtualDirectory.cs +++ b/src/Ramstack.FileSystem.Abstractions/VirtualDirectory.cs @@ -1,6 +1,3 @@ -using System.Diagnostics; -using System.Runtime.CompilerServices; - using Ramstack.Globbing.Traversal; namespace Ramstack.FileSystem; @@ -121,16 +118,7 @@ public IAsyncEnumerable GetDirectoriesAsync(CancellationToken public IAsyncEnumerable GetFileNodesAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(patterns); - - return new FileTreeAsyncEnumerable(this, cancellationToken) - { - Patterns = patterns, - Excludes = excludes ?? [], - FileNameSelector = node => node.FullName, - ShouldRecursePredicate = node => node is VirtualDirectory, - ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), - ResultSelector = node => node - }; + return GetFileNodesCoreAsync(patterns, excludes, cancellationToken); } /// @@ -146,17 +134,7 @@ public IAsyncEnumerable GetFileNodesAsync(string[] patterns, string public IAsyncEnumerable GetFilesAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(patterns); - - return new FileTreeAsyncEnumerable(this, cancellationToken) - { - Patterns = patterns, - Excludes = excludes ?? [], - FileNameSelector = node => node.FullName, - ShouldIncludePredicate = node => node is VirtualFile, - ShouldRecursePredicate = node => node is VirtualDirectory, - ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), - ResultSelector = node => (VirtualFile)node - }; + return GetFilesCoreAsync(patterns, excludes, cancellationToken); } /// @@ -172,17 +150,7 @@ public IAsyncEnumerable GetFilesAsync(string[] patterns, string[]? public IAsyncEnumerable GetDirectoriesAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(patterns); - - return new FileTreeAsyncEnumerable(this, cancellationToken) - { - Patterns = patterns, - Excludes = excludes ?? [], - FileNameSelector = node => node.FullName, - ShouldIncludePredicate = node => node is VirtualDirectory, - ShouldRecursePredicate = node => node is VirtualDirectory, - ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), - ResultSelector = node => (VirtualDirectory)node - }; + return GetDirectoriesCoreAsync(patterns, excludes, cancellationToken); } /// @@ -243,4 +211,75 @@ protected virtual async IAsyncEnumerable GetDirectoriesCoreAsy yield return directory; } } + + /// + /// Core implementation for asynchronously returning an async-enumerable collection of file nodes (both directories and files) + /// within the current directory that match any of the specified glob patterns. + /// + /// An array of glob patterns to match against the names of file nodes. + /// An optional array of glob patterns to exclude file nodes. + /// An optional cancellation token to cancel the operation. + /// + /// An async-enumerable collection of instances. + /// + protected virtual IAsyncEnumerable GetFileNodesCoreAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken) + { + return new FileTreeAsyncEnumerable(this, cancellationToken) + { + Patterns = patterns, + Excludes = excludes ?? [], + FileNameSelector = node => node.Name, + ShouldRecursePredicate = node => node is VirtualDirectory, + ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), + ResultSelector = node => node + }; + } + + /// + /// Core implementation for asynchronously returning an async-enumerable collection of files within the current directory + /// that match any of the specified glob patterns. + /// + /// An array of glob patterns to match against the names of files. + /// An optional array of glob patterns to exclude files. + /// An optional cancellation token to cancel the operation. + /// + /// An async-enumerable collection of instances. + /// + protected virtual IAsyncEnumerable GetFilesCoreAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken) + { + return new FileTreeAsyncEnumerable(this, cancellationToken) + { + Patterns = patterns, + Excludes = excludes ?? [], + FileNameSelector = node => node.Name, + ShouldIncludePredicate = node => node is VirtualFile, + ShouldRecursePredicate = node => node is VirtualDirectory, + ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), + ResultSelector = node => (VirtualFile)node + }; + } + + /// + /// Core implementation for asynchronously returning an async-enumerable collection of directories within the current directory + /// that match any of the specified glob patterns. + /// + /// An array of glob patterns to match against the names of directories. + /// An optional array of glob patterns to exclude directories. + /// An optional cancellation token to cancel the operation. + /// + /// An async-enumerable collection of instances. + /// + protected virtual IAsyncEnumerable GetDirectoriesCoreAsync(string[] patterns, string[]? excludes, CancellationToken cancellationToken) + { + return new FileTreeAsyncEnumerable(this, cancellationToken) + { + Patterns = patterns, + Excludes = excludes ?? [], + FileNameSelector = node => node.Name, + ShouldIncludePredicate = node => node is VirtualDirectory, + ShouldRecursePredicate = node => node is VirtualDirectory, + ChildrenSelector = (node, token) => ((VirtualDirectory)node).GetFileNodesCoreAsync(token), + ResultSelector = node => (VirtualDirectory)node + }; + } } diff --git a/src/Ramstack.FileSystem.Abstractions/VirtualFile.cs b/src/Ramstack.FileSystem.Abstractions/VirtualFile.cs index d0f2065..76d74b9 100644 --- a/src/Ramstack.FileSystem.Abstractions/VirtualFile.cs +++ b/src/Ramstack.FileSystem.Abstractions/VirtualFile.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; - namespace Ramstack.FileSystem; /// diff --git a/src/Ramstack.FileSystem.Abstractions/VirtualNode.cs b/src/Ramstack.FileSystem.Abstractions/VirtualNode.cs index 62e7610..e7db08c 100644 --- a/src/Ramstack.FileSystem.Abstractions/VirtualNode.cs +++ b/src/Ramstack.FileSystem.Abstractions/VirtualNode.cs @@ -1,6 +1,3 @@ -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; - namespace Ramstack.FileSystem; /// diff --git a/src/Ramstack.FileSystem.Abstractions/VirtualNodeProperties.cs b/src/Ramstack.FileSystem.Abstractions/VirtualNodeProperties.cs index 503f9c5..32bd808 100644 --- a/src/Ramstack.FileSystem.Abstractions/VirtualNodeProperties.cs +++ b/src/Ramstack.FileSystem.Abstractions/VirtualNodeProperties.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; - namespace Ramstack.FileSystem; /// diff --git a/src/Ramstack.FileSystem.Abstractions/VirtualPath.cs b/src/Ramstack.FileSystem.Abstractions/VirtualPath.cs index d2a700e..ef964e0 100644 --- a/src/Ramstack.FileSystem.Abstractions/VirtualPath.cs +++ b/src/Ramstack.FileSystem.Abstractions/VirtualPath.cs @@ -118,7 +118,8 @@ public static ReadOnlySpan GetFileName(ReadOnlySpan path) /// /// The path to retrieve the directory portion from. /// - /// Directory portion for , or an empty string if the path denotes a root directory. + /// The directory portion of , or an empty span if + /// is empty or denotes a root directory. /// public static string GetDirectoryName(string path) { @@ -140,7 +141,8 @@ public static string GetDirectoryName(string path) /// /// The path to retrieve the directory portion from. /// - /// Directory portion for , or an empty string if path denotes a root directory. + /// The directory portion of , or an empty span if + /// is empty or denotes a root directory. /// public static ReadOnlySpan GetDirectoryName(ReadOnlySpan path) { diff --git a/src/Ramstack.FileSystem.Amazon/S3Directory.cs b/src/Ramstack.FileSystem.Amazon/S3Directory.cs index 2e1641a..7d4aae9 100644 --- a/src/Ramstack.FileSystem.Amazon/S3Directory.cs +++ b/src/Ramstack.FileSystem.Amazon/S3Directory.cs @@ -2,6 +2,8 @@ using Amazon.S3.Model; +using Ramstack.Globbing; + namespace Ramstack.FileSystem.Amazon; /// @@ -11,7 +13,6 @@ namespace Ramstack.FileSystem.Amazon; internal sealed class S3Directory : VirtualDirectory { private readonly AmazonS3FileSystem _fs; - private readonly string _prefix; /// public override IVirtualFileSystem FileSystem => _fs; @@ -21,11 +22,8 @@ internal sealed class S3Directory : VirtualDirectory /// /// The file system associated with this directory. /// The path to the directory within the specified Amazon S3 bucket. - public S3Directory(AmazonS3FileSystem fileSystem, string path) : base(path) - { + public S3Directory(AmazonS3FileSystem fileSystem, string path) : base(path) => _fs = fileSystem; - _prefix = path == "/" ? "" : $"{path[1..]}/"; - } /// protected override ValueTask GetPropertiesCoreAsync(CancellationToken cancellationToken) => @@ -41,7 +39,7 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio var lr = new ListObjectsV2Request { BucketName = _fs.BucketName, - Prefix = _prefix + Prefix = GetPrefix(FullName) }; var dr = new DeleteObjectsRequest @@ -80,7 +78,7 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En var request = new ListObjectsV2Request { BucketName = _fs.BucketName, - Prefix = _prefix, + Prefix = GetPrefix(FullName), Delimiter = "/" }; @@ -94,7 +92,7 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En yield return new S3Directory(_fs, VirtualPath.Normalize(prefix)); foreach (var obj in response.S3Objects) - yield return new S3File(_fs, VirtualPath.Normalize(obj.Key)); + yield return CreateVirtualFile(obj); request.ContinuationToken = response.NextContinuationToken; } @@ -107,7 +105,7 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer var request = new ListObjectsV2Request { BucketName = _fs.BucketName, - Prefix = _prefix, + Prefix = GetPrefix(FullName), Delimiter = "/" }; @@ -118,7 +116,7 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer .ConfigureAwait(false); foreach (var obj in response.S3Objects) - yield return new S3File(_fs, VirtualPath.Normalize(obj.Key)); + yield return CreateVirtualFile(obj); request.ContinuationToken = response.NextContinuationToken; } @@ -131,7 +129,7 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs var request = new ListObjectsV2Request { BucketName = _fs.BucketName, - Prefix = _prefix, + Prefix = GetPrefix(FullName), Delimiter = "/" }; @@ -148,4 +146,223 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs } while (request.ContinuationToken is not null && !cancellationToken.IsCancellationRequested); } + + /// + protected override async IAsyncEnumerable GetFileNodesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var request = new ListObjectsV2Request + { + BucketName = _fs.BucketName, + Prefix = GetPrefix(FullName) + }; + + var directories = new HashSet + { + FullName + }; + + do + { + var response = await _fs.AmazonClient + .ListObjectsV2Async(request, cancellationToken) + .ConfigureAwait(false); + + foreach (var obj in response.S3Objects) + { + var directoryPath = VirtualPath.GetDirectoryName( + VirtualPath.Join("/", obj.Key)); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new S3Directory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + + if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj); + } + + request.ContinuationToken = response.NextContinuationToken; + } + while (request.ContinuationToken is not null && !cancellationToken.IsCancellationRequested); + } + + /// + protected override async IAsyncEnumerable GetFilesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var request = new ListObjectsV2Request + { + BucketName = _fs.BucketName, + Prefix = GetPrefix(FullName) + }; + + do + { + var response = await _fs.AmazonClient + .ListObjectsV2Async(request, cancellationToken) + .ConfigureAwait(false); + + foreach (var obj in response.S3Objects) + if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj); + + request.ContinuationToken = response.NextContinuationToken; + } + while (request.ContinuationToken is not null && !cancellationToken.IsCancellationRequested); + } + + /// + protected override async IAsyncEnumerable GetDirectoriesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var request = new ListObjectsV2Request + { + BucketName = _fs.BucketName, + Prefix = GetPrefix(FullName) + }; + + var directories = new HashSet + { + FullName + }; + + do + { + var response = await _fs.AmazonClient + .ListObjectsV2Async(request, cancellationToken) + .ConfigureAwait(false); + + foreach (var obj in response.S3Objects) + { + var directoryPath = VirtualPath.GetDirectoryName( + VirtualPath.Join("/", obj.Key)); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new S3Directory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + } + + request.ContinuationToken = response.NextContinuationToken; + } + while (request.ContinuationToken is not null && !cancellationToken.IsCancellationRequested); + } + + /// + /// Creates a instance based on the specified object. + /// + /// The representing the file. + /// + /// A new instance representing the file. + /// + private S3File CreateVirtualFile(S3Object obj) + { + var properties = VirtualNodeProperties + .CreateFileProperties( + creationTime: default, + lastAccessTime: default, + lastWriteTime: obj.LastModified, + length: obj.Size); + + var path = VirtualPath.Normalize(obj.Key); + return new S3File(_fs, path, properties); + } + + /// + /// Determines whether the specified path matches any of the inclusion patterns and none of the exclusion patterns. + /// + /// The path to match. + /// The inclusion patterns to match against the path. + /// An optional array of exclusion patterns. If the path matches any of these, + /// the method returns . + /// + /// if the path matches at least one inclusion pattern and no exclusion patterns; + /// otherwise, . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsMatched(scoped ReadOnlySpan path, string[] patterns, string[]? excludes) + { + if (excludes is not null) + foreach (var pattern in excludes) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return false; + + foreach (var pattern in patterns) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return true; + + return false; + } + + /// + /// Returns a cloud storage compatible prefix for the specified directory path. + /// + /// The directory path. + /// + /// A string formatted for cloud storage prefix filtering: + /// + /// Empty string for the root directory ("/") + /// Path without leading slash but with trailing slash for subdirectories + /// + /// + /// + /// + /// GetPrefix("/") // returns "" + /// GetPrefix("/folder") // returns "folder/" + /// GetPrefix("/sub/folder") // returns "sub/folder/" + /// + /// + private static string GetPrefix(string path) => + path == "/" ? "" : $"{path[1..]}/"; } diff --git a/src/Ramstack.FileSystem.Amazon/S3File.cs b/src/Ramstack.FileSystem.Amazon/S3File.cs index 63792eb..cdca028 100644 --- a/src/Ramstack.FileSystem.Amazon/S3File.cs +++ b/src/Ramstack.FileSystem.Amazon/S3File.cs @@ -24,6 +24,15 @@ internal sealed class S3File : VirtualFile public S3File(AmazonS3FileSystem fileSystem, string path) : base(path) => (_fs, _key) = (fileSystem, path[1..]); + /// + /// Initializes a new instance of the class with the specified path and properties. + /// + /// The file system associated with this file. + /// The full path of the file. + /// The properties of the file. + public S3File(AmazonS3FileSystem fileSystem, string path, VirtualNodeProperties? properties) : base(path, properties) => + (_fs, _key) = (fileSystem, path[1..]); + /// protected override async ValueTask GetPropertiesCoreAsync(CancellationToken cancellationToken) { diff --git a/src/Ramstack.FileSystem.Azure/AzureDirectory.cs b/src/Ramstack.FileSystem.Azure/AzureDirectory.cs index 334382e..db9d180 100644 --- a/src/Ramstack.FileSystem.Azure/AzureDirectory.cs +++ b/src/Ramstack.FileSystem.Azure/AzureDirectory.cs @@ -5,6 +5,8 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Ramstack.Globbing; + namespace Ramstack.FileSystem.Azure; /// @@ -141,14 +143,137 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs yield return new AzureDirectory(_fs, VirtualPath.Normalize(item.Prefix)); } + /// + protected override async IAsyncEnumerable GetFileNodesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all blobs in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var prefix = GetPrefix(FullName); + var directories = new HashSet { FullName }; + + await foreach (var page in _fs.AzureClient + .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .AsPages() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + foreach (var blob in page.Values) + { + var directoryPath = VirtualPath.GetDirectoryName( + VirtualPath.Join("/", blob.Name)); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new AzureDirectory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + + if (IsMatched(blob.Name.AsSpan(prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(blob); + } + } + } + + /// + protected override async IAsyncEnumerable GetFilesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all blobs in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var prefix = GetPrefix(FullName); + + await foreach (var page in _fs.AzureClient + .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .AsPages() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + foreach (var blob in page.Values) + if (IsMatched(blob.Name.AsSpan(prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(blob); + } + } + + /// + protected override async IAsyncEnumerable GetDirectoriesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all blobs in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var prefix = GetPrefix(FullName); + var directories = new HashSet { FullName }; + + await foreach (var page in _fs.AzureClient + .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .AsPages() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + foreach (var blob in page.Values) + { + var directoryPath = VirtualPath.GetDirectoryName( + VirtualPath.Join("/", blob.Name)); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new AzureDirectory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + } + } + } + /// - /// Creates a instance based on the specified blob item. + /// Creates a instance based on the specified blob item. /// /// The representing the file. /// /// A new instance representing the file. /// - private VirtualFile CreateVirtualFile(BlobItem blob) + private AzureFile CreateVirtualFile(BlobItem blob) { var info = blob.Properties; var properties = VirtualNodeProperties.CreateFileProperties( @@ -162,12 +287,48 @@ private VirtualFile CreateVirtualFile(BlobItem blob) } /// - /// Returns the blob prefix for the specified directory path. + /// Determines whether the specified path matches any of the inclusion patterns and none of the exclusion patterns. + /// + /// The path to match. + /// The inclusion patterns to match against the path. + /// An optional array of exclusion patterns. If the path matches any of these, + /// the method returns . + /// + /// if the path matches at least one inclusion pattern and no exclusion patterns; + /// otherwise, . + /// + private static bool IsMatched(scoped ReadOnlySpan path, string[] patterns, string[]? excludes) + { + if (excludes is not null) + foreach (var pattern in excludes) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return false; + + foreach (var pattern in patterns) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return true; + + return false; + } + + /// + /// Returns a cloud storage compatible prefix for the specified directory path. /// - /// The directory path for which to get the prefix. + /// The directory path. /// - /// The blob prefix associated with the directory. + /// A string formatted for cloud storage prefix filtering: + /// + /// Empty string for the root directory ("/") + /// Path without leading slash but with trailing slash for subdirectories + /// /// - private static string GetPrefix(string directoryPath) => - directoryPath == "/" ? "" : $"{directoryPath[1..]}/"; + /// + /// + /// GetPrefix("/") // returns "" + /// GetPrefix("/folder") // returns "folder/" + /// GetPrefix("/sub/folder") // returns "sub/folder/" + /// + /// + private static string GetPrefix(string path) => + path == "/" ? "" : $"{path[1..]}/"; } diff --git a/src/Ramstack.FileSystem.Globbing/GlobbingDirectory.cs b/src/Ramstack.FileSystem.Globbing/GlobbingDirectory.cs index 2022fa4..8598b85 100644 --- a/src/Ramstack.FileSystem.Globbing/GlobbingDirectory.cs +++ b/src/Ramstack.FileSystem.Globbing/GlobbingDirectory.cs @@ -60,17 +60,15 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En { await foreach (var node in _directory.GetFileNodesAsync(cancellationToken).ConfigureAwait(false)) { - if (!_fs.IsExcluded(node.FullName)) + if (node is VirtualFile file) { - if (node is VirtualFile file) - { - if (_fs.IsIncluded(node.FullName)) - yield return new GlobbingFile(_fs, file, true); - } - else - { - yield return new GlobbingDirectory(_fs, (VirtualDirectory)node, true); - } + if (_fs.IsFileIncluded(file.FullName)) + yield return new GlobbingFile(_fs, file, included: true); + } + else + { + if (_fs.IsDirectoryIncluded(node.FullName)) + yield return new GlobbingDirectory(_fs, (VirtualDirectory)node, included: true); } } } @@ -84,7 +82,7 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer await foreach (var file in _directory.GetFilesAsync(cancellationToken).ConfigureAwait(false)) { if (_fs.IsFileIncluded(file.FullName)) - yield return new GlobbingFile(_fs, file, true); + yield return new GlobbingFile(_fs, file, included: true); } } } @@ -97,7 +95,7 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs await foreach (var directory in _directory.GetDirectoriesAsync(cancellationToken).ConfigureAwait(false)) { if (_fs.IsDirectoryIncluded(directory.FullName)) - yield return new GlobbingDirectory(_fs, directory, true); + yield return new GlobbingDirectory(_fs, directory, included: true); } } } diff --git a/src/Ramstack.FileSystem.Globbing/GlobbingFileSystem.cs b/src/Ramstack.FileSystem.Globbing/GlobbingFileSystem.cs index 7fb42bf..18a793b 100644 --- a/src/Ramstack.FileSystem.Globbing/GlobbingFileSystem.cs +++ b/src/Ramstack.FileSystem.Globbing/GlobbingFileSystem.cs @@ -1,4 +1,4 @@ -using Ramstack.Globbing; +using Ramstack.FileSystem.Globbing.Internal; namespace Ramstack.FileSystem.Globbing; @@ -92,7 +92,7 @@ public void Dispose() => /// otherwise, . /// internal bool IsFileIncluded(string path) => - !IsExcluded(path) && IsIncluded(path); + !PathHelper.IsMatch(path, _excludes) && PathHelper.IsMatch(path, _patterns); /// /// Determines if a directory is included based on the specified exclusions. @@ -103,39 +103,5 @@ internal bool IsFileIncluded(string path) => /// otherwise, . /// internal bool IsDirectoryIncluded(string path) => - !IsExcluded(path); - - /// - /// Checks if a path matches any of the include patterns. - /// - /// The path to check. - /// - /// if the path matches an include pattern; - /// otherwise, . - /// - internal bool IsIncluded(string path) - { - foreach (var pattern in _patterns) - if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) - return true; - - return false; - } - - /// - /// Checks if a path matches any of the exclude patterns. - /// - /// The path to check. - /// - /// if the path matches an exclude pattern; - /// otherwise, . - /// - internal bool IsExcluded(string path) - { - foreach (var pattern in _excludes) - if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) - return true; - - return false; - } + path == "/" || !PathHelper.IsMatch(path, _excludes) && PathHelper.IsPartialMatch(path, _patterns); } diff --git a/src/Ramstack.FileSystem.Globbing/Internal/PathHelper.cs b/src/Ramstack.FileSystem.Globbing/Internal/PathHelper.cs new file mode 100644 index 0000000..b2fd46c --- /dev/null +++ b/src/Ramstack.FileSystem.Globbing/Internal/PathHelper.cs @@ -0,0 +1,340 @@ +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +using Ramstack.Globbing; + +namespace Ramstack.FileSystem.Globbing.Internal; + +/// +/// Provides helper methods for path manipulations. +/// +internal static class PathHelper +{ + /// + /// Determines whether the specified path matches any of the specified patterns. + /// + /// The path to match for a match. + /// An array of patterns to match against the path. + /// + /// if the path matches any of the patterns; + /// otherwise, . + /// + public static bool IsMatch(scoped ReadOnlySpan path, string[] patterns) + { + foreach (var pattern in patterns) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return true; + + return false; + } + + /// + /// Determines whether the specified path partially matches any of the specified patterns. + /// + /// The path to be partially matched. + /// An array of patterns to match against the path. + /// + /// if the path partially matches any of the patterns; + /// otherwise, . + /// + public static bool IsPartialMatch(scoped ReadOnlySpan path, string[] patterns) + { + Debug.Assert(path is not "/"); + + var count = CountPathSegments(path); + + foreach (var pattern in patterns) + if (Matcher.IsMatch(path, GetPartialPattern(pattern, count), MatchFlags.Unix)) + return true; + + return false; + } + + /// + /// Counts the number of segments in the specified path. + /// + /// The path to count segments for. + /// + /// The number of segments in the path. + /// + public static int CountPathSegments(scoped ReadOnlySpan path) + { + var count = 0; + var iterator = new PathSegmentIterator(); + ref var s = ref Unsafe.AsRef(in MemoryMarshal.GetReference(path)); + var length = path.Length; + + while (true) + { + var r = iterator.GetNext(ref s, length); + + if (r.start != r.final) + count++; + + if (r.final == length) + break; + } + + if (count == 0) + count = 1; + + return count; + } + + /// + /// Returns a partial pattern from the specified pattern string based on the specified depth. + /// + /// The pattern string to extract from. + /// The depth level to extract the partial pattern up to. + /// + /// A representing the partial pattern. + /// + public static ReadOnlySpan GetPartialPattern(string pattern, int depth) + { + Debug.Assert(depth >= 1); + + var iterator = new PathSegmentIterator(); + ref var s = ref Unsafe.AsRef(in pattern.GetPinnableReference()); + var length = pattern.Length; + + while (true) + { + var r = iterator.GetNext(ref s, length); + if (r.start != r.final) + depth--; + + if (depth < 1 + || r.final == length + || IsGlobStar(ref s, r.start, r.final)) + return MemoryMarshal.CreateReadOnlySpan(ref s, r.final); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool IsGlobStar(ref char s, int index, int final) => + index + 2 == final && Unsafe.ReadUnaligned( + ref Unsafe.As( + ref Unsafe.Add(ref s, (nint)(uint)index))) == ('*' << 16 | '*'); + } + + #region Vector helper methods + + /// + /// Loads a 256-bit vector from the specified source. + /// + /// The source from which the vector will be loaded. + /// The offset from the from which the vector will be loaded. + /// + /// The loaded 256-bit vector. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 LoadVector256(ref char source, nint offset) => + Unsafe.ReadUnaligned>( + ref Unsafe.As(ref Unsafe.Add(ref source, offset))); + + /// + /// Loads a 128-bit vector from the specified source. + /// + /// The source from which the vector will be loaded. + /// The offset from from which the vector will be loaded. + /// + /// The loaded 128-bit vector. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadVector128(ref char source, nint offset) => + Unsafe.ReadUnaligned>( + ref Unsafe.As( + ref Unsafe.Add(ref source, offset))); + + #endregion + + #region Inner type: PathSegmentIterator + + /// + /// Provides functionality to iterate over segments of a path. + /// + private struct PathSegmentIterator + { + private int _last; + private nint _position; + private uint _mask; + + /// + /// Initializes a new instance of the structure. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PathSegmentIterator() => + _last = -1; + + /// + /// Retrieves the next segment of the path. + /// + /// A reference to the starting character of the path. + /// The total number of characters in the input path starting from . + /// + /// A tuple containing the start and end indices of the next path segment. + /// start indicates the beginning of the segment, and final satisfies + /// the condition that final - start equals the length of the segment. + /// The end of the iteration is indicated by final being equal to the length of the path. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (int start, int final) GetNext(ref char source, int length) + { + var start = _last + 1; + + while ((int)_position < length) + { + if ((Avx2.IsSupported || Sse2.IsSupported || AdvSimd.Arm64.IsSupported) && _mask != 0) + { + var offset = BitOperations.TrailingZeroCount(_mask); + if (AdvSimd.IsSupported) + { + // + // On ARM, ExtractMostSignificantBits returns a mask where each bit + // represents one vector element (1 bit per ushort), so offset + // directly corresponds to the element index + // + _last = (int)(_position + (nint)(uint)offset); + + // + // Clear the bits for the current separator + // + _mask &= ~(1u << offset); + } + else + { + // + // On x86, MoveMask (and ExtractMostSignificantBits on byte-based vectors) + // returns a mask where each bit represents one byte (2 bits per ushort), + // so we need to divide offset by 2 to get the actual element index + // + _last = (int)(_position + (nint)((uint)offset >> 1)); + + // + // Clear the bits for the current separator + // + _mask &= ~(0b_11u << offset); + } + + // + // Advance position to the next chunk when no separators remain in the mask + // + if (_mask == 0) + { + // + // https://github.com/dotnet/runtime/issues/117416 + // + // Precompute the stride size instead of calculating it inline + // to avoid stack spilling. For some unknown reason, the JIT + // fails to optimize properly when this is written inline, like so: + // _position += Avx2.IsSupported + // ? Vector256.Count + // : Vector128.Count; + // + + var stride = Avx2.IsSupported + ? Vector256.Count + : Vector128.Count; + + _position += stride; + } + + return (start, _last); + } + + if (Avx2.IsSupported && (int)_position + Vector256.Count <= length) + { + var chunk = LoadVector256(ref source, _position); + var slash = Vector256.Create('/'); + var comparison = Avx2.CompareEqual(chunk, slash); + + // + // Store the comparison bitmask and reuse it across iterations + // as long as it contains non-zero bits. + // This avoids reloading SIMD registers and repeating comparisons + // on the same chunk of data. + // + _mask = (uint)Avx2.MoveMask(comparison.AsByte()); + + // + // Advance position to the next chunk when no separators found + // + if (_mask == 0) + _position += Vector256.Count; + } + else if (Sse2.IsSupported && !Avx2.IsSupported && (int)_position + Vector128.Count <= length) + { + var chunk = LoadVector128(ref source, _position); + var slash = Vector128.Create('/'); + var comparison = Sse2.CompareEqual(chunk, slash); + + // + // Store the comparison bitmask and reuse it across iterations + // as long as it contains non-zero bits. + // This avoids reloading SIMD registers and repeating comparisons + // on the same chunk of data. + // + _mask = (uint)Sse2.MoveMask(comparison.AsByte()); + + // + // Advance position to the next chunk when no separators found + // + if (_mask == 0) + _position += Vector128.Count; + } + else if (AdvSimd.Arm64.IsSupported && (int)_position + Vector128.Count <= length) + { + var chunk = LoadVector128(ref source, _position); + var slash = Vector128.Create('/'); + var comparison = AdvSimd.CompareEqual(chunk, slash); + + // + // Store the comparison bitmask and reuse it across iterations + // as long as it contains non-zero bits. + // This avoids reloading SIMD registers and repeating comparisons + // on the same chunk of data. + // + _mask = ExtractMostSignificantBits(comparison); + + // + // Advance position to the next chunk when no separators found + // + if (_mask == 0) + _position += Vector128.Count; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static uint ExtractMostSignificantBits(Vector128 v) + { + var sum = AdvSimd.Arm64.AddAcross( + AdvSimd.ShiftLogical( + AdvSimd.And(v, Vector128.Create((ushort)0x8000)), + Vector128.Create(-15, -14, -13, -12, -11, -10, -9, -8))); + return sum.ToScalar(); + } + } + else + { + for (; (int)_position < length; _position++) + { + var ch = Unsafe.Add(ref source, _position); + if (ch == '/') + { + _last = (int)_position; + _position++; + + return (start, _last); + } + } + } + } + + return (start, length); + } + } + + #endregion +} diff --git a/src/Ramstack.FileSystem.Google/GcsDirectory.cs b/src/Ramstack.FileSystem.Google/GcsDirectory.cs index 348d3f8..80e8866 100644 --- a/src/Ramstack.FileSystem.Google/GcsDirectory.cs +++ b/src/Ramstack.FileSystem.Google/GcsDirectory.cs @@ -4,6 +4,10 @@ using Google; using Google.Cloud.Storage.V1; +using Ramstack.Globbing; + +using GcsObject = Google.Apis.Storage.v1.Data.Object; + namespace Ramstack.FileSystem.Google; /// @@ -13,7 +17,6 @@ namespace Ramstack.FileSystem.Google; internal sealed class GcsDirectory : VirtualDirectory { private readonly GoogleFileSystem _fs; - private readonly string _prefix; /// public override IVirtualFileSystem FileSystem => _fs; @@ -23,11 +26,8 @@ internal sealed class GcsDirectory : VirtualDirectory /// /// The file system associated with this directory. /// The path to the directory within the Google Cloud Storage bucket. - public GcsDirectory(GoogleFileSystem fileSystem, string path) : base(path) - { + public GcsDirectory(GoogleFileSystem fileSystem, string path) : base(path) => _fs = fileSystem; - _prefix = path == "/" ? "" : $"{path[1..]}/"; - } /// protected override ValueTask GetPropertiesCoreAsync(CancellationToken cancellationToken) => @@ -44,7 +44,7 @@ protected override ValueTask CreateCoreAsync(CancellationToken cancellationToken /// protected override async ValueTask DeleteCoreAsync(CancellationToken cancellationToken) { - await foreach (var obj in _fs.StorageClient.ListObjectsAsync(_fs.BucketName, _prefix).WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (var obj in _fs.StorageClient.ListObjectsAsync(_fs.BucketName, GetPrefix(FullName)).WithCancellation(cancellationToken).ConfigureAwait(false)) { try { @@ -69,7 +69,7 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En }; var responses = _fs.StorageClient - .ListObjectsAsync(_fs.BucketName, _prefix, options) + .ListObjectsAsync(_fs.BucketName, GetPrefix(FullName), options) .AsRawResponses(); await foreach (var page in responses.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -82,14 +82,8 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En continue; foreach (var obj in page.Items) - { - var properties = VirtualNodeProperties.CreateFileProperties( - obj.TimeCreatedDateTimeOffset.GetValueOrDefault(), - DateTimeOffset.MinValue, - obj.UpdatedDateTimeOffset.GetValueOrDefault(), - (long)(obj.Size ?? 0)); - yield return new GcsFile(_fs, VirtualPath.Normalize(obj.Name), properties); - } + if (!obj.Name.EndsWith('/')) + yield return CreateVirtualFile(obj); } } @@ -103,18 +97,11 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer IncludeTrailingDelimiter = true }; - var response = _fs.StorageClient.ListObjectsAsync(_fs.BucketName, _prefix, options); + var response = _fs.StorageClient.ListObjectsAsync(_fs.BucketName, GetPrefix(FullName), options); await foreach (var obj in response.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - var properties = VirtualNodeProperties.CreateFileProperties( - obj.TimeCreatedDateTimeOffset.GetValueOrDefault(), - DateTimeOffset.MinValue, - obj.UpdatedDateTimeOffset.GetValueOrDefault(), - (long)(obj.Size ?? 0)); - - yield return new GcsFile(_fs, VirtualPath.Normalize(obj.Name), properties); - } + if (!obj.Name.EndsWith('/')) + yield return CreateVirtualFile(obj); } /// @@ -128,7 +115,7 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs }; var response = _fs.StorageClient - .ListObjectsAsync(_fs.BucketName, _prefix, options) + .ListObjectsAsync(_fs.BucketName, GetPrefix(FullName), options) .AsRawResponses(); await foreach (var page in response.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -140,4 +127,185 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs yield return new GcsDirectory(_fs, VirtualPath.Normalize(prefix)); } } + + /// + protected override async IAsyncEnumerable GetFileNodesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var prefix = GetPrefix(FullName); + var directories = new HashSet { FullName }; + var response = _fs.StorageClient.ListObjectsAsync(_fs.BucketName, prefix); + + await foreach (var obj in response.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + var path = VirtualPath.Normalize(obj.Name); + var directoryPath = obj.Name.EndsWith('/') ? path : VirtualPath.GetDirectoryName(path); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new GcsDirectory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + + if (!obj.Name.EndsWith('/')) + if (IsMatched(obj.Name.AsSpan(prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj, path); + } + } + + /// + protected override async IAsyncEnumerable GetFilesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var prefix = GetPrefix(FullName); + + // + // Google Cloud Storage supports native glob matching in ListObjects, + // but we can't use this feature due to some limitation: + // GCS's MatchGlob doesn't support brace expansion in the middle of paths. + // For example, patterns like "project/{images,styles}/*.{svg,css}" won't work. + // + + var response = _fs.StorageClient.ListObjectsAsync(_fs.BucketName, prefix); + + await foreach (var obj in response.WithCancellation(cancellationToken).ConfigureAwait(false)) + if (!obj.Name.EndsWith('/')) + if (IsMatched(obj.Name.AsSpan(prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj); + } + + /// + protected override async IAsyncEnumerable GetDirectoriesCoreAsync(string[] patterns, string[]? excludes, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // + // Cloud storage optimization strategy: + // 1. List all objects in one batch using prefix filtering (matches our virtual directory). + // 2. Perform pattern matching and exclusion filters locally. + // + // Benefits: + // - Single API call instead of per-directory requests + // - Reduced network latency, especially for deep directory structures + // - More efficient than recursive directory scanning + // + + var response = _fs.StorageClient.ListObjectsAsync(_fs.BucketName, GetPrefix(FullName)); + var directories = new HashSet { FullName }; + + await foreach (var obj in response.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + var path = VirtualPath.Normalize(obj.Name); + var directoryPath = obj.Name.EndsWith('/') ? path : VirtualPath.GetDirectoryName(path); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new GcsDirectory(_fs, VirtualPath.Normalize(directoryPath)); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + } + } + + /// + /// Creates a instance based on the specified object item. + /// + /// The representing the file. + /// The normalized name of the object. + /// + /// A new instance representing the file. + /// + private GcsFile CreateVirtualFile(GcsObject obj, string? normalizedPath = null) + { + var properties = VirtualNodeProperties.CreateFileProperties( + obj.TimeCreatedDateTimeOffset.GetValueOrDefault(), + DateTimeOffset.MinValue, + obj.UpdatedDateTimeOffset.GetValueOrDefault(), + (long)(obj.Size ?? 0)); + + var path = normalizedPath ?? VirtualPath.Normalize(obj.Name); + return new GcsFile(_fs, path, properties); + } + + /// + /// Determines whether the specified path matches any of the inclusion patterns and none of the exclusion patterns. + /// + /// The path to match. + /// The inclusion patterns to match against the path. + /// An optional array of exclusion patterns. If the path matches any of these, + /// the method returns . + /// + /// if the path matches at least one inclusion pattern and no exclusion patterns; + /// otherwise, . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsMatched(scoped ReadOnlySpan path, string[] patterns, string[]? excludes) + { + if (excludes is not null) + foreach (var pattern in excludes) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return false; + + foreach (var pattern in patterns) + if (Matcher.IsMatch(path, pattern, MatchFlags.Unix)) + return true; + + return false; + } + + /// + /// Returns a cloud storage compatible prefix for the specified directory path. + /// + /// The directory path. + /// + /// A string formatted for cloud storage prefix filtering: + /// + /// Empty string for the root directory ("/") + /// Path without leading slash but with trailing slash for subdirectories + /// + /// + /// + /// + /// GetPrefix("/") // returns "" + /// GetPrefix("/folder") // returns "folder/" + /// GetPrefix("/sub/folder") // returns "sub/folder/" + /// + /// + private static string GetPrefix(string path) => + path == "/" ? "" : $"{path[1..]}/"; } diff --git a/src/Ramstack.FileSystem.Prefixed/PrefixedFileSystem.cs b/src/Ramstack.FileSystem.Prefixed/PrefixedFileSystem.cs index 291073a..d720901 100644 --- a/src/Ramstack.FileSystem.Prefixed/PrefixedFileSystem.cs +++ b/src/Ramstack.FileSystem.Prefixed/PrefixedFileSystem.cs @@ -139,7 +139,7 @@ protected override ValueTask CreateCoreAsync(CancellationToken cancellationToken /// protected override ValueTask DeleteCoreAsync(CancellationToken cancellationToken) => - _fs.GetDirectory(_fs._prefix).DeleteAsync(cancellationToken); + throw new UnauthorizedAccessException($"Access to the path '{FullName}' is denied."); /// protected override IAsyncEnumerable GetFileNodesCoreAsync(CancellationToken cancellationToken) => diff --git a/tests/Ramstack.FileSystem.Abstractions.Tests/VirtualPathTests.cs b/tests/Ramstack.FileSystem.Abstractions.Tests/VirtualPathTests.cs index d9a751b..b02ea29 100644 --- a/tests/Ramstack.FileSystem.Abstractions.Tests/VirtualPathTests.cs +++ b/tests/Ramstack.FileSystem.Abstractions.Tests/VirtualPathTests.cs @@ -52,6 +52,7 @@ public void GetFileName(string path, string expected) [TestCase("", "")] [TestCase("/", "")] [TestCase("/dir", "/")] + [TestCase("dir", "")] [TestCase("/dir/file", "/dir")] [TestCase("/dir/dir/", "/dir/dir")] [TestCase("dir/dir", "dir")] diff --git a/tests/Ramstack.FileSystem.Amazon.Tests/WritableAmazonFileSystemTests.cs b/tests/Ramstack.FileSystem.Amazon.Tests/WritableAmazonFileSystemTests.cs index a6cbeca..f009143 100644 --- a/tests/Ramstack.FileSystem.Amazon.Tests/WritableAmazonFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Amazon.Tests/WritableAmazonFileSystemTests.cs @@ -72,10 +72,9 @@ public async Task File_OpenWrite_InternalBufferWriteError_DoesNotCreateFile() { await stream.WriteAsync(new ReadOnlyMemory(new byte[1024])); } - catch (Exception exception) + catch { - Console.WriteLine("Exception expected!"); - Console.WriteLine(exception); + // Ignore } } @@ -160,6 +159,9 @@ public async Task File_CopyTo_File_DifferentFileSystems() Assert.That( await reader.ReadToEndAsync(), Is.EqualTo(content)); + + await source.DeleteAsync(); + await destination.DeleteAsync(); } [Test] @@ -188,6 +190,9 @@ public async Task File_CopyTo_File_DifferentStorages() Assert.That( await reader.ReadToEndAsync(), Is.EqualTo(content)); + + await source.DeleteAsync(); + await destination.DeleteAsync(); } @@ -212,8 +217,8 @@ await fs.GetFilesAsync("/temp").CountAsync(), await fs.DeleteDirectoryAsync("/temp"); Assert.That( - await fs.GetFilesAsync("/temp").CountAsync(), - Is.EqualTo(0)); + await fs.GetFilesAsync("/temp").AnyAsync(), + Is.False); } protected override AmazonS3FileSystem GetFileSystem() => diff --git a/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs b/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs index 3b20174..8a81080 100644 --- a/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs @@ -119,8 +119,8 @@ await fs.GetFilesAsync("/temp").CountAsync(), await fs.DeleteDirectoryAsync("/temp"); Assert.That( - await fs.GetFilesAsync("/temp").CountAsync(), - Is.EqualTo(0)); + await fs.GetFilesAsync("/temp").AnyAsync(), + Is.False); } protected override AzureFileSystem GetFileSystem() => diff --git a/tests/Ramstack.FileSystem.Composite.Tests/CompositeFileSystemTests.cs b/tests/Ramstack.FileSystem.Composite.Tests/CompositeFileSystemTests.cs index 312f365..17fc98e 100644 --- a/tests/Ramstack.FileSystem.Composite.Tests/CompositeFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Composite.Tests/CompositeFileSystemTests.cs @@ -8,23 +8,34 @@ namespace Ramstack.FileSystem.Composite; [TestFixture] public class CompositeFileSystemTests : VirtualFileSystemSpecificationTests { - private readonly TempFileStorage _storage = new TempFileStorage(); + private readonly TempFileStorage _storage1; + private readonly TempFileStorage _storage2; + private readonly TempFileStorage _storage3; private readonly CompositeFileSystem _fs; public CompositeFileSystemTests() { - var root = Path.Join(_storage.Root, "project"); + _storage1 = new TempFileStorage(); + _storage2 = new TempFileStorage(_storage1); + _storage3 = new TempFileStorage(_storage1); - foreach (var file in Directory.GetFiles(root)) + foreach (var directory in Directory.GetDirectories(Path.Join(_storage2.Root, "project"))) + Directory.Delete(directory, recursive: true); + + foreach (var file in Directory.GetFiles(Path.Join(_storage3.Root, "project"))) File.Delete(file); - var list = new List(); - foreach (var directory in Directory.GetDirectories(root)) + var list = new List { - var fileName = Path.GetFileName(directory); - Console.WriteLine(fileName); + new PhysicalFileSystem(_storage2.Root) + }; - var fs = new PrefixedFileSystem(fileName, new PhysicalFileSystem(directory)); + foreach (var directory in Directory.GetDirectories(Path.Join(_storage3.Root, "project"))) + { + var fs = new PrefixedFileSystem( + $"/project/{Path.GetFileName(directory)}", + new PhysicalFileSystem(directory) + ); list.Add(fs); } @@ -35,11 +46,14 @@ public CompositeFileSystemTests() public void Cleanup() { _fs.Dispose(); + _storage1.Dispose(); + _storage2.Dispose(); + _storage3.Dispose(); } protected override IVirtualFileSystem GetFileSystem() => _fs; protected override DirectoryInfo GetDirectoryInfo() => - new DirectoryInfo(Path.Join(_storage.Root, "project")); + new DirectoryInfo(_storage1.Root); } diff --git a/tests/Ramstack.FileSystem.Globbing.Tests/GlobingFileSystemTests.cs b/tests/Ramstack.FileSystem.Globbing.Tests/GlobingFileSystemTests.cs index b12855d..0fed676 100644 --- a/tests/Ramstack.FileSystem.Globbing.Tests/GlobingFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Globbing.Tests/GlobingFileSystemTests.cs @@ -7,33 +7,84 @@ namespace Ramstack.FileSystem.Globbing; [TestFixture] public class GlobingFileSystemTests : VirtualFileSystemSpecificationTests { - private readonly TempFileStorage _storage = new TempFileStorage(); + private readonly TempFileStorage _storage1; + private readonly TempFileStorage _storage2; + + public GlobingFileSystemTests() + { + _storage1 = new TempFileStorage(); + _storage2 = new TempFileStorage(_storage1); + } [OneTimeSetUp] public void Setup() { - var path = Path.Join(_storage.Root, "project"); - var directory = new DirectoryInfo(path); + var directory = new DirectoryInfo(Path.Join(_storage1.Root, "project")); foreach (var di in directory.GetDirectories("*", SearchOption.TopDirectoryOnly)) - if (di.Name != "docs") + if (di.Name != "assets") di.Delete(recursive: true); foreach (var fi in directory.GetFiles("*", SearchOption.TopDirectoryOnly)) - fi.Delete(); - - File.Delete(Path.Join(_storage.Root, "project/docs/troubleshooting/common_issues.txt")); + if (fi.Name != "README.md") + fi.Delete(); } [OneTimeTearDown] - public void Cleanup() => - _storage.Dispose(); + public void Cleanup() + { + _storage1.Dispose(); + _storage2.Dispose(); + } + + [Test] + public async Task Glob_MatchStructures() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs + .GetDirectoriesAsync("/", "**") + .OrderBy(f => f.FullName) + .Select(f => f.FullName) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project", + "/project/assets", + "/project/assets/fonts", + "/project/assets/images", + "/project/assets/images/backgrounds", + "/project/assets/styles" + ])); + + Assert.That( + await fs + .GetFilesAsync("/", "**") + .OrderBy(f => f.FullName) + .Select(f => f.FullName) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/fonts/Arial.ttf", + "/project/assets/fonts/Roboto.ttf", + "/project/assets/images/backgrounds/dark.jpeg", + "/project/assets/images/backgrounds/light.jpg", + "/project/assets/images/icon.svg", + "/project/assets/images/logo.png", + "/project/assets/styles/main.css", + "/project/assets/styles/print.css", + "/project/README.md" + ])); + } [Test] public async Task ExcludedDirectory_HasNoFileNodes() { - using var storage = new TempFileStorage(); - using var fs = new GlobbingFileSystem(new PhysicalFileSystem(storage.Root), "**", exclude: "/project/src/**"); + using var fs = new GlobbingFileSystem( + new PhysicalFileSystem(_storage2.Root), + pattern: "**", + exclude: "/project/src/**"); var directories = new[] { @@ -47,15 +98,13 @@ public async Task ExcludedDirectory_HasNoFileNodes() foreach (var path in directories) { - var directory = fs.GetDirectory(path); - Assert.That( - await directory.ExistsAsync(), + await fs.DirectoryExistsAsync(path), Is.False); Assert.That( - await directory.GetFileNodesAsync("**").CountAsync(), - Is.Zero); + await fs.GetFileNodesAsync(path, "**").AnyAsync(), + Is.False); } var files = new[] @@ -74,18 +123,17 @@ await directory.GetFileNodesAsync("**").CountAsync(), }; foreach (var path in files) - { - var file = fs.GetFile(path); - Assert.That(await file.ExistsAsync(), Is.False); - } + Assert.That(await fs.FileExistsAsync(path), Is.False); } protected override IVirtualFileSystem GetFileSystem() { - var fs = new PhysicalFileSystem(_storage.Root); - return new GlobbingFileSystem(fs, "project/docs/**", exclude: "**/*.txt"); + var fs = new PhysicalFileSystem(_storage2.Root); + return new GlobbingFileSystem(fs, + patterns: ["project/assets/**", "project/README.md", "project/global.json"], + excludes: ["project/*.json"]); } protected override DirectoryInfo GetDirectoryInfo() => - new(_storage.Root); + new(_storage1.Root); } diff --git a/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs b/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs index 36053fe..fc6c802 100644 --- a/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs @@ -16,9 +16,6 @@ public class ReadonlyGoogleFileSystemTests : VirtualFileSystemSpecificationTests [OneTimeSetUp] public async Task Setup() { - // if (Environment.GetEnvironmentVariable("STORAGE_EMULATOR_HOST") is null) - // Environment.SetEnvironmentVariable("STORAGE_EMULATOR_HOST", "http://localhost:4443/storage/v1/"); - using var fs = CreateFileSystem(isReadOnly: false); await fs.CreateBucketAsync("ramstack-project"); @@ -46,14 +43,6 @@ protected override DirectoryInfo GetDirectoryInfo() => private static GoogleFileSystem CreateFileSystem(bool isReadOnly) { - // var builder = new StorageClientBuilder - // { - // BaseUri = Environment.GetEnvironmentVariable("STORAGE_EMULATOR_HOST"), - // UnauthenticatedAccess = true - // }; - // - // var client = builder.Build(); - var client = StorageClient.Create( GoogleCredential.FromFile("credentials.json")); diff --git a/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs b/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs index f3cd860..11f08cf 100644 --- a/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs @@ -20,9 +20,6 @@ public class WritableGoogleFileSystemTests : VirtualFileSystemSpecificationTests [OneTimeSetUp] public async Task Setup() { - // if (Environment.GetEnvironmentVariable("STORAGE_EMULATOR_HOST") is null) - // Environment.SetEnvironmentVariable("STORAGE_EMULATOR_HOST", "http://localhost:4443/storage/v1/"); - using var fs = GetFileSystem(); await fs.CreateBucketAsync("ramstack-project"); @@ -204,12 +201,6 @@ private GoogleFileSystem CreateFileSystem(string bucket) { _buckets.Add(bucket); - // var client = new StorageClientBuilder - // { - // BaseUri = Environment.GetEnvironmentVariable("STORAGE_EMULATOR_HOST"), - // UnauthenticatedAccess = true - // }.Build(); - var client = StorageClient.Create( GoogleCredential.FromFile("credentials.json")); diff --git a/tests/Ramstack.FileSystem.Prefixed.Tests/PrefixedFileSystemTests.cs b/tests/Ramstack.FileSystem.Prefixed.Tests/PrefixedFileSystemTests.cs index 1cc72a7..3b240b7 100644 --- a/tests/Ramstack.FileSystem.Prefixed.Tests/PrefixedFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Prefixed.Tests/PrefixedFileSystemTests.cs @@ -5,14 +5,63 @@ namespace Ramstack.FileSystem.Prefixed; [TestFixture] -public class PrefixedFileSystemTests() : VirtualFileSystemSpecificationTests(Prefix) +public class PrefixedFileSystemTests : VirtualFileSystemSpecificationTests { - private const string Prefix = "solution/app"; + private readonly TempFileStorage _storage = new TempFileStorage(); - private readonly TempFileStorage _storage = new TempFileStorage(Prefix); + [Test] + public async Task File_Create_InsideArtificialDirectory_ThrowsException() + { + using var fs = GetFileSystem(); + + await Assert.ThatAsync( + async () => await fs.WriteAsync("/ea5fd219.txt", Stream.Null), + Throws.Exception); + + Assert.That( + await fs.FileExistsAsync("/ea5fd219.txt"), + Is.False); + } + + [Test] + public async Task Directory_Delete_ArtificialDirectory_ThrowsException() + { + using var storage = new TempFileStorage(); + using var fs = new PrefixedFileSystem("/bin/apps/myapp1", new PhysicalFileSystem(storage.Root));; + + await Assert.ThatAsync( + async () => await fs.DeleteDirectoryAsync("/"), + Throws.Exception); + + await Assert.ThatAsync( + async () => await fs.DeleteDirectoryAsync("/bin"), + Throws.Exception); + + await Assert.ThatAsync( + async () => await fs.DeleteDirectoryAsync("/bin/apps"), + Throws.Exception); + + Assert.That( + await fs.FileExistsAsync("/bin/apps/myapp1/project/README.md"), + Is.True); + + await Assert.ThatAsync( + async () => await fs.DeleteDirectoryAsync("/bin/apps/myapp1"), + Throws.Nothing); + + Assert.That( + await fs.FileExistsAsync("/bin/apps/myapp1/project/README.md"), + Is.False); + } + + [OneTimeTearDown] + public void Cleanup() => + _storage.Dispose(); protected override IVirtualFileSystem GetFileSystem() => - new PrefixedFileSystem(Prefix, new PhysicalFileSystem(_storage.PrefixedPath)); + new PrefixedFileSystem("/project", + new PhysicalFileSystem( + Path.Join(_storage.Root, "project"))); /// protected override DirectoryInfo GetDirectoryInfo() => diff --git a/tests/Ramstack.FileSystem.Specification.Tests/Utilities/TempFileStorage.cs b/tests/Ramstack.FileSystem.Specification.Tests/Utilities/TempFileStorage.cs index 05c8185..f6403ad 100644 --- a/tests/Ramstack.FileSystem.Specification.Tests/Utilities/TempFileStorage.cs +++ b/tests/Ramstack.FileSystem.Specification.Tests/Utilities/TempFileStorage.cs @@ -3,94 +3,83 @@ namespace Ramstack.FileSystem.Specification.Tests.Utilities; public sealed class TempFileStorage : IDisposable { public string Root { get; } = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - public string PrefixedPath { get; } - public TempFileStorage(string prefix = "") + public TempFileStorage() { var path = Root; - - if (!string.IsNullOrEmpty(prefix)) - path = Path.GetFullPath(Path.Join(Root, prefix)); - - PrefixedPath = path; - var list = new[] - { - "project/docs/user_manual.pdf", - "project/docs/api_reference.md", - "project/docs/troubleshooting/common_issues.txt", - "project/docs/troubleshooting/faq.docx", - - "project/docs_generated/html/index.html", - "project/docs_generated/html/api.html", - "project/docs_generated/pdf/documentation.pdf", - - "project/src/App/App.csproj", - "project/src/App/Program.cs", - "project/src/App/Utils.cs", - "project/src/Modules/Module1/Module1.cs", - "project/src/Modules/Module1/Module1.csproj", - "project/src/Modules/Module1/Submodule/Submodule1.cs", - "project/src/Modules/Module1/Submodule/Submodule2.cs", - "project/src/Modules/Module1/Submodule/Submodule.csproj", - "project/src/Modules/Module2/Module2.cs", - "project/src/Modules/Module2/Module2.csproj", - "project/src/App.sln", - - "project/tests/TestMain.cs", - "project/tests/TestUtils.cs", - "project/tests/Tests.csproj", - "project/tests/Fixtures/SampleData.json", - "project/tests/Fixtures/MockResponses.xml", - - "project/data/raw/dataset1.csv", - "project/data/raw/dataset2.csv", - "project/data/raw/dataset-[data]-{1}.csv", - "project/data/processed/cleaned_data.csv", - "project/data/processed/aggregated_results.json", - - "project/data/temp/temp_file1.tmp", - "project/data/temp/temp_file2.tmp", - "project/data/temp/ac/b2/34/2d/7e/temp_file2.tmp", - "project/data/temp/hidden-folder/temp_1.tmp", - "project/data/temp/hidden-folder/temp_2.tmp", - "project/data/temp/hidden/temp_hidden3.dat", - "project/data/temp/hidden/temp_hidden4.dat", - - "project/scripts/setup.p1", - "project/scripts/deploy.ps1", - "project/scripts/build.bat", - "project/scripts/build.sh", - - "project/logs/app.log", - "project/logs/error.log", - "project/logs/archive/2019/01/app_2019-01.log", - "project/logs/archive/2019/02/app_2019-02.log", - "project/logs/archive/2019/03/app_2019-03.log", - - "project/config/production.json", - "project/config/development.json", - "project/config/test.json", - - "project/assets/images/logo.png", - "project/assets/images/icon.svg", - "project/assets/images/backgrounds/light.jpg", - "project/assets/images/backgrounds/dark.jpeg", - - "project/assets/fonts/opensans.ttf", - "project/assets/fonts/roboto.ttf", - "project/assets/styles/main.css", - "project/assets/styles/print.css", - - "project/packages/Ramstack.Globbing.2.1.0/lib/net60/Ramstack.Globbing.dll", - "project/packages/Ramstack.Globbing.2.1.0/Ramstack.Globbing.2.1.0.nupkg", - - "project/.gitignore", - "project/.editorconfig", - "project/README.md", - "project/global.json", - "project/nuget.config", - } + { + "project/docs/user_manual.pdf", + "project/docs/api_reference.md", + "project/docs/troubleshooting/common_issues.txt", + "project/docs/troubleshooting/faq.docx", + + "project/docs_generated/html/index.html", + "project/docs_generated/html/api.html", + "project/docs_generated/pdf/documentation.pdf", + + "project/src/App/App.csproj", + "project/src/App/Program.cs", + "project/src/App/Utils.cs", + "project/src/Modules/Module1/Module1.cs", + "project/src/Modules/Module1/Module1.csproj", + "project/src/Modules/Module1/Submodule/Submodule1.cs", + "project/src/Modules/Module1/Submodule/Submodule2.cs", + "project/src/Modules/Module1/Submodule/Submodule.csproj", + "project/src/Modules/Module2/Module2.cs", + "project/src/Modules/Module2/Module2.csproj", + "project/src/App.sln", + + "project/tests/TestMain.cs", + "project/tests/TestUtils.cs", + "project/tests/Tests.csproj", + "project/tests/Fixtures/SampleData.json", + "project/tests/Fixtures/MockResponses.xml", + + "project/data/raw/dataset1.csv", + "project/data/raw/dataset2.csv", + "project/data/raw/dataset-[data]-{1}.csv", + "project/data/processed/cleaned_data.csv", + "project/data/processed/aggregated_results.json", + + "project/data/temp/temp_file1.tmp", + "project/data/temp/temp_file2.tmp", + "project/data/temp/ac/b2/34/2d/7e/temp_file2.tmp", + + "project/scripts/setup.p1", + "project/scripts/deploy.ps1", + "project/scripts/build.bat", + "project/scripts/build.sh", + + "project/logs/app.log", + "project/logs/error.log", + "project/logs/archive/2019/01/app_2019-01.log", + "project/logs/archive/2019/02/app_2019-02.log", + "project/logs/archive/2019/03/app_2019-03.log", + + "project/config/production.json", + "project/config/development.json", + "project/config/test.json", + + "project/assets/images/logo.png", + "project/assets/images/icon.svg", + "project/assets/images/backgrounds/light.jpg", + "project/assets/images/backgrounds/dark.jpeg", + + "project/assets/fonts/Arial.ttf", + "project/assets/fonts/Roboto.ttf", + "project/assets/styles/main.css", + "project/assets/styles/print.css", + + "project/packages/Ramstack.Globbing.2.1.0/lib/net60/Ramstack.Globbing.dll", + "project/packages/Ramstack.Globbing.2.1.0/Ramstack.Globbing.2.1.0.zip", + + "project/.gitignore", + "project/.editorconfig", + "project/README.md", + "project/global.json", + "project/nuget.config", + } .Select(p => Path.Combine(path, p)) .ToArray(); @@ -105,14 +94,34 @@ public TempFileStorage(string prefix = "") foreach (var f in list) File.WriteAllText(f, $"Automatically generated on {DateTime.Now:s}\n\nId:{Guid.NewGuid()}"); + } - var hiddenFolder = directories.First(p => p.Contains("hidden-folder")); - File.SetAttributes(hiddenFolder, FileAttributes.Hidden); + public TempFileStorage(TempFileStorage storage) + { + CopyDirectory(storage.Root, Root); - foreach (var hiddenFile in list.Where(p => p.Contains("temp_hidden"))) - File.SetAttributes(hiddenFile, FileAttributes.Hidden); + static void CopyDirectory(string sourcePath, string destinationPath) + { + Directory.CreateDirectory(destinationPath); + + foreach (var sourceFileName in Directory.GetFiles(sourcePath)) + { + var name = Path.GetFileName(sourceFileName); + var destFileName = Path.Join(destinationPath, name); + File.Copy(sourceFileName, destFileName); + } + + foreach (var directoryPath in Directory.GetDirectories(sourcePath)) + { + var destination = Path.Join(destinationPath, Path.GetFileName(directoryPath)); + CopyDirectory(directoryPath, destination); + } + } } - public void Dispose() => - Directory.Delete(Root, true); + public void Dispose() + { + if (Directory.Exists(Root)) + Directory.Delete(Root, true); + } } diff --git a/tests/Ramstack.FileSystem.Specification.Tests/VirtualFileSystemSpecificationTests.cs b/tests/Ramstack.FileSystem.Specification.Tests/VirtualFileSystemSpecificationTests.cs index 7aa8dac..182ec64 100644 --- a/tests/Ramstack.FileSystem.Specification.Tests/VirtualFileSystemSpecificationTests.cs +++ b/tests/Ramstack.FileSystem.Specification.Tests/VirtualFileSystemSpecificationTests.cs @@ -1,4 +1,4 @@ -using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; using System.Text; namespace Ramstack.FileSystem.Specification.Tests; @@ -6,13 +6,12 @@ namespace Ramstack.FileSystem.Specification.Tests; /// /// Represents a base class for specification tests of virtual file systems. /// -/// The safe path for modifications. Defaults to "/". /// /// This class defines common functionality and setup for tests that validate the behavior /// of virtual file systems. Derived classes should implement the abstract methods to provide specific /// details about the virtual file system being tested. /// -public abstract class VirtualFileSystemSpecificationTests(string safePath = "/") +public abstract class VirtualFileSystemSpecificationTests { [Test] [Order(-1003)] @@ -84,19 +83,8 @@ public async Task Exists_ReturnsTrue_For_ExistingFile() { using var fs = GetFileSystem(); - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); - - await foreach (var node in fs.GetFileNodesAsync("/", "**")) - { - VirtualNode byPath = node is VirtualFile - ? node.FileSystem.GetFile(node.FullName) - : node.FileSystem.GetDirectory(node.FullName); - - Assert.That(await node.ExistsAsync(), Is.True); - Assert.That(await byPath.ExistsAsync(), Is.True); - } + Assert.That(await fs.DirectoryExistsAsync("/project/assets"), Is.True); + Assert.That(await fs.FileExistsAsync("/project/README.md"), Is.True); } [Test] @@ -104,39 +92,26 @@ public async Task File_Exists_ReturnsFalse_For_NonExistingFile() { using var fs = GetFileSystem(); - var file = fs.GetFile($"{safePath}/{Guid.NewGuid()}"); - Assert.That(await file.ExistsAsync(), Is.False); + Assert.That(await fs.FileExistsAsync($"/{Guid.NewGuid()}"), Is.False); } [Test] public async Task File_OpenRead_ReturnsReadableStream() { using var fs = GetFileSystem(); + await using var stream = await fs.OpenReadAsync("/project/README.md"); - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); - - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - await using var stream = await file.OpenReadAsync(); - Assert.That(stream.CanRead, Is.True); - - stream.ReadByte(); - } + Assert.That(stream.CanRead, Is.True); + stream.ReadByte(); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void File_OpenRead_ThrowsException_For_NonExistingFile() + public async Task File_OpenRead_ThrowsException_For_NonExistingFile() { using var fs = GetFileSystem(); - var name = Guid.NewGuid().ToString(); - - Assert.That(() => fs.OpenReadAsync($"/{name}.txt"), Throws.Exception); - Assert.That(() => fs.OpenReadAsync($"{safePath}/{name}.txt"), Throws.Exception); - Assert.That(() => fs.OpenReadAsync($"{safePath}/{name}/{name}.txt"), Throws.Exception); + await Assert.ThatAsync(async () => await fs.OpenReadAsync("/6b01ba26.txt"), Throws.Exception); + await Assert.ThatAsync(async () => await fs.OpenReadAsync("/6b01ba26/6b01ba26.txt"), Throws.Exception); } [Test] @@ -147,25 +122,17 @@ public async Task File_OpenWrite_ReturnsWritableStream() if (fs.IsReadOnly) return; - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); + var text = $"Id:{Guid.NewGuid()}"; - await foreach (var file in fs.GetFilesAsync("/", "**")) + await using (var stream = await fs.OpenWriteAsync("/project/README.md")) { - var content = $"Id:{Guid.NewGuid()}"; - - await using (var stream = await file.OpenWriteAsync()) - { - Assert.That(stream.CanWrite, Is.True); - - await stream.WriteAsync(Encoding.UTF8.GetBytes(content)); - } - - Assert.That( - await file.ReadAllTextAsync(), - Is.EqualTo(content)); + Assert.That(stream.CanWrite, Is.True); + await stream.WriteAsync(Encoding.UTF8.GetBytes(text)); } + + Assert.That( + await fs.ReadAllTextAsync("/project/README.md"), + Is.EqualTo(text)); } [Test] @@ -176,23 +143,23 @@ public async Task File_OpenWrite_NewFile() if (fs.IsReadOnly) return; - var content = $"Automatically generated on {DateTime.Now:s}\n\nNew Id:{Guid.NewGuid()}"; - var name = $"{safePath}/{Guid.NewGuid()}"; + var path = "/project/6b01ba2608be"; + var text = $"Automatically generated on {DateTime.Now:s}\n\nNew Id:{Guid.NewGuid()}"; - await using (var stream = await fs.OpenWriteAsync(name)) + await using (var stream = await fs.OpenWriteAsync(path)) { Assert.That(stream.CanWrite, Is.True); - await stream.WriteAsync(Encoding.UTF8.GetBytes(content)); + await stream.WriteAsync(Encoding.UTF8.GetBytes(text)); } Assert.That( - await fs.ReadAllTextAsync(name), - Is.EqualTo(content)); + await fs.ReadAllTextAsync(path), + Is.EqualTo(text)); - await fs.DeleteFileAsync(name); + await fs.DeleteFileAsync(path); Assert.That( - await fs.GetFile(name).ExistsAsync(), + await fs.FileExistsAsync(path), Is.False); } @@ -204,24 +171,18 @@ public async Task File_Write_OverwritesContent_When_OverwriteIsTrue() if (fs.IsReadOnly) return; - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); + var file = fs.GetFile("/project/README.md"); + var text = $"Id:{Guid.NewGuid()}"; - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - var content = $"Id:{Guid.NewGuid()}"; - - var ms = new MemoryStream(); - ms.Write(Encoding.UTF8.GetBytes(content)); - ms.Position = 0; + var ms = new MemoryStream(); + ms.Write(Encoding.UTF8.GetBytes(text)); + ms.Position = 0; - await file.WriteAsync(ms, overwrite: true); + await file.WriteAsync(ms, overwrite: true); - Assert.That( - await file.ReadAllTextAsync(), - Is.EqualTo(content)); - } + Assert.That( + await file.ReadAllTextAsync(), + Is.EqualTo(text)); } [Test] @@ -232,22 +193,20 @@ public async Task File_Write_ThrowsException_For_ExistingFile_When_OverwriteIsFa if (fs.IsReadOnly) return; - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); + var file = fs.GetFile("/project/README.md"); + var text = await file.ReadAllTextAsync(); - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - var current = await file.ReadAllTextAsync(); + var ms = new MemoryStream(); + ms.Write(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())); + ms.Position = 0; - Assert.That( - () => file.WriteAsync(new MemoryStream(), overwrite: false), - Throws.Exception); + Assert.That( + () => file.WriteAsync(ms, overwrite: false), + Throws.Exception); - Assert.That( - await file.ReadAllTextAsync(), - Is.EqualTo(current)); - } + Assert.That( + await file.ReadAllTextAsync(), + Is.EqualTo(text)); } [Test] @@ -258,11 +217,11 @@ public async Task File_Write_NewFile() if (fs.IsReadOnly) return; - var content = $"Automatically generated on {DateTime.Now:s}\n\nNew Id:{Guid.NewGuid()}"; - var path = $"{safePath}/{Guid.NewGuid()}"; + var path = "/project/0fce49e0.txt"; + var text = $"Automatically generated on {DateTime.Now:s}\n\nNew Id:{Guid.NewGuid()}"; var ms = new MemoryStream(); - ms.Write(Encoding.UTF8.GetBytes(content)); + ms.Write(Encoding.UTF8.GetBytes(text)); ms.Position = 0; var file = fs.GetFile(path); @@ -271,7 +230,7 @@ public async Task File_Write_NewFile() await file.WriteAsync(ms); Assert.That(await file.ExistsAsync(), Is.True); - Assert.That(await file.ReadAllTextAsync(), Is.EqualTo(content)); + Assert.That(await file.ReadAllTextAsync(), Is.EqualTo(text)); await file.DeleteAsync(); @@ -286,12 +245,12 @@ public async Task File_Delete_For_ExistingFile() if (fs.IsReadOnly) return; - var path = $"{safePath}/{Guid.NewGuid()}"; + var path = "/project/793cd29d96c4.txt"; var file = fs.GetFile(path); Assert.That(await file.ExistsAsync(), Is.False); - await file.WriteAsync(new MemoryStream()); + await file.WriteAsync(Stream.Null); Assert.That(await file.ExistsAsync(), Is.True); @@ -308,11 +267,8 @@ public async Task File_Delete_For_NonExistingFile() if (fs.IsReadOnly) return; - var name = Guid.NewGuid().ToString(); - - await fs.DeleteFileAsync($"/{name}.txt"); - await fs.DeleteFileAsync($"{safePath}/{name}.txt"); - await fs.DeleteFileAsync($"{safePath}/{name}/{name}.txt"); + await fs.DeleteFileAsync("/1861cb25.txt"); + await fs.DeleteFileAsync("/1861cb25/1861cb25.txt"); } [Test] @@ -323,33 +279,21 @@ public async Task File_Readonly_OpenWrite_ThrowsException() if (!fs.IsReadOnly) return; - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); - - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - Assert.That( - async () => { await using var stream = await file.OpenWriteAsync(); }, - Throws.Exception); - } + await Assert.ThatAsync( + async () => { await using var stream = await fs.OpenWriteAsync("/project/README.md"); }, + Throws.Exception); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void File_Readonly_OpenWrite_ThrowsException_For_NewFile() + public async Task File_Readonly_OpenWrite_ThrowsException_For_NewFile() { using var fs = GetFileSystem(); if (!fs.IsReadOnly) return; - Assert.That( - () => fs.OpenWriteAsync($"/{Guid.NewGuid()}"), - Throws.Exception); - - Assert.That( - () => fs.OpenWriteAsync($"{safePath}/{Guid.NewGuid()}"), + await Assert.ThatAsync( + async () => await fs.OpenWriteAsync("/project/6d368a9d0e97.md"), Throws.Exception); } @@ -361,29 +305,22 @@ public async Task File_Readonly_Write_ThrowsException() if (!fs.IsReadOnly) return; - Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), - Is.True); - - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - Assert.That( - () => file.WriteAsync(new MemoryStream()), - Throws.Exception); - } + await Assert.ThatAsync( + async () => await fs.WriteAsync("/project/README.md", Stream.Null), + Throws.Exception); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void File_Readonly_Write_ThrowsException_For_NewFile() + public async Task File_Readonly_Write_ThrowsException_For_NewFile() { using var fs = GetFileSystem(); if (!fs.IsReadOnly) return; - Assert.That(() => fs.WriteAsync($"/{Guid.NewGuid()}", new MemoryStream()), Throws.Exception); - Assert.That(() => fs.WriteAsync($"{safePath}/{Guid.NewGuid()}", new MemoryStream()), Throws.Exception); + await Assert.ThatAsync( + async () => await fs.WriteAsync("/project/9286d04f.pdf", Stream.Null), + Throws.Exception); } [Test] @@ -394,25 +331,26 @@ public async Task File_Readonly_Delete_ThrowsException() if (!fs.IsReadOnly) return; + await Assert.ThatAsync( + async () => await fs.DeleteFileAsync("/project/README.md"), + Throws.Exception); + Assert.That( - await fs.GetFilesAsync("/", "**").AnyAsync(), + await fs.FileExistsAsync("/project/README.md"), Is.True); - - await foreach (var file in fs.GetFilesAsync("/", "**")) - Assert.That(() => file.DeleteAsync(), Throws.Exception); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void File_Readonly_Delete_ThrowsException_For_NonExistingFile() + public async Task File_Readonly_Delete_ThrowsException_For_NonExistingFile() { using var fs = GetFileSystem(); if (!fs.IsReadOnly) return; - Assert.That(() => fs.DeleteFileAsync($"/{Guid.NewGuid()}"), Throws.Exception); - Assert.That(() => fs.DeleteFileAsync($"{safePath}/{Guid.NewGuid()}"), Throws.Exception); + await Assert.ThatAsync( + async () => await fs.DeleteFileAsync("/project/c180408e8005.png"), + Throws.Exception); } [Test] @@ -422,20 +360,26 @@ public async Task File_CopyTo_Path() if (fs.IsReadOnly) return; - var file = await fs.GetFilesAsync("/", "**").FirstAsync(); - var destinationPath = file.FullName + ".copy"; + var sourcePath = "/project/README.md"; + var destinationPath = "/project/README - Copy.md"; - Assert.That(await fs.FileExistsAsync(destinationPath), Is.False); + Assert.That( + await fs.FileExistsAsync(destinationPath), + Is.False); - await file.CopyToAsync(destinationPath); - Assert.That(await fs.FileExistsAsync(destinationPath), Is.True); + await fs.CopyFileAsync(sourcePath, destinationPath); + Assert.That( + await fs.FileExistsAsync(destinationPath), + Is.True); Assert.That( - await fs.GetFile(destinationPath).ReadAllTextAsync(), - Is.EqualTo(await file.ReadAllTextAsync())); + await fs.ReadAllTextAsync(destinationPath), + Is.EqualTo(await fs.ReadAllTextAsync(sourcePath))); await fs.DeleteFileAsync(destinationPath); - Assert.That(await fs.FileExistsAsync(destinationPath), Is.False); + Assert.That( + await fs.FileExistsAsync(destinationPath), + Is.False); } [Test] @@ -445,16 +389,23 @@ public async Task File_CopyTo_Path_NoOverwrite_ThrowsException() if (fs.IsReadOnly) return; - var file = await fs.GetFilesAsync("/", "**").FirstAsync(); - var destinationPath = file.FullName + ".copy"; + var sourcePath = "/project/README.md"; + var destinationPath = "/project/README - Copy.md"; - await file.CopyToAsync(destinationPath); + await fs.CopyFileAsync(sourcePath, destinationPath); - Assert.That(await fs.FileExistsAsync(destinationPath), Is.True); - Assert.That(() => file.CopyToAsync(destinationPath), Throws.Exception); + Assert.That( + await fs.FileExistsAsync(destinationPath), + Is.True); + + await Assert.ThatAsync( + async () => await fs.CopyFileAsync(sourcePath, destinationPath), + Throws.Exception); await fs.DeleteFileAsync(destinationPath); - Assert.That(await fs.FileExistsAsync(destinationPath), Is.False); + Assert.That( + await fs.FileExistsAsync(destinationPath), + Is.False); } [Test] @@ -464,20 +415,27 @@ public async Task File_CopyTo_File_SameFileSystems() if (fs.IsReadOnly) return; - var file = await fs.GetFilesAsync("/", "**").FirstAsync(); - var destination = fs.GetFile(file.FullName + ".copy"); + var source = fs.GetFile("/project/README.md"); + var destination = fs.GetFile("/project/README - Copy.md"); + + Assert.That( + await destination.ExistsAsync(), + Is.False); - Assert.That(await destination.ExistsAsync(), Is.False); + await source.CopyToAsync(destination); - await file.CopyToAsync(destination); + Assert.That( + await destination.ExistsAsync(), + Is.True); - Assert.That(await destination.ExistsAsync(), Is.True); Assert.That( await destination.ReadAllTextAsync(), - Is.EqualTo(await file.ReadAllTextAsync())); + Is.EqualTo(await source.ReadAllTextAsync())); await destination.DeleteAsync(); - Assert.That(await destination.ExistsAsync(), Is.False); + Assert.That( + await destination.ExistsAsync(), + Is.False); } [Test] @@ -489,20 +447,27 @@ public async Task File_CopyTo_File_NotSameFileSystems() if (fs1.IsReadOnly) return; - var file = await fs1.GetFilesAsync("/", "**").FirstAsync(); - var destination = fs2.GetFile(file.FullName + ".copy"); + var source = fs1.GetFile("/project/README.md"); + var destination = fs2.GetFile("/project/README - Copy.md"); + + Assert.That( + await destination.ExistsAsync(), + Is.False); - Assert.That(await destination.ExistsAsync(), Is.False); + await source.CopyToAsync(destination); - await file.CopyToAsync(destination); + Assert.That( + await destination.ExistsAsync(), + Is.True); - Assert.That(await destination.ExistsAsync(), Is.True); Assert.That( await destination.ReadAllTextAsync(), - Is.EqualTo(await file.ReadAllTextAsync())); + Is.EqualTo(await source.ReadAllTextAsync())); await destination.DeleteAsync(); - Assert.That(await destination.ExistsAsync(), Is.False); + Assert.That( + await destination.ExistsAsync(), + Is.False); } [Test] @@ -512,51 +477,61 @@ public async Task File_CopyTo_File_NoOverwrite_ThrowsException() if (fs.IsReadOnly) return; - var file = await fs.GetFilesAsync("/", "**").FirstAsync(); - var destination = fs.GetFile(file.FullName + ".copy"); + var source = fs.GetFile("/project/README.md"); + var destination = fs.GetFile("/project/README - Copy.md"); + + Assert.That( + await destination.ExistsAsync(), + Is.False); - Assert.That(await destination.ExistsAsync(), Is.False); - await file.CopyToAsync(destination); + await source.CopyToAsync(destination); - Assert.That(await destination.ExistsAsync(), Is.True); - Assert.That(() => file.CopyToAsync(destination), Throws.Exception); + Assert.That( + await destination.ExistsAsync(), + Is.True); + + await Assert.ThatAsync( + async () => await source.CopyToAsync(destination), + Throws.Exception); await destination.DeleteAsync(); - Assert.That(await destination.ExistsAsync(), Is.False); + Assert.That( + await destination.ExistsAsync(), + Is.False); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] public async Task File_CopyTo_ThrowsException_When_CopyingToItself() { - using var fs1 = GetFileSystem(); - using var fs2 = GetFileSystem(); + using var fs = GetFileSystem(); - if (fs1.IsReadOnly) + if (fs.IsReadOnly) return; - var file = await fs1.GetFilesAsync("/", "**").FirstAsync(); - Assert.That(() => file.CopyToAsync(file.FullName), Throws.Exception); - Assert.That(() => file.CopyToAsync(file), Throws.Exception); - Assert.That(() => file.CopyToAsync(fs1.GetFile(file.FullName)), Throws.Exception); - Assert.That(() => file.CopyToAsync(fs2.GetFile(file.FullName)), Throws.Exception); + var file = fs.GetFile("/project/README.md"); + await Assert.ThatAsync(async () => await file.CopyToAsync(file.FullName), Throws.Exception); + await Assert.ThatAsync(async () => await file.CopyToAsync(file), Throws.Exception); + await Assert.ThatAsync(async () => await file.CopyToAsync(fs.GetFile(file.FullName)), Throws.Exception); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void File_CopyTo_ThrowException_For_NonExistingFile() + public async Task File_CopyTo_ThrowException_For_NonExistingFile() { using var fs = GetFileSystem(); if (fs.IsReadOnly) return; - Assert.That(() => fs.CopyFileAsync($"/{Guid.NewGuid()}", "/test.txt"), Throws.Exception); - Assert.That(() => fs.CopyFileAsync($"{safePath}/{Guid.NewGuid()}", $"/{safePath}/test.txt"), Throws.Exception); + await Assert.ThatAsync( + async () => await fs.CopyFileAsync("/project/800569b11aaa0438.txt", "/project/test-b11aaa.txt"), + Throws.Exception); + + Assert.That( + await fs.FileExistsAsync("/project/test-b11aaa.txt"), + Is.False); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] public async Task File_Readonly_CopyTo_ThrowException_For_NonExistingFile() { using var fs = GetFileSystem(); @@ -564,8 +539,13 @@ public async Task File_Readonly_CopyTo_ThrowException_For_NonExistingFile() if (!fs.IsReadOnly) return; - var file = await fs.GetFilesAsync("/", "**").FirstAsync(); - Assert.That(() => file.CopyToAsync(file.FullName + ".copy"), Throws.Exception); + await Assert.ThatAsync( + async () => await fs.CopyFileAsync("/project/README.md", "/project/README - Copy.md"), + Throws.Exception); + + Assert.That( + await fs.FileExistsAsync("/project/README - Copy.md"), + Is.False); } [Test] @@ -573,16 +553,20 @@ public async Task File_ReadAllBytes() { using var fs = GetFileSystem(); - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - var stream = await file.OpenReadAsync(); - using var reader = new BinaryReader(stream); + var file = fs.GetFile("/project/README.md"); + using var reader = new BinaryReader(await file.OpenReadAsync()); - var bytes = await fs.ReadAllBytesAsync(file.FullName); - var expected = reader.ReadBytes(4096); + var bytes = await fs.ReadAllBytesAsync(file.FullName); + var expected = reader.ReadBytes(4096); + var properties = await file.GetPropertiesAsync(); - Assert.That(bytes.SequenceEqual(expected), Is.True); - } + Assert.That( + bytes.SequenceEqual(expected), + Is.True); + + Assert.That( + bytes.Length, + Is.EqualTo(properties.Length)); } [Test] @@ -590,15 +574,13 @@ public async Task File_ReadAllText() { using var fs = GetFileSystem(); - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - var stream = await file.OpenReadAsync(); - using var reader = new StreamReader(stream); + var file = fs.GetFile("/project/README.md"); + var stream = await file.OpenReadAsync(); + using var reader = new StreamReader(stream); - Assert.That( - await fs.ReadAllTextAsync(file.FullName), - Is.EqualTo(await reader.ReadToEndAsync())); - } + Assert.That( + await fs.ReadAllTextAsync(file.FullName), + Is.EqualTo(await reader.ReadToEndAsync())); } [Test] @@ -606,19 +588,17 @@ public async Task File_ReadAllLines() { using var fs = GetFileSystem(); - await foreach (var file in fs.GetFilesAsync("/", "**")) - { - var stream = await file.OpenReadAsync(); - using var reader = new StreamReader(stream); + var file = fs.GetFile("/project/README.md"); + var stream = await file.OpenReadAsync(); + using var reader = new StreamReader(stream); - var lines = new List(); - while (await reader.ReadLineAsync() is {} line) - lines.Add(line); + var lines = new List(); + while (await reader.ReadLineAsync() is {} line) + lines.Add(line); - Assert.That( - await fs.ReadAllLinesAsync(file.FullName), - Is.EquivalentTo(lines)); - } + Assert.That( + await fs.ReadAllLinesAsync(file.FullName), + Is.EquivalentTo(lines)); } [Test] @@ -628,14 +608,15 @@ public async Task File_WriteAllBytes() if (fs.IsReadOnly) return; - var expected = new byte[1024 * 1024]; - Random.Shared.NextBytes(expected); - var path = $"{safePath}/{Guid.NewGuid()}"; - await fs.WriteAllBytesAsync(path, expected); + var path = "/project/b02a67d8.bin"; + var data = RandomNumberGenerator.GetBytes(10_000); - var data = await fs.ReadAllBytesAsync(path); - Assert.That(data.SequenceEqual(expected), Is.True); + await fs.WriteAllBytesAsync(path, data); + + Assert.That( + await fs.ReadAllBytesAsync(path), + Is.EquivalentTo(data)); await fs.DeleteFileAsync(path); } @@ -648,17 +629,17 @@ public async Task File_WriteAllText() return; var list = new List(); - for (var i = 0; i < 10240; i++) + for (var i = 0; i < 1000; i++) list.Add($"Hello, 世界! Unicode test: café, weiß, Привет, ёжик! こんにちは! {Guid.NewGuid()}"); - var contents = string.Join(Environment.NewLine, list); + var data = string.Join(Environment.NewLine, list); + var path = "/project/5cc34e90a7c9.txt"; - var path = $"{safePath}/{Guid.NewGuid()}"; - await fs.WriteAllTextAsync(path, contents); + await fs.WriteAllTextAsync(path, data); Assert.That( await fs.ReadAllTextAsync(path), - Is.EqualTo(contents)); + Is.EqualTo(data)); await fs.DeleteFileAsync(path); } @@ -671,10 +652,10 @@ public async Task File_WriteAllLines() return; var list = new List(); - for (var i = 0; i < 10240; i++) + for (var i = 0; i < 1000; i++) list.Add($"Hello, 世界! Unicode test: café, weiß, Привет, ёжик! こんにちは! {Guid.NewGuid()}"); - var path = $"{safePath}/{Guid.NewGuid()}"; + var path = "/project/76bb51ee6cb7.txt"; await fs.WriteAllLinesAsync(path, list); Assert.That( @@ -684,16 +665,219 @@ await fs.ReadAllLinesAsync(path), await fs.DeleteFileAsync(path); } + [Test] + public async Task Directory_GetFiles() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets").GetFilesAsync().AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project/assets/images") + .GetFilesAsync() + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/images/icon.svg", + "/project/assets/images/logo.png" + ])); + } + + [Test] + public async Task Directory_GetDirectories() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets/fonts").GetDirectoriesAsync().AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project/assets") + .GetDirectoriesAsync() + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/fonts", + "/project/assets/images", + "/project/assets/styles" + ]) + ); + } + + [Test] + public async Task Directory_GetFileNodes() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets/scripts").GetFileNodesAsync().AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project/assets/images") + .GetFileNodesAsync() + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/images/backgrounds", + "/project/assets/images/icon.svg", + "/project/assets/images/logo.png" + ]) + ); + } + + [Test] + public async Task Directory_Glob_GetFiles() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets").GetFilesAsync("*/*.otf").AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project") + .GetFilesAsync( + pattern: "[a][s]set[s]/{images,styles}/{main.css,logo.png}") + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/images/logo.png", + "/project/assets/styles/main.css" + ]) + ); + + Assert.That( + await fs + .GetDirectory("/project") + .GetFilesAsync( + patterns: ["[a]sset{,s}/*/*.{svg,png}", "assets/{images,styles}/*.css"], + excludes: ["assets/imag*/*icon*", "**/style*/print.css"]) + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/images/logo.png", + "/project/assets/styles/main.css" + ]) + ); + } + + [Test] + public async Task Directory_Glob_GetDirectories() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets").GetDirectoriesAsync("{styles,fonts}/*").AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project") + .GetDirectoriesAsync("[a]sset{,s}/*/*") + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo(["/project/assets/images/backgrounds"])); + + Assert.That( + await fs + .GetDirectory("/project") + .GetDirectoriesAsync( + patterns: ["[a]sset{,s}/*/*", "*/**/*s"], + excludes: ["src/**", "tests/**/Fixtures"]) + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/fonts", + "/project/assets/images", + "/project/assets/images/backgrounds", + "/project/assets/styles" + ]) + ); + } + + [Test] + public async Task Directory_Glob_GetFileNodes() + { + using var fs = GetFileSystem(); + + Assert.That( + await fs.GetDirectory("/project/assets").GetFileNodesAsync("{styles,fonts}/*/*").AnyAsync(), + Is.False); + + Assert.That( + await fs + .GetDirectory("/project") + .GetFileNodesAsync("[a]sset{,s}/*/*") + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/fonts/Arial.ttf", + "/project/assets/fonts/Roboto.ttf", + "/project/assets/images/backgrounds", + "/project/assets/images/icon.svg", + "/project/assets/images/logo.png", + "/project/assets/styles/main.css", + "/project/assets/styles/print.css" + ])); + + Assert.That( + await fs + .GetDirectory("/project") + .GetFileNodesAsync( + patterns: ["[a]sset{,s}/*/*", "*/**/*s"], + excludes: ["src/**", "tests/**"]) + .Select(f => f.FullName) + .OrderBy(p => p) + .ToArrayAsync(), + Is.EquivalentTo( + [ + "/project/assets/fonts", + "/project/assets/fonts/Arial.ttf", + "/project/assets/fonts/Roboto.ttf", + "/project/assets/images", + "/project/assets/images/backgrounds", + "/project/assets/images/icon.svg", + "/project/assets/images/logo.png", + "/project/assets/styles", + "/project/assets/styles/main.css", + "/project/assets/styles/print.css" + ]) + ); + } + [Test] public async Task Directory_Enumerate_ReturnsEmpty_For_NonExistingDirectory() { using var fs = GetFileSystem(); - var directory = fs.GetDirectory($"/{Guid.NewGuid()}"); + var directory = fs.GetDirectory("/project/b02a67d85cc34e90"); - Assert.That(await directory.GetFileNodesAsync().CountAsync(), Is.Zero); - Assert.That(await directory.GetFilesAsync().CountAsync(), Is.Zero); - Assert.That(await directory.GetDirectoriesAsync().CountAsync(), Is.Zero); + Assert.That(await directory.GetFileNodesAsync().AnyAsync(), Is.False); + Assert.That(await directory.GetFilesAsync().AnyAsync(), Is.False); + Assert.That(await directory.GetDirectoriesAsync().AnyAsync(), Is.False); } [Test] @@ -704,12 +888,7 @@ public async Task Directory_Create_For_ExistingDirectory() if (fs.IsReadOnly) return; - Assert.That( - await fs.GetDirectoriesAsync("/", "**").AnyAsync(), - Is.True); - - await foreach (var directory in fs.GetDirectoriesAsync("/", "**")) - await directory.CreateAsync(); + await fs.CreateDirectoryAsync("/project/assets"); } [Test] @@ -720,11 +899,13 @@ public async Task Directory_Create_For_NonExistingDirectory() if (fs.IsReadOnly) return; - var name = Guid.NewGuid().ToString(); - var directory = fs.GetDirectory($"{safePath}/{name}/{name}"); + await fs.CreateDirectoryAsync("/project/dcc2d926/8ddb/4a79/8246/a60ed6146c53"); + + Assert.That( + await fs.DirectoryExistsAsync("/project/dcc2d926/8ddb/4a79/8246/a60ed6146c53"), + Is.True); - await directory.CreateAsync(); - Assert.That(await directory.ExistsAsync(), Is.True); + await fs.DeleteDirectoryAsync("/project/dcc2d926"); } [Test] @@ -735,25 +916,20 @@ public async Task Directory_Delete() if (fs.IsReadOnly) return; - var name = Guid.NewGuid().ToString(); + var directory = fs.GetDirectory("/project/1111b18e"); - var directory = fs.GetDirectory($"{safePath}/{name}-dir"); - - for (var i = 0; i < 10; i++) - { - await fs.WriteAsync($"{directory.FullName}/{Guid.NewGuid()}.txt", new MemoryStream()); - await fs.WriteAsync($"{directory.FullName}/{Guid.NewGuid()}/{Guid.NewGuid()}.txt", new MemoryStream()); - } + await fs.WriteAsync("/project/1111b18e/4bb69411.txt", Stream.Null); + await fs.WriteAsync("/project/1111b18e/f1e5f79eb6be/909c4bb69411.txt", Stream.Null); Assert.That( await directory.GetFilesAsync("**").CountAsync(), - Is.EqualTo(20)); + Is.EqualTo(2)); await directory.DeleteAsync(); Assert.That( - await directory.GetFilesAsync("**").CountAsync(), - Is.Zero); + await directory.GetFilesAsync("**").AnyAsync(), + Is.False); } [Test] @@ -764,35 +940,26 @@ public async Task Directory_Delete_For_NonExistingDirectory() if (fs.IsReadOnly) return; - var name = Guid.NewGuid().ToString(); + var directory = fs.GetDirectory("/project/73d0e99e"); - var dir1 = fs.GetDirectory($"/{name}-1"); - var dir2 = fs.GetDirectory($"/{name}-2/{name}-3"); - var dir3 = fs.GetDirectory($"{safePath}/{name}-4"); - var dir4 = fs.GetDirectory($"{safePath}/{name}-5/{name}-6"); - - Assert.That(await dir1.GetFileNodesAsync().CountAsync(), Is.Zero); - Assert.That(await dir2.GetFileNodesAsync().CountAsync(), Is.Zero); - Assert.That(await dir3.GetFileNodesAsync().CountAsync(), Is.Zero); - Assert.That(await dir4.GetFileNodesAsync().CountAsync(), Is.Zero); + Assert.That( + await directory.GetFileNodesAsync().AnyAsync(), + Is.False); - await dir1.DeleteAsync(); - await dir2.DeleteAsync(); - await dir3.DeleteAsync(); - await dir4.DeleteAsync(); + await directory.DeleteAsync(); } [Test] - [SuppressMessage("ReSharper", "AccessToDisposedClosure")] - public void Directory_Readonly_Delete_ThrowsException_For_NonExistingDirectory() + public async Task Directory_Readonly_Delete_ThrowsException_For_NonExistingDirectory() { using var fs = GetFileSystem(); if (!fs.IsReadOnly) return; - Assert.That(() => fs.DeleteDirectoryAsync($"/{Guid.NewGuid()}"), Throws.Exception); - Assert.That(() => fs.DeleteDirectoryAsync($"{safePath}/{Guid.NewGuid()}"), Throws.Exception); + await Assert.ThatAsync( + async () => await fs.DeleteDirectoryAsync("/project/22ef456"), + Throws.Exception); } /// diff --git a/tests/Ramstack.FileSystem.Sub.Tests/Ramstack.FileSystem.Sub.Tests.csproj b/tests/Ramstack.FileSystem.Sub.Tests/Ramstack.FileSystem.Sub.Tests.csproj index 32f405a..377b86d 100644 --- a/tests/Ramstack.FileSystem.Sub.Tests/Ramstack.FileSystem.Sub.Tests.csproj +++ b/tests/Ramstack.FileSystem.Sub.Tests/Ramstack.FileSystem.Sub.Tests.csproj @@ -21,6 +21,7 @@ + diff --git a/tests/Ramstack.FileSystem.Sub.Tests/SubFileSystemTests.cs b/tests/Ramstack.FileSystem.Sub.Tests/SubFileSystemTests.cs index 25d34e5..a535b39 100644 --- a/tests/Ramstack.FileSystem.Sub.Tests/SubFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Sub.Tests/SubFileSystemTests.cs @@ -1,4 +1,5 @@ using Ramstack.FileSystem.Physical; +using Ramstack.FileSystem.Prefixed; using Ramstack.FileSystem.Specification.Tests; using Ramstack.FileSystem.Specification.Tests.Utilities; @@ -14,8 +15,10 @@ public void Cleanup() => _storage.Dispose(); protected override IVirtualFileSystem GetFileSystem() => - new SubFileSystem("project/docs", new PhysicalFileSystem(_storage.Root)); + new SubFileSystem("/bin/app", + new PrefixedFileSystem("/bin/app", + new PhysicalFileSystem(_storage.Root))); protected override DirectoryInfo GetDirectoryInfo() => - new(Path.Join(_storage.Root, "project", "docs")); + new DirectoryInfo(_storage.Root); }