From 1d5bfd0e38ea4993a7252a05a6505a8f7d54cc34 Mon Sep 17 00:00:00 2001 From: Lucki2g Date: Tue, 18 Nov 2025 17:20:11 +0100 Subject: [PATCH 01/23] chore: tooltips on header buttons --- Website/components/shared/Header.tsx | 82 +++++++++++++++------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/Website/components/shared/Header.tsx b/Website/components/shared/Header.tsx index c72417c..e88635c 100644 --- a/Website/components/shared/Header.tsx +++ b/Website/components/shared/Header.tsx @@ -2,7 +2,7 @@ import { useLoading } from '@/hooks/useLoading'; import { useAuth } from '@/contexts/AuthContext'; import { useSettings } from '@/contexts/SettingsContext'; import { useRouter, usePathname } from 'next/navigation'; -import { AppBar, Toolbar, Box, LinearProgress, Button, Stack } from '@mui/material'; +import { AppBar, Toolbar, Box, LinearProgress, Button, Stack, Tooltip } from '@mui/material'; import SettingsPane from './elements/SettingsPane'; import { useIsMobile } from '@/hooks/use-mobile'; import { useSidebar } from '@/contexts/SidebarContext'; @@ -72,49 +72,57 @@ const Header = ({ }: HeaderProps) => { /> )} {isMobile && !sidebarOpen && isAuthenticated && ( - + + + )} {/* Right side - navigation buttons */} - + + + {isAuthenticated && ( <> - - + + + + + + )} From 7772903f81ea214e2d545198031a6b8e05cc83d7 Mon Sep 17 00:00:00 2001 From: Lucki2g Date: Tue, 18 Nov 2025 17:26:53 +0100 Subject: [PATCH 02/23] chore: adjustments to search box on metadata --- Website/components/datamodelview/TimeSlicedSearch.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Website/components/datamodelview/TimeSlicedSearch.tsx b/Website/components/datamodelview/TimeSlicedSearch.tsx index f3e421c..8ad9c06 100644 --- a/Website/components/datamodelview/TimeSlicedSearch.tsx +++ b/Website/components/datamodelview/TimeSlicedSearch.tsx @@ -237,7 +237,7 @@ export const TimeSlicedSearch = ({ }; const searchInput = ( - + From 698f2504a712350ef9c3616422faf921b3e8c7b9 Mon Sep 17 00:00:00 2001 From: Lucki2g Date: Tue, 18 Nov 2025 17:55:52 +0100 Subject: [PATCH 03/23] chore: refactored dataverseservice into smaller services --- Generator/DTO/Attributes/LookupAttribute.cs | 11 +- Generator/DataverseService.cs | 718 +----------------- Generator/Generator.csproj | 1 + Generator/Program.cs | 107 ++- Generator/Services/AttributeMappingService.cs | 58 ++ Generator/Services/EntityIconService.cs | 72 ++ Generator/Services/EntityMetadataService.cs | 72 ++ Generator/Services/RecordMappingService.cs | 80 ++ Generator/Services/RelationshipService.cs | 133 ++++ Generator/Services/SecurityRoleService.cs | 121 +++ Generator/Services/SolutionService.cs | 298 ++++++++ 11 files changed, 977 insertions(+), 694 deletions(-) create mode 100644 Generator/Services/AttributeMappingService.cs create mode 100644 Generator/Services/EntityIconService.cs create mode 100644 Generator/Services/EntityMetadataService.cs create mode 100644 Generator/Services/RecordMappingService.cs create mode 100644 Generator/Services/RelationshipService.cs create mode 100644 Generator/Services/SecurityRoleService.cs create mode 100644 Generator/Services/SolutionService.cs diff --git a/Generator/DTO/Attributes/LookupAttribute.cs b/Generator/DTO/Attributes/LookupAttribute.cs index 198fae7..5f6a4a6 100644 --- a/Generator/DTO/Attributes/LookupAttribute.cs +++ b/Generator/DTO/Attributes/LookupAttribute.cs @@ -13,10 +13,17 @@ internal class LookupAttribute : Attribute { public IEnumerable Targets { get; } - public LookupAttribute(LookupAttributeMetadata metadata, Dictionary logicalToSchema, ILogger logger) + public LookupAttribute(LookupAttributeMetadata metadata, Dictionary logicalToSchema, ILogger? logger = null) : base(metadata) { - foreach (var target in metadata.Targets) { if (!logicalToSchema.ContainsKey(target)) logger.LogError($"Missing logicalname in logicalToSchema {target}, on entity {metadata.EntityLogicalName}."); } + if (logger != null) + { + foreach (var target in metadata.Targets) + { + if (!logicalToSchema.ContainsKey(target)) + logger.LogError($"Missing logicalname in logicalToSchema {target}, on entity {metadata.EntityLogicalName}."); + } + } Targets = metadata.Targets diff --git a/Generator/DataverseService.cs b/Generator/DataverseService.cs index fe3975a..c37c882 100644 --- a/Generator/DataverseService.cs +++ b/Generator/DataverseService.cs @@ -1,6 +1,4 @@ -using Azure.Core; -using Azure.Identity; -using Generator.DTO; +using Generator.DTO; using Generator.DTO.Attributes; using Generator.DTO.Warnings; using Generator.Queries; @@ -8,18 +6,11 @@ using Generator.Services.Plugins; using Generator.Services.PowerAutomate; using Generator.Services.WebResources; -using Microsoft.Crm.Sdk.Messages; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; -using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata; -using Microsoft.Xrm.Sdk.Query; -using System.Collections.Concurrent; using System.Diagnostics; -using System.Reflection; -using Attribute = Generator.DTO.Attributes.Attribute; namespace Generator { @@ -28,25 +19,32 @@ internal class DataverseService private readonly ServiceClient client; private readonly IConfiguration configuration; private readonly ILogger logger; + private readonly EntityMetadataService entityMetadataService; + private readonly SolutionService solutionService; + private readonly SecurityRoleService securityRoleService; + private readonly EntityIconService entityIconService; + private readonly RecordMappingService recordMappingService; private readonly List analyzerRegistrations; - public DataverseService(IConfiguration configuration, ILogger logger) + public DataverseService( + ServiceClient client, + IConfiguration configuration, + ILogger logger, + EntityMetadataService entityMetadataService, + SolutionService solutionService, + SecurityRoleService securityRoleService, + EntityIconService entityIconService, + RecordMappingService recordMappingService) { + this.client = client; this.configuration = configuration; this.logger = logger; - - var cache = new MemoryCache(new MemoryCacheOptions()); - - var dataverseUrl = configuration["DataverseUrl"]; - if (dataverseUrl == null) - { - throw new Exception("DataverseUrl is required"); - } - - client = new ServiceClient( - instanceUrl: new Uri(dataverseUrl), - tokenProviderFunction: url => TokenProviderFunction(url, cache, logger)); + this.entityMetadataService = entityMetadataService; + this.solutionService = solutionService; + this.securityRoleService = securityRoleService; + this.entityIconService = entityIconService; + this.recordMappingService = recordMappingService; // Register all analyzers with their query functions analyzerRegistrations = new List @@ -69,8 +67,8 @@ public DataverseService(IConfiguration configuration, ILogger public async Task<(IEnumerable, IEnumerable, IEnumerable)> GetFilteredMetadata() { var warnings = new List(); // used to collect warnings for the insights dashboard - var (solutionIds, solutionEntities) = await GetSolutionIds(); - var solutionComponents = await GetSolutionComponents(solutionIds); // (id, type, rootcomponentbehavior, solutionid) + var (solutionIds, solutionEntities) = await solutionService.GetSolutionIds(); + var solutionComponents = await solutionService.GetSolutionComponents(solutionIds); // (id, type, rootcomponentbehavior, solutionid) var entitiesInSolution = solutionComponents.Where(x => x.ComponentType == 1).Select(x => x.ObjectId).Distinct().ToList(); var entityRootBehaviour = solutionComponents @@ -85,7 +83,7 @@ public DataverseService(IConfiguration configuration, ILogger var attributesInSolution = solutionComponents.Where(x => x.ComponentType == 2).Select(x => x.ObjectId).ToHashSet(); var rolesInSolution = solutionComponents.Where(x => x.ComponentType == 20).Select(x => x.ObjectId).Distinct().ToList(); - var entitiesInSolutionMetadata = await GetEntityMetadata(entitiesInSolution); + var entitiesInSolutionMetadata = await entityMetadataService.GetEntityMetadataByObjectIds(entitiesInSolution); var logicalNameToKeys = entitiesInSolutionMetadata.ToDictionary( entity => entity.LogicalName, @@ -95,7 +93,7 @@ public DataverseService(IConfiguration configuration, ILogger key.KeyAttributes) ).ToList()); - var logicalNameToSecurityRoles = await GetSecurityRoles(rolesInSolution, entitiesInSolutionMetadata.ToDictionary(x => x.LogicalName, x => x.Privileges)); + var logicalNameToSecurityRoles = await securityRoleService.GetSecurityRoles(rolesInSolution, entitiesInSolutionMetadata.ToDictionary(x => x.LogicalName, x => x.Privileges)); var entityLogicalNamesInSolution = entitiesInSolutionMetadata.Select(e => e.LogicalName).ToHashSet(); logger.LogInformation("There are {Count} entities in the solution.", entityLogicalNamesInSolution.Count); @@ -111,13 +109,13 @@ public DataverseService(IConfiguration configuration, ILogger foreach (var target in entityLogicalNamesOutsideSolution) relatedEntityLogicalNames.Add(target); } logger.LogInformation("There are {Count} entities referenced outside the solution.", relatedEntityLogicalNames.Count); - var referencedEntityMetadata = await GetEntityMetadataByLogicalName(relatedEntityLogicalNames.ToList()); + var referencedEntityMetadata = await entityMetadataService.GetEntityMetadataByLogicalNames(relatedEntityLogicalNames.ToList()); var allEntityMetadata = entitiesInSolutionMetadata.Concat(referencedEntityMetadata).ToList(); var logicalToSchema = allEntityMetadata.ToDictionary(x => x.LogicalName, x => new ExtendedEntityInformation { Name = x.SchemaName, IsInSolution = entitiesInSolutionMetadata.Any(e => e.LogicalName == x.LogicalName) }); var attributeLogicalToSchema = allEntityMetadata.ToDictionary(x => x.LogicalName, x => x.Attributes?.ToDictionary(attr => attr.LogicalName, attr => attr.DisplayName.UserLocalizedLabel?.Label ?? attr.SchemaName) ?? []); - var entityIconMap = await GetEntityIconMap(allEntityMetadata); + var entityIconMap = await entityIconService.GetEntityIconMap(allEntityMetadata); // Processes analysis var attributeUsages = new Dictionary>>(); @@ -154,7 +152,7 @@ public DataverseService(IConfiguration configuration, ILogger new AttributeWarning($"{attributeDict.Key} was used inside a {usage.ComponentType} component [{usage.Name}]. However, the entity {entityKey} could not be resolved in the provided solutions."))))); // Create solutions with their components - var solutions = await CreateSolutions(solutionEntities, solutionComponents, allEntityMetadata); + var solutions = await solutionService.CreateSolutions(solutionEntities, solutionComponents, allEntityMetadata); return (records .Select(x => @@ -162,8 +160,7 @@ public DataverseService(IConfiguration configuration, ILogger logicalNameToSecurityRoles.TryGetValue(x.EntityMetadata.LogicalName, out var securityRoles); logicalNameToKeys.TryGetValue(x.EntityMetadata.LogicalName, out var keys); - return MakeRecord( - logger, + return recordMappingService.CreateRecord( x.EntityMetadata, x.RelevantAttributes, x.RelevantManyToMany, @@ -172,666 +169,11 @@ public DataverseService(IConfiguration configuration, ILogger securityRoles ?? [], keys ?? [], entityIconMap, - attributeUsages, - configuration); + attributeUsages); }), warnings, solutions); } - - private async Task> CreateSolutions( - List solutionEntities, - IEnumerable<(Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId)> solutionComponents, - List allEntityMetadata) - { - var solutions = new List(); - - // Create lookup dictionaries for faster access - var entityLookup = allEntityMetadata.ToDictionary(e => e.MetadataId ?? Guid.Empty, e => e); - - // Fetch all unique publishers for the solutions - var publisherIds = solutionEntities - .Select(s => s.GetAttributeValue("publisherid").Id) - .Distinct() - .ToList(); - - var publisherQuery = new QueryExpression("publisher") - { - ColumnSet = new ColumnSet("publisherid", "friendlyname", "customizationprefix"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("publisherid", ConditionOperator.In, publisherIds) - } - } - }; - - var publishers = await client.RetrieveMultipleAsync(publisherQuery); - var publisherLookup = publishers.Entities.ToDictionary( - p => p.GetAttributeValue("publisherid"), - p => ( - Name: p.GetAttributeValue("friendlyname") ?? "Unknown Publisher", - Prefix: p.GetAttributeValue("customizationprefix") ?? string.Empty - )); - - // Group components by solution - var componentsBySolution = solutionComponents.GroupBy(c => c.SolutionId).ToDictionary(g => g.Key.Id, g => g); - - // Process ALL solutions from configuration, not just those with components - foreach (var solutionEntity in solutionEntities) - { - var solutionId = solutionEntity.GetAttributeValue("solutionid"); - - var solutionName = solutionEntity.GetAttributeValue("friendlyname") ?? - solutionEntity.GetAttributeValue("uniquename") ?? - "Unknown Solution"; - - var publisherId = solutionEntity.GetAttributeValue("publisherid").Id; - var publisher = publisherLookup.GetValueOrDefault(publisherId); - - var components = new List(); - - // Add components if this solution has any - if (componentsBySolution.TryGetValue(solutionId, out var solutionGroup)) - { - foreach (var component in solutionGroup) - { - var solutionComponent = CreateSolutionComponent(component, entityLookup, allEntityMetadata, publisherLookup); - if (solutionComponent != null) - { - components.Add(solutionComponent); - } - } - } - - // Add solution even if components list is empty (e.g., flow-only solutions) - solutions.Add(new Solution( - solutionName, - publisher.Name, - publisher.Prefix, - components)); - } - - return solutions.AsEnumerable(); - } - - private SolutionComponent? CreateSolutionComponent( - (Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId) component, - Dictionary entityLookup, - List allEntityMetadata, - Dictionary publisherLookup) - { - try - { - switch (component.ComponentType) - { - case 1: // Entity - // Try to find entity by MetadataId first, then by searching all entities - if (entityLookup.TryGetValue(component.ObjectId, out var entityMetadata)) - { - var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(entityMetadata.SchemaName, publisherLookup); - return new SolutionComponent( - entityMetadata.DisplayName?.UserLocalizedLabel?.Label ?? entityMetadata.SchemaName, - entityMetadata.SchemaName, - entityMetadata.Description?.UserLocalizedLabel?.Label ?? string.Empty, - SolutionComponentType.Entity, - publisherName, - publisherPrefix); - } - - // Entity lookup by ObjectId is complex in Dataverse, so we'll skip the fallback for now - // The primary lookup by MetadataId should handle most cases - break; - - case 2: // Attribute - // Search for attribute across all entities - foreach (var entity in allEntityMetadata) - { - var attribute = entity.Attributes?.FirstOrDefault(a => a.MetadataId == component.ObjectId); - if (attribute != null) - { - var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(attribute.SchemaName, publisherLookup); - return new SolutionComponent( - attribute.DisplayName?.UserLocalizedLabel?.Label ?? attribute.SchemaName, - attribute.SchemaName, - attribute.Description?.UserLocalizedLabel?.Label ?? string.Empty, - SolutionComponentType.Attribute, - publisherName, - publisherPrefix); - } - } - break; - - case 3: // Relationship (if you want to add this to the enum later) - // Search for relationships across all entities - foreach (var entity in allEntityMetadata) - { - // Check one-to-many relationships - var oneToMany = entity.OneToManyRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); - if (oneToMany != null) - { - var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(oneToMany.SchemaName, publisherLookup); - return new SolutionComponent( - oneToMany.SchemaName, - oneToMany.SchemaName, - $"One-to-Many: {entity.SchemaName} -> {oneToMany.ReferencingEntity}", - SolutionComponentType.Relationship, - publisherName, - publisherPrefix); - } - - // Check many-to-one relationships - var manyToOne = entity.ManyToOneRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); - if (manyToOne != null) - { - var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(manyToOne.SchemaName, publisherLookup); - return new SolutionComponent( - manyToOne.SchemaName, - manyToOne.SchemaName, - $"Many-to-One: {entity.SchemaName} -> {manyToOne.ReferencedEntity}", - SolutionComponentType.Relationship, - publisherName, - publisherPrefix); - } - - // Check many-to-many relationships - var manyToMany = entity.ManyToManyRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); - if (manyToMany != null) - { - var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(manyToMany.SchemaName, publisherLookup); - return new SolutionComponent( - manyToMany.SchemaName, - manyToMany.SchemaName, - $"Many-to-Many: {manyToMany.Entity1LogicalName} <-> {manyToMany.Entity2LogicalName}", - SolutionComponentType.Relationship, - publisherName, - publisherPrefix); - } - } - break; - - case 20: // Security Role - skip for now as not in enum - case 92: // SDK Message Processing Step (Plugin) - skip for now as not in enum - break; - } - } - catch (Exception ex) - { - logger.LogWarning($"Failed to create solution component for ObjectId {component.ObjectId}, ComponentType {component.ComponentType}: {ex.Message}"); - } - - return null; - } - - private static (string PublisherName, string PublisherPrefix) GetPublisherFromSchemaName( - string schemaName, - Dictionary publisherLookup) - { - // Extract prefix from schema name (e.g., "contoso_entity" -> "contoso") - var parts = schemaName.Split('_', 2); - - if (parts.Length == 2) - { - var prefix = parts[0]; - - // Find publisher by matching prefix - foreach (var publisher in publisherLookup.Values) - { - if (publisher.Prefix.Equals(prefix, StringComparison.OrdinalIgnoreCase)) - { - return (publisher.Name, publisher.Prefix); - } - } - } - - // Default to Microsoft if no prefix or prefix not found - return ("Microsoft", ""); - } - - private static Record MakeRecord( - ILogger logger, - EntityMetadata entity, - List relevantAttributes, - List relevantManyToMany, - Dictionary logicalToSchema, - Dictionary> attributeLogicalToSchema, - List securityRoles, - List keys, - Dictionary entityIconMap, - Dictionary>> attributeUsages, - IConfiguration configuration) - { - var attributes = - relevantAttributes - .Select(metadata => - { - var attr = GetAttribute(metadata, entity, logicalToSchema, attributeUsages, logger); - attr.IsStandardFieldModified = MetadataExtensions.StandardFieldHasChanged(metadata, entity.DisplayName.UserLocalizedLabel?.Label ?? string.Empty, entity.IsCustomEntity ?? false); - return attr; - }) - .Where(x => !string.IsNullOrEmpty(x.DisplayName)) - .ToList(); - - var oneToMany = (entity.OneToManyRelationships ?? Enumerable.Empty()) - .Where(x => logicalToSchema.ContainsKey(x.ReferencingEntity) && logicalToSchema[x.ReferencingEntity].IsInSolution && attributeLogicalToSchema[x.ReferencingEntity].ContainsKey(x.ReferencingAttribute)) - .Select(x => new DTO.Relationship( - x.IsCustomRelationship ?? false, - x.ReferencingEntityNavigationPropertyName, - logicalToSchema[x.ReferencingEntity].Name, - attributeLogicalToSchema[x.ReferencingEntity][x.ReferencingAttribute], - x.SchemaName, - IsManyToMany: false, - x.CascadeConfiguration)) - .ToList(); - - var manyToMany = relevantManyToMany - .Where(x => logicalToSchema.ContainsKey(x.Entity1LogicalName) && logicalToSchema[x.Entity1LogicalName].IsInSolution) - .Select(x => - { - var useEntity1 = x.Entity1LogicalName == entity.LogicalName; - - var label = !useEntity1 - ? x.Entity1AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity1NavigationPropertyName - : x.Entity2AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity2NavigationPropertyName; - - return new DTO.Relationship( - x.IsCustomRelationship ?? false, - label ?? x.SchemaName, // Fallback to schema name if no localized label is available, this is relevant for some Default/System Many to Many relationships. - logicalToSchema[!useEntity1 ? x.Entity1LogicalName : x.Entity2LogicalName].Name, - "-", - x.SchemaName, - IsManyToMany: true, - null - ); - }) - .ToList(); - - Dictionary tablegroups = []; // logicalname -> group - var tablegroupstring = configuration["TableGroups"]; - if (tablegroupstring?.Length > 0) - { - var groupEntries = tablegroupstring.Split(';', StringSplitOptions.RemoveEmptyEntries); - foreach (var g in groupEntries) - { - var tables = g.Split(':'); - if (tables.Length != 2) - { - logger.LogError($"Invalid format for tablegroup entry: ({g})"); - continue; - } - - var logicalNames = tables[1].Split(',', StringSplitOptions.RemoveEmptyEntries); - foreach (var logicalName in logicalNames) - if (!tablegroups.TryAdd(logicalName.Trim().ToLower(), tables[0].Trim())) - { - logger.LogWarning($"Dublicate logicalname detected: {logicalName} (already in tablegroup '{tablegroups[logicalName]}', dublicate found in group '{g}')"); - continue; - } - } - } - var (group, description) = GetGroupAndDescription(entity, tablegroups); - - entityIconMap.TryGetValue(entity.LogicalName, out string? iconBase64); - - return new Record( - entity.DisplayName.UserLocalizedLabel?.Label ?? string.Empty, - entity.SchemaName, - group, - description?.PrettyDescription(), - entity.IsAuditEnabled.Value, - entity.IsActivity ?? false, - entity.IsCustomEntity ?? false, - entity.OwnershipType ?? OwnershipTypes.UserOwned, - entity.HasNotes ?? false, - attributes, - oneToMany.Concat(manyToMany).ToList(), - securityRoles, - keys, - iconBase64); - } - - private static Attribute GetAttribute(AttributeMetadata metadata, EntityMetadata entity, Dictionary logicalToSchema, Dictionary>> attributeUsages, ILogger logger) - { - Attribute attr = metadata switch - { - PicklistAttributeMetadata picklist => new ChoiceAttribute(picklist), - MultiSelectPicklistAttributeMetadata multiSelect => new ChoiceAttribute(multiSelect), - LookupAttributeMetadata lookup => new LookupAttribute(lookup, logicalToSchema, logger), - StateAttributeMetadata state => new ChoiceAttribute(state), - StatusAttributeMetadata status => new StatusAttribute(status, (StateAttributeMetadata)entity.Attributes.First(x => x is StateAttributeMetadata)), - StringAttributeMetadata stringMetadata => new StringAttribute(stringMetadata), - IntegerAttributeMetadata integer => new IntegerAttribute(integer), - DateTimeAttributeMetadata dateTimeAttributeMetadata => new DateTimeAttribute(dateTimeAttributeMetadata), - MoneyAttributeMetadata money => new DecimalAttribute(money), - DecimalAttributeMetadata decimalAttribute => new DecimalAttribute(decimalAttribute), - MemoAttributeMetadata memo => new StringAttribute(memo), - BooleanAttributeMetadata booleanAttribute => new BooleanAttribute(booleanAttribute), - FileAttributeMetadata fileAttribute => new FileAttribute(fileAttribute), - _ => new GenericAttribute(metadata) - }; - - var schemaname = attributeUsages.GetValueOrDefault(entity.LogicalName)?.GetValueOrDefault(metadata.LogicalName) ?? []; - // also check the plural name, as some workflows like Power Automate use collectionname - var pluralname = attributeUsages.GetValueOrDefault(entity.LogicalCollectionName)?.GetValueOrDefault(metadata.LogicalName) ?? []; - - attr.AttributeUsages = [.. schemaname, .. pluralname]; - return attr; - } - - private static (string? Group, string? Description) GetGroupAndDescription(EntityMetadata entity, IDictionary tableGroups) - { - var description = entity.Description.UserLocalizedLabel?.Label ?? string.Empty; - if (!description.StartsWith("#")) - { - if (tableGroups.TryGetValue(entity.LogicalName, out var tablegroup)) - return (tablegroup, description); - return (null, description); - } - - var newlineIndex = description.IndexOf("\n"); - if (newlineIndex != -1) - { - var group = description.Substring(1, newlineIndex - 1).Trim(); - description = description.Substring(newlineIndex + 1); - return (group, description); - } - - var withoutHashtag = description.Substring(1).Trim(); - var firstSpace = withoutHashtag.IndexOf(" "); - if (firstSpace != -1) - return (withoutHashtag.Substring(0, firstSpace), withoutHashtag.Substring(firstSpace + 1)); - - return (withoutHashtag, null); - } - - public async Task> GetEntityMetadata(List entityObjectIds) - { - ConcurrentBag metadata = new(); - - // Disable affinity cookie - client.EnableAffinityCookie = false; - - var parallelOptions = new ParallelOptions() - { - MaxDegreeOfParallelism = client.RecommendedDegreesOfParallelism - }; - - await Parallel.ForEachAsync( - source: entityObjectIds, - parallelOptions: parallelOptions, - async (objectId, token) => - { - metadata.Add(await client.RetrieveEntityAsync(objectId, token)); - }); - - return metadata; - } - public async Task> GetEntityMetadataByLogicalName(List entityLogicalNames) - { - ConcurrentBag metadata = new(); - - // Disable affinity cookie - client.EnableAffinityCookie = false; - - var parallelOptions = new ParallelOptions() - { - MaxDegreeOfParallelism = client.RecommendedDegreesOfParallelism - }; - - await Parallel.ForEachAsync( - source: entityLogicalNames, - parallelOptions: parallelOptions, - async (logicalName, token) => - { - metadata.Add(await client.RetrieveEntityByLogicalNameAsync(logicalName, token)); - }); - - return metadata; - } - - private async Task<(List SolutionIds, List SolutionEntities)> GetSolutionIds() - { - var solutionNameArg = configuration["DataverseSolutionNames"]; - if (solutionNameArg == null) - { - throw new Exception("Specify one or more solutions"); - } - var solutionNames = solutionNameArg.Split(",").Select(x => x.Trim().ToLower()).ToList(); - - var resp = await client.RetrieveMultipleAsync(new QueryExpression("solution") - { - ColumnSet = new ColumnSet("publisherid", "friendlyname", "uniquename", "solutionid"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("uniquename", ConditionOperator.In, solutionNames) - } - } - }); - - return (resp.Entities.Select(e => e.GetAttributeValue("solutionid")).ToList(), resp.Entities.ToList()); - } - - public async Task> GetSolutionComponents(List solutionIds) - { - var entityQuery = new QueryExpression("solutioncomponent") - { - ColumnSet = new ColumnSet("objectid", "componenttype", "rootcomponentbehavior", "solutionid"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("componenttype", ConditionOperator.In, new List() { 1, 2, 20, 29, 92 }), // entity, attribute, role, workflow/flow, sdkpluginstep (https://learn.microsoft.com/en-us/power-apps/developer/data-platform/reference/entities/solutioncomponent) - new ConditionExpression("solutionid", ConditionOperator.In, solutionIds) - } - } - }; - - return - (await client.RetrieveMultipleAsync(entityQuery)) - .Entities - .Select(e => (e.GetAttributeValue("objectid"), e.GetAttributeValue("componenttype").Value, e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, e.GetAttributeValue("solutionid"))) - .ToList(); - } - - private async Task>> GetSecurityRoles(List rolesInSolution, Dictionary priviledges) - { - if (rolesInSolution.Count == 0) return []; - - var query = new QueryExpression("role") - { - ColumnSet = new ColumnSet("name"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("roleid", ConditionOperator.In, rolesInSolution) - } - }, - LinkEntities = - { - new LinkEntity("role", "roleprivileges", "roleid", "roleid", JoinOperator.Inner) - { - EntityAlias = "rolepriv", - Columns = new ColumnSet("privilegedepthmask"), - LinkEntities = - { - new LinkEntity("roleprivileges", "privilege", "privilegeid", "privilegeid", JoinOperator.Inner) - { - EntityAlias = "priv", - Columns = new ColumnSet("accessright"), - LinkEntities = - { - new LinkEntity("privilege", "privilegeobjecttypecodes", "privilegeid", "privilegeid", JoinOperator.Inner) - { - EntityAlias = "privotc", - Columns = new ColumnSet("objecttypecode") - } - } - } - } - } - } - }; - - var roles = await client.RetrieveMultipleAsync(query); - - var privileges = roles.Entities.Select(e => - { - var name = e.GetAttributeValue("name"); - var depth = (PrivilegeDepth)e.GetAttributeValue("rolepriv.privilegedepthmask").Value; - var accessRight = (AccessRights)e.GetAttributeValue("priv.accessright").Value; - var objectTypeCode = e.GetAttributeValue("privotc.objecttypecode").Value as string; - - return new - { - name, - depth, - accessRight, - objectTypeCode = objectTypeCode ?? string.Empty - }; - }); - - static PrivilegeDepth? GetDepth(Dictionary dict, AccessRights right, SecurityPrivilegeMetadata? meta) - { - if (!dict.TryGetValue(right, out var value)) - return meta?.CanBeGlobal ?? false ? 0 : null; - return value; - } - - return privileges - .GroupBy(x => x.objectTypeCode) - .ToDictionary(byLogicalName => byLogicalName.Key, byLogicalName => - byLogicalName - .GroupBy(x => x.name) - .Select(byRole => - { - var accessRights = byRole - .GroupBy(x => x.accessRight) - .ToDictionary(x => x.Key, x => x.First().depth); - - var priviledgeMetadata = priviledges.GetValueOrDefault(byLogicalName.Key) ?? []; - - return new SecurityRole( - byRole.Key, - byLogicalName.Key, - GetDepth(accessRights, AccessRights.CreateAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Create)), - GetDepth(accessRights, AccessRights.ReadAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Read)), - GetDepth(accessRights, AccessRights.WriteAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Write)), - GetDepth(accessRights, AccessRights.DeleteAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Delete)), - GetDepth(accessRights, AccessRights.AppendAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Append)), - GetDepth(accessRights, AccessRights.AppendToAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.AppendTo)), - GetDepth(accessRights, AccessRights.AssignAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Assign)), - GetDepth(accessRights, AccessRights.ShareAccess, priviledgeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Share)) - ); - }) - .ToList()); - } - - private async Task> GetEntityIconMap(IEnumerable entities) - { - var logicalNameToIconName = - entities - .Where(x => x.IconVectorName != null) - .ToDictionary(x => x.LogicalName, x => x.IconVectorName); - - var query = new QueryExpression("webresource") - { - ColumnSet = new ColumnSet("content", "name"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("name", ConditionOperator.In, logicalNameToIconName.Values.ToList()) - } - } - }; - - var webresources = await client.RetrieveMultipleAsync(query); - var iconNameToSvg = webresources.Entities.ToDictionary(x => x.GetAttributeValue("name"), x => x.GetAttributeValue("content")); - - var logicalNameToSvg = - logicalNameToIconName - .Where(x => iconNameToSvg.ContainsKey(x.Value) && !string.IsNullOrEmpty(iconNameToSvg[x.Value])) - .ToDictionary(x => x.Key, x => iconNameToSvg.GetValueOrDefault(x.Value) ?? string.Empty); - - var sourceDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - var iconDirectory = Path.Combine(sourceDirectory ?? string.Empty, "../../../entityicons"); - - var iconFiles = Directory.GetFiles(iconDirectory).ToDictionary(x => Path.GetFileNameWithoutExtension(x), x => x); - - foreach (var entity in entities) - { - if (logicalNameToSvg.ContainsKey(entity.LogicalName)) - { - continue; - } - - var iconKey = $"svg_{entity.ObjectTypeCode}"; - if (iconFiles.ContainsKey(iconKey)) - { - logicalNameToSvg[entity.LogicalName] = Convert.ToBase64String(File.ReadAllBytes(iconFiles[iconKey])); - } - } - - return logicalNameToSvg; - } - - private async Task TokenProviderFunction(string dataverseUrl, IMemoryCache cache, ILogger logger) - { - var cacheKey = $"AccessToken_{dataverseUrl}"; - - logger.LogTrace($"Attempting to retrieve access token for {dataverseUrl}"); - - return (await cache.GetOrCreateAsync(cacheKey, async cacheEntry => - { - cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(50); - var credential = GetTokenCredential(logger); - var scope = BuildScopeString(dataverseUrl); - - return await FetchAccessToken(credential, scope, logger); - })).Token; - } - - private TokenCredential GetTokenCredential(ILogger logger) - { - - if (configuration["DataverseClientId"] != null && configuration["DataverseClientSecret"] != null) - return new ClientSecretCredential(configuration["TenantId"], configuration["DataverseClientId"], configuration["DataverseClientSecret"]); - - return new DefaultAzureCredential(); // in azure this will be managed identity, locally this depends... se midway of this post for the how local identity is chosen: https://dreamingincrm.com/2021/11/16/connecting-to-dataverse-from-function-app-using-managed-identity/ - } - - private static string BuildScopeString(string dataverseUrl) - { - return $"{GetCoreUrl(dataverseUrl)}/.default"; - } - - private static string GetCoreUrl(string url) - { - var uri = new Uri(url); - return $"{uri.Scheme}://{uri.Host}"; - } - - private static async Task FetchAccessToken(TokenCredential credential, string scope, ILogger logger) - { - var tokenRequestContext = new TokenRequestContext(new[] { scope }); - - try - { - logger.LogTrace("Requesting access token..."); - var accessToken = await credential.GetTokenAsync(tokenRequestContext, CancellationToken.None); - logger.LogTrace("Access token successfully retrieved."); - return accessToken; - } - catch (Exception ex) - { - logger.LogError($"Failed to retrieve access token: {ex.Message}"); - throw; - } - } } /// diff --git a/Generator/Generator.csproj b/Generator/Generator.csproj index 73b95c5..c467319 100644 --- a/Generator/Generator.csproj +++ b/Generator/Generator.csproj @@ -11,6 +11,7 @@ + diff --git a/Generator/Program.cs b/Generator/Program.cs index 11c451b..ee0508d 100644 --- a/Generator/Program.cs +++ b/Generator/Program.cs @@ -1,6 +1,12 @@ -using Generator; +using Azure.Core; +using Azure.Identity; +using Generator; +using Generator.Services; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.PowerPlatform.Dataverse.Client; var configuration = new ConfigurationBuilder() @@ -9,17 +15,110 @@ .Build(); var verbose = configuration.GetValue("Verbosity", LogLevel.Information); -using var loggerFactory = LoggerFactory.Create(builder => +// Set up dependency injection +var services = new ServiceCollection(); + +// Add logging +services.AddLogging(builder => { builder .SetMinimumLevel(verbose) .AddConsole(); }); -var logger = loggerFactory.CreateLogger(); -var dataverseService = new DataverseService(configuration, logger); +// Add configuration as a singleton +services.AddSingleton(configuration); + +// Add ServiceClient as a singleton +services.AddSingleton(sp => +{ + var config = sp.GetRequiredService(); + var loggerFactory = sp.GetRequiredService(); + var logger = loggerFactory.CreateLogger("ServiceClient"); + var cache = new MemoryCache(new MemoryCacheOptions()); + + var dataverseUrl = config["DataverseUrl"]; + if (dataverseUrl == null) + { + throw new Exception("DataverseUrl is required"); + } + + return new ServiceClient( + instanceUrl: new Uri(dataverseUrl), + tokenProviderFunction: async url => await GetTokenAsync(url, cache, logger, config)); +}); + +// Register services +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); + +// Build service provider +var serviceProvider = services.BuildServiceProvider(); + +// Resolve and use DataverseService +var dataverseService = serviceProvider.GetRequiredService(); var (entities, warnings, solutions) = await dataverseService.GetFilteredMetadata(); var websiteBuilder = new WebsiteBuilder(configuration, entities, warnings, solutions); websiteBuilder.AddData(); +// Token provider function +static async Task GetTokenAsync(string dataverseUrl, IMemoryCache cache, ILogger logger, IConfiguration configuration) +{ + var cacheKey = $"AccessToken_{dataverseUrl}"; + + logger.LogTrace($"Attempting to retrieve access token for {dataverseUrl}"); + + return (await cache.GetOrCreateAsync(cacheKey, async cacheEntry => + { + cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(50); + var credential = GetTokenCredential(logger, configuration); + var scope = BuildScopeString(dataverseUrl); + + return await FetchAccessToken(credential, scope, logger); + }))!.Token; +} + +static TokenCredential GetTokenCredential(ILogger logger, IConfiguration configuration) +{ + if (configuration["DataverseClientId"] != null && configuration["DataverseClientSecret"] != null) + return new ClientSecretCredential(configuration["TenantId"], configuration["DataverseClientId"], configuration["DataverseClientSecret"]); + + return new DefaultAzureCredential(); // in azure this will be managed identity, locally this depends... se midway of this post for the how local identity is chosen: https://dreamingincrm.com/2021/11/16/connecting-to-dataverse-from-function-app-using-managed-identity/ +} + +static string BuildScopeString(string dataverseUrl) +{ + return $"{GetCoreUrl(dataverseUrl)}/.default"; +} + +static string GetCoreUrl(string url) +{ + var uri = new Uri(url); + return $"{uri.Scheme}://{uri.Host}"; +} + +static async Task FetchAccessToken(TokenCredential credential, string scope, ILogger logger) +{ + var tokenRequestContext = new TokenRequestContext(new[] { scope }); + + try + { + logger.LogTrace("Requesting access token..."); + var accessToken = await credential.GetTokenAsync(tokenRequestContext, CancellationToken.None); + logger.LogTrace("Access token successfully retrieved."); + return accessToken; + } + catch (Exception ex) + { + logger.LogError($"Failed to retrieve access token: {ex.Message}"); + throw; + } +} + diff --git a/Generator/Services/AttributeMappingService.cs b/Generator/Services/AttributeMappingService.cs new file mode 100644 index 0000000..7c93d3f --- /dev/null +++ b/Generator/Services/AttributeMappingService.cs @@ -0,0 +1,58 @@ +using Generator.DTO; +using Generator.DTO.Attributes; +using Microsoft.Extensions.Logging; +using Microsoft.Xrm.Sdk.Metadata; +using Attribute = Generator.DTO.Attributes.Attribute; + +namespace Generator.Services +{ + /// + /// Service responsible for mapping Dataverse AttributeMetadata to DTO Attribute objects + /// + internal class AttributeMappingService + { + private readonly ILogger logger; + + public AttributeMappingService(ILogger logger) + { + this.logger = logger; + } + + /// + /// Maps AttributeMetadata to the appropriate DTO Attribute type + /// + public Attribute MapAttribute( + AttributeMetadata metadata, + EntityMetadata entity, + Dictionary logicalToSchema, + Dictionary>> attributeUsages) + { + Attribute attr = metadata switch + { + PicklistAttributeMetadata picklist => new ChoiceAttribute(picklist), + MultiSelectPicklistAttributeMetadata multiSelect => new ChoiceAttribute(multiSelect), + LookupAttributeMetadata lookup => new LookupAttribute(lookup, logicalToSchema), + StateAttributeMetadata state => new ChoiceAttribute(state), + StatusAttributeMetadata status => new StatusAttribute(status, (StateAttributeMetadata)entity.Attributes.First(x => x is StateAttributeMetadata)), + StringAttributeMetadata stringMetadata => new StringAttribute(stringMetadata), + IntegerAttributeMetadata integer => new IntegerAttribute(integer), + DateTimeAttributeMetadata dateTimeAttributeMetadata => new DateTimeAttribute(dateTimeAttributeMetadata), + MoneyAttributeMetadata money => new DecimalAttribute(money), + DecimalAttributeMetadata decimalAttribute => new DecimalAttribute(decimalAttribute), + MemoAttributeMetadata memo => new StringAttribute(memo), + BooleanAttributeMetadata booleanAttribute => new BooleanAttribute(booleanAttribute), + FileAttributeMetadata fileAttribute => new FileAttribute(fileAttribute), + _ => new GenericAttribute(metadata) + }; + + var schemaname = attributeUsages.GetValueOrDefault(entity.LogicalName)?.GetValueOrDefault(metadata.LogicalName) ?? []; + // also check the plural name, as some workflows like Power Automate use collectionname + var pluralname = attributeUsages.GetValueOrDefault(entity.LogicalCollectionName)?.GetValueOrDefault(metadata.LogicalName) ?? []; + + attr.AttributeUsages = [.. schemaname, .. pluralname]; + attr.IsStandardFieldModified = MetadataExtensions.StandardFieldHasChanged(metadata, entity.DisplayName.UserLocalizedLabel?.Label ?? string.Empty, entity.IsCustomEntity ?? false); + + return attr; + } + } +} diff --git a/Generator/Services/EntityIconService.cs b/Generator/Services/EntityIconService.cs new file mode 100644 index 0000000..c5c04da --- /dev/null +++ b/Generator/Services/EntityIconService.cs @@ -0,0 +1,72 @@ +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; +using System.Reflection; + +namespace Generator.Services +{ + /// + /// Service responsible for retrieving entity icons from Dataverse and local files + /// + internal class EntityIconService + { + private readonly ServiceClient client; + + public EntityIconService(ServiceClient client) + { + this.client = client; + } + + /// + /// Retrieves entity icons, first from Dataverse webresources, then from local entityicons directory + /// + public async Task> GetEntityIconMap(IEnumerable entities) + { + var logicalNameToIconName = + entities + .Where(x => x.IconVectorName != null) + .ToDictionary(x => x.LogicalName, x => x.IconVectorName); + + var query = new QueryExpression("webresource") + { + ColumnSet = new ColumnSet("content", "name"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("name", ConditionOperator.In, logicalNameToIconName.Values.ToList()) + } + } + }; + + var webresources = await client.RetrieveMultipleAsync(query); + var iconNameToSvg = webresources.Entities.ToDictionary(x => x.GetAttributeValue("name"), x => x.GetAttributeValue("content")); + + var logicalNameToSvg = + logicalNameToIconName + .Where(x => iconNameToSvg.ContainsKey(x.Value) && !string.IsNullOrEmpty(iconNameToSvg[x.Value])) + .ToDictionary(x => x.Key, x => iconNameToSvg.GetValueOrDefault(x.Value) ?? string.Empty); + + var sourceDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + var iconDirectory = Path.Combine(sourceDirectory ?? string.Empty, "../../../entityicons"); + + var iconFiles = Directory.GetFiles(iconDirectory).ToDictionary(x => Path.GetFileNameWithoutExtension(x), x => x); + + foreach (var entity in entities) + { + if (logicalNameToSvg.ContainsKey(entity.LogicalName)) + { + continue; + } + + var iconKey = $"svg_{entity.ObjectTypeCode}"; + if (iconFiles.ContainsKey(iconKey)) + { + logicalNameToSvg[entity.LogicalName] = Convert.ToBase64String(File.ReadAllBytes(iconFiles[iconKey])); + } + } + + return logicalNameToSvg; + } + } +} diff --git a/Generator/Services/EntityMetadataService.cs b/Generator/Services/EntityMetadataService.cs new file mode 100644 index 0000000..266f33a --- /dev/null +++ b/Generator/Services/EntityMetadataService.cs @@ -0,0 +1,72 @@ +using Generator.Queries; +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk.Metadata; +using System.Collections.Concurrent; + +namespace Generator.Services +{ + /// + /// Service responsible for retrieving entity metadata from Dataverse + /// + internal class EntityMetadataService + { + private readonly ServiceClient client; + + public EntityMetadataService(ServiceClient client) + { + this.client = client; + } + + /// + /// Retrieves entity metadata by object IDs + /// + public async Task> GetEntityMetadataByObjectIds(List entityObjectIds) + { + ConcurrentBag metadata = new(); + + // Disable affinity cookie + client.EnableAffinityCookie = false; + + var parallelOptions = new ParallelOptions() + { + MaxDegreeOfParallelism = client.RecommendedDegreesOfParallelism + }; + + await Parallel.ForEachAsync( + source: entityObjectIds, + parallelOptions: parallelOptions, + async (objectId, token) => + { + metadata.Add(await client.RetrieveEntityAsync(objectId, token)); + }); + + return metadata; + } + + /// + /// Retrieves entity metadata by logical names + /// + public async Task> GetEntityMetadataByLogicalNames(List entityLogicalNames) + { + ConcurrentBag metadata = new(); + + // Disable affinity cookie + client.EnableAffinityCookie = false; + + var parallelOptions = new ParallelOptions() + { + MaxDegreeOfParallelism = client.RecommendedDegreesOfParallelism + }; + + await Parallel.ForEachAsync( + source: entityLogicalNames, + parallelOptions: parallelOptions, + async (logicalName, token) => + { + metadata.Add(await client.RetrieveEntityByLogicalNameAsync(logicalName, token)); + }); + + return metadata; + } + } +} diff --git a/Generator/Services/RecordMappingService.cs b/Generator/Services/RecordMappingService.cs new file mode 100644 index 0000000..aeead52 --- /dev/null +++ b/Generator/Services/RecordMappingService.cs @@ -0,0 +1,80 @@ +using Generator.DTO; +using Generator.DTO.Attributes; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Xrm.Sdk.Metadata; + +namespace Generator.Services +{ + /// + /// Service responsible for creating Record DTOs from entity metadata + /// Orchestrates attribute mapping, relationship mapping, and grouping logic + /// + internal class RecordMappingService + { + private readonly AttributeMappingService attributeMappingService; + private readonly RelationshipService relationshipService; + private readonly IConfiguration configuration; + private readonly ILogger logger; + private readonly ILogger relationshipLogger; + + public RecordMappingService( + AttributeMappingService attributeMappingService, + RelationshipService relationshipService, + IConfiguration configuration, + ILogger logger, + ILogger relationshipLogger) + { + this.attributeMappingService = attributeMappingService; + this.relationshipService = relationshipService; + this.configuration = configuration; + this.logger = logger; + this.relationshipLogger = relationshipLogger; + } + + /// + /// Creates a Record DTO from entity metadata + /// + public Record CreateRecord( + EntityMetadata entity, + List relevantAttributes, + List relevantManyToMany, + Dictionary logicalToSchema, + Dictionary> attributeLogicalToSchema, + List securityRoles, + List keys, + Dictionary entityIconMap, + Dictionary>> attributeUsages) + { + var attributes = + relevantAttributes + .Select(metadata => attributeMappingService.MapAttribute(metadata, entity, logicalToSchema, attributeUsages)) + .Where(x => !string.IsNullOrEmpty(x.DisplayName)) + .ToList(); + + var oneToMany = relationshipService.MapOneToManyRelationships(entity, logicalToSchema, attributeLogicalToSchema); + var manyToMany = relationshipService.MapManyToManyRelationships(entity, relevantManyToMany, logicalToSchema); + + var tablegroups = relationshipService.ParseTableGroups(relationshipLogger); + var (group, description) = relationshipService.GetGroupAndDescription(entity, tablegroups); + + entityIconMap.TryGetValue(entity.LogicalName, out string? iconBase64); + + return new Record( + entity.DisplayName.UserLocalizedLabel?.Label ?? string.Empty, + entity.SchemaName, + group, + description?.PrettyDescription(), + entity.IsAuditEnabled.Value, + entity.IsActivity ?? false, + entity.IsCustomEntity ?? false, + entity.OwnershipType ?? OwnershipTypes.UserOwned, + entity.HasNotes ?? false, + attributes, + oneToMany.Concat(manyToMany).ToList(), + securityRoles, + keys, + iconBase64); + } + } +} diff --git a/Generator/Services/RelationshipService.cs b/Generator/Services/RelationshipService.cs new file mode 100644 index 0000000..0bfb3f5 --- /dev/null +++ b/Generator/Services/RelationshipService.cs @@ -0,0 +1,133 @@ +using Generator.DTO; +using Generator.DTO.Attributes; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Xrm.Sdk.Metadata; + +namespace Generator.Services +{ + /// + /// Service responsible for mapping entity relationships + /// + internal class RelationshipService + { + private readonly IConfiguration configuration; + + public RelationshipService(IConfiguration configuration) + { + this.configuration = configuration; + } + + /// + /// Maps one-to-many relationships for an entity + /// + public List MapOneToManyRelationships( + EntityMetadata entity, + Dictionary logicalToSchema, + Dictionary> attributeLogicalToSchema) + { + return (entity.OneToManyRelationships ?? Enumerable.Empty()) + .Where(x => logicalToSchema.ContainsKey(x.ReferencingEntity) && logicalToSchema[x.ReferencingEntity].IsInSolution && attributeLogicalToSchema[x.ReferencingEntity].ContainsKey(x.ReferencingAttribute)) + .Select(x => new Relationship( + x.IsCustomRelationship ?? false, + x.ReferencingEntityNavigationPropertyName, + logicalToSchema[x.ReferencingEntity].Name, + attributeLogicalToSchema[x.ReferencingEntity][x.ReferencingAttribute], + x.SchemaName, + IsManyToMany: false, + x.CascadeConfiguration)) + .ToList(); + } + + /// + /// Maps many-to-many relationships for an entity + /// + public List MapManyToManyRelationships( + EntityMetadata entity, + List relevantManyToMany, + Dictionary logicalToSchema) + { + return relevantManyToMany + .Where(x => logicalToSchema.ContainsKey(x.Entity1LogicalName) && logicalToSchema[x.Entity1LogicalName].IsInSolution) + .Select(x => + { + var useEntity1 = x.Entity1LogicalName == entity.LogicalName; + + var label = !useEntity1 + ? x.Entity1AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity1NavigationPropertyName + : x.Entity2AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity2NavigationPropertyName; + + return new DTO.Relationship( + x.IsCustomRelationship ?? false, + label ?? x.SchemaName, // Fallback to schema name if no localized label is available, this is relevant for some Default/System Many to Many relationships. + logicalToSchema[!useEntity1 ? x.Entity1LogicalName : x.Entity2LogicalName].Name, + "-", + x.SchemaName, + IsManyToMany: true, + null + ); + }) + .ToList(); + } + + /// + /// Parses table groups from configuration + /// + public Dictionary ParseTableGroups(ILogger logger) + { + Dictionary tablegroups = []; // logicalname -> group + var tablegroupstring = configuration["TableGroups"]; + if (tablegroupstring?.Length > 0) + { + var groupEntries = tablegroupstring.Split(';', StringSplitOptions.RemoveEmptyEntries); + foreach (var g in groupEntries) + { + var tables = g.Split(':'); + if (tables.Length != 2) + { + logger.LogError($"Invalid format for tablegroup entry: ({g})"); + continue; + } + + var logicalNames = tables[1].Split(',', StringSplitOptions.RemoveEmptyEntries); + foreach (var logicalName in logicalNames) + if (!tablegroups.TryAdd(logicalName.Trim().ToLower(), tables[0].Trim())) + { + logger.LogWarning($"Dublicate logicalname detected: {logicalName} (already in tablegroup '{tablegroups[logicalName]}', dublicate found in group '{g}')"); + continue; + } + } + } + return tablegroups; + } + + /// + /// Extracts group and description from entity metadata + /// + public (string? Group, string? Description) GetGroupAndDescription(EntityMetadata entity, IDictionary tableGroups) + { + var description = entity.Description.UserLocalizedLabel?.Label ?? string.Empty; + if (!description.StartsWith("#")) + { + if (tableGroups.TryGetValue(entity.LogicalName, out var tablegroup)) + return (tablegroup, description); + return (null, description); + } + + var newlineIndex = description.IndexOf("\n"); + if (newlineIndex != -1) + { + var group = description.Substring(1, newlineIndex - 1).Trim(); + description = description.Substring(newlineIndex + 1); + return (group, description); + } + + var withoutHashtag = description.Substring(1).Trim(); + var firstSpace = withoutHashtag.IndexOf(" "); + if (firstSpace != -1) + return (withoutHashtag.Substring(0, firstSpace), withoutHashtag.Substring(firstSpace + 1)); + + return (withoutHashtag, null); + } + } +} diff --git a/Generator/Services/SecurityRoleService.cs b/Generator/Services/SecurityRoleService.cs new file mode 100644 index 0000000..834aadd --- /dev/null +++ b/Generator/Services/SecurityRoleService.cs @@ -0,0 +1,121 @@ +using Generator.DTO; +using Microsoft.Crm.Sdk.Messages; +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; + +namespace Generator.Services +{ + /// + /// Service responsible for querying and mapping security roles + /// + internal class SecurityRoleService + { + private readonly ServiceClient client; + + public SecurityRoleService(ServiceClient client) + { + this.client = client; + } + + /// + /// Retrieves and maps security roles with their privileges + /// + public async Task>> GetSecurityRoles( + List rolesInSolution, + Dictionary privileges) + { + if (rolesInSolution.Count == 0) return []; + + var query = new QueryExpression("role") + { + ColumnSet = new ColumnSet("name"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("roleid", ConditionOperator.In, rolesInSolution) + } + }, + LinkEntities = + { + new LinkEntity("role", "roleprivileges", "roleid", "roleid", JoinOperator.Inner) + { + EntityAlias = "rolepriv", + Columns = new ColumnSet("privilegedepthmask"), + LinkEntities = + { + new LinkEntity("roleprivileges", "privilege", "privilegeid", "privilegeid", JoinOperator.Inner) + { + EntityAlias = "priv", + Columns = new ColumnSet("accessright"), + LinkEntities = + { + new LinkEntity("privilege", "privilegeobjecttypecodes", "privilegeid", "privilegeid", JoinOperator.Inner) + { + EntityAlias = "privotc", + Columns = new ColumnSet("objecttypecode") + } + } + } + } + } + } + }; + + var roles = await client.RetrieveMultipleAsync(query); + + var rolePrivileges = roles.Entities.Select(e => + { + var name = e.GetAttributeValue("name"); + var depth = (PrivilegeDepth)e.GetAttributeValue("rolepriv.privilegedepthmask").Value; + var accessRight = (AccessRights)e.GetAttributeValue("priv.accessright").Value; + var objectTypeCode = e.GetAttributeValue("privotc.objecttypecode").Value as string; + + return new + { + name, + depth, + accessRight, + objectTypeCode = objectTypeCode ?? string.Empty + }; + }); + + static PrivilegeDepth? GetDepth(Dictionary dict, AccessRights right, SecurityPrivilegeMetadata? meta) + { + if (!dict.TryGetValue(right, out var value)) + return meta?.CanBeGlobal ?? false ? 0 : null; + return value; + } + + return rolePrivileges + .GroupBy(x => x.objectTypeCode) + .ToDictionary(byLogicalName => byLogicalName.Key, byLogicalName => + byLogicalName + .GroupBy(x => x.name) + .Select(byRole => + { + var accessRights = byRole + .GroupBy(x => x.accessRight) + .ToDictionary(x => x.Key, x => x.First().depth); + + var privilegeMetadata = privileges.GetValueOrDefault(byLogicalName.Key) ?? []; + + return new SecurityRole( + byRole.Key, + byLogicalName.Key, + GetDepth(accessRights, AccessRights.CreateAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Create)), + GetDepth(accessRights, AccessRights.ReadAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Read)), + GetDepth(accessRights, AccessRights.WriteAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Write)), + GetDepth(accessRights, AccessRights.DeleteAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Delete)), + GetDepth(accessRights, AccessRights.AppendAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Append)), + GetDepth(accessRights, AccessRights.AppendToAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.AppendTo)), + GetDepth(accessRights, AccessRights.AssignAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Assign)), + GetDepth(accessRights, AccessRights.ShareAccess, privilegeMetadata.FirstOrDefault(p => p.PrivilegeType == PrivilegeType.Share)) + ); + }) + .ToList()); + } + } +} diff --git a/Generator/Services/SolutionService.cs b/Generator/Services/SolutionService.cs new file mode 100644 index 0000000..2f6bb8e --- /dev/null +++ b/Generator/Services/SolutionService.cs @@ -0,0 +1,298 @@ +using Generator.DTO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; + +namespace Generator.Services +{ + /// + /// Service responsible for solution queries and component mapping + /// + internal class SolutionService + { + private readonly ServiceClient client; + private readonly IConfiguration configuration; + private readonly ILogger logger; + + public SolutionService(ServiceClient client, IConfiguration configuration, ILogger logger) + { + this.client = client; + this.configuration = configuration; + this.logger = logger; + } + + /// + /// Retrieves solution IDs based on configuration + /// + public async Task<(List SolutionIds, List SolutionEntities)> GetSolutionIds() + { + var solutionNameArg = configuration["DataverseSolutionNames"]; + if (solutionNameArg == null) + { + throw new Exception("Specify one or more solutions"); + } + var solutionNames = solutionNameArg.Split(",").Select(x => x.Trim().ToLower()).ToList(); + + var resp = await client.RetrieveMultipleAsync(new QueryExpression("solution") + { + ColumnSet = new ColumnSet("publisherid", "friendlyname", "uniquename", "solutionid"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("uniquename", ConditionOperator.In, solutionNames) + } + } + }); + + return (resp.Entities.Select(e => e.GetAttributeValue("solutionid")).ToList(), resp.Entities.ToList()); + } + + /// + /// Retrieves solution components for given solution IDs + /// + public async Task> GetSolutionComponents(List solutionIds) + { + var entityQuery = new QueryExpression("solutioncomponent") + { + ColumnSet = new ColumnSet("objectid", "componenttype", "rootcomponentbehavior", "solutionid"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("componenttype", ConditionOperator.In, new List() { 1, 2, 20, 29, 92 }), // entity, attribute, role, workflow/flow, sdkpluginstep (https://learn.microsoft.com/en-us/power-apps/developer/data-platform/reference/entities/solutioncomponent) + new ConditionExpression("solutionid", ConditionOperator.In, solutionIds) + } + } + }; + + return + (await client.RetrieveMultipleAsync(entityQuery)) + .Entities + .Select(e => (e.GetAttributeValue("objectid"), e.GetAttributeValue("componenttype").Value, e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, e.GetAttributeValue("solutionid"))) + .ToList(); + } + + /// + /// Creates Solution DTOs with their components + /// + public async Task> CreateSolutions( + List solutionEntities, + IEnumerable<(Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId)> solutionComponents, + List allEntityMetadata) + { + var solutions = new List(); + + // Create lookup dictionaries for faster access + var entityLookup = allEntityMetadata.ToDictionary(e => e.MetadataId ?? Guid.Empty, e => e); + + // Fetch all unique publishers for the solutions + var publisherIds = solutionEntities + .Select(s => s.GetAttributeValue("publisherid").Id) + .Distinct() + .ToList(); + + var publisherQuery = new QueryExpression("publisher") + { + ColumnSet = new ColumnSet("publisherid", "friendlyname", "customizationprefix"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("publisherid", ConditionOperator.In, publisherIds) + } + } + }; + + var publishers = await client.RetrieveMultipleAsync(publisherQuery); + var publisherLookup = publishers.Entities.ToDictionary( + p => p.GetAttributeValue("publisherid"), + p => ( + Name: p.GetAttributeValue("friendlyname") ?? "Unknown Publisher", + Prefix: p.GetAttributeValue("customizationprefix") ?? string.Empty + )); + + // Group components by solution + var componentsBySolution = solutionComponents.GroupBy(c => c.SolutionId).ToDictionary(g => g.Key.Id, g => g); + + // Process ALL solutions from configuration, not just those with components + foreach (var solutionEntity in solutionEntities) + { + var solutionId = solutionEntity.GetAttributeValue("solutionid"); + + var solutionName = solutionEntity.GetAttributeValue("friendlyname") ?? + solutionEntity.GetAttributeValue("uniquename") ?? + "Unknown Solution"; + + var publisherId = solutionEntity.GetAttributeValue("publisherid").Id; + var publisher = publisherLookup.GetValueOrDefault(publisherId); + + var components = new List(); + + // Add components if this solution has any + if (componentsBySolution.TryGetValue(solutionId, out var solutionGroup)) + { + foreach (var component in solutionGroup) + { + var solutionComponent = CreateSolutionComponent(component, entityLookup, allEntityMetadata, publisherLookup); + if (solutionComponent != null) + { + components.Add(solutionComponent); + } + } + } + + // Add solution even if components list is empty (e.g., flow-only solutions) + solutions.Add(new Solution( + solutionName, + publisher.Name, + publisher.Prefix, + components)); + } + + return solutions.AsEnumerable(); + } + + /// + /// Creates a solution component DTO from component metadata + /// + private SolutionComponent? CreateSolutionComponent( + (Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId) component, + Dictionary entityLookup, + List allEntityMetadata, + Dictionary publisherLookup) + { + try + { + switch (component.ComponentType) + { + case 1: // Entity + // Try to find entity by MetadataId first, then by searching all entities + if (entityLookup.TryGetValue(component.ObjectId, out var entityMetadata)) + { + var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(entityMetadata.SchemaName, publisherLookup); + return new SolutionComponent( + entityMetadata.DisplayName?.UserLocalizedLabel?.Label ?? entityMetadata.SchemaName, + entityMetadata.SchemaName, + entityMetadata.Description?.UserLocalizedLabel?.Label ?? string.Empty, + SolutionComponentType.Entity, + publisherName, + publisherPrefix); + } + + // Entity lookup by ObjectId is complex in Dataverse, so we'll skip the fallback for now + // The primary lookup by MetadataId should handle most cases + break; + + case 2: // Attribute + // Search for attribute across all entities + foreach (var entity in allEntityMetadata) + { + var attribute = entity.Attributes?.FirstOrDefault(a => a.MetadataId == component.ObjectId); + if (attribute != null) + { + var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(attribute.SchemaName, publisherLookup); + return new SolutionComponent( + attribute.DisplayName?.UserLocalizedLabel?.Label ?? attribute.SchemaName, + attribute.SchemaName, + attribute.Description?.UserLocalizedLabel?.Label ?? string.Empty, + SolutionComponentType.Attribute, + publisherName, + publisherPrefix); + } + } + break; + + case 3: // Relationship (if you want to add this to the enum later) + // Search for relationships across all entities + foreach (var entity in allEntityMetadata) + { + // Check one-to-many relationships + var oneToMany = entity.OneToManyRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); + if (oneToMany != null) + { + var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(oneToMany.SchemaName, publisherLookup); + return new SolutionComponent( + oneToMany.SchemaName, + oneToMany.SchemaName, + $"One-to-Many: {entity.SchemaName} -> {oneToMany.ReferencingEntity}", + SolutionComponentType.Relationship, + publisherName, + publisherPrefix); + } + + // Check many-to-one relationships + var manyToOne = entity.ManyToOneRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); + if (manyToOne != null) + { + var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(manyToOne.SchemaName, publisherLookup); + return new SolutionComponent( + manyToOne.SchemaName, + manyToOne.SchemaName, + $"Many-to-One: {entity.SchemaName} -> {manyToOne.ReferencedEntity}", + SolutionComponentType.Relationship, + publisherName, + publisherPrefix); + } + + // Check many-to-many relationships + var manyToMany = entity.ManyToManyRelationships?.FirstOrDefault(r => r.MetadataId == component.ObjectId); + if (manyToMany != null) + { + var (publisherName, publisherPrefix) = GetPublisherFromSchemaName(manyToMany.SchemaName, publisherLookup); + return new SolutionComponent( + manyToMany.SchemaName, + manyToMany.SchemaName, + $"Many-to-Many: {manyToMany.Entity1LogicalName} <-> {manyToMany.Entity2LogicalName}", + SolutionComponentType.Relationship, + publisherName, + publisherPrefix); + } + } + break; + + case 20: // Security Role - skip for now as not in enum + case 92: // SDK Message Processing Step (Plugin) - skip for now as not in enum + break; + } + } + catch (Exception ex) + { + logger.LogWarning($"Failed to create solution component for ObjectId {component.ObjectId}, ComponentType {component.ComponentType}: {ex.Message}"); + } + + return null; + } + + /// + /// Extracts publisher information from schema name + /// + private static (string PublisherName, string PublisherPrefix) GetPublisherFromSchemaName( + string schemaName, + Dictionary publisherLookup) + { + // Extract prefix from schema name (e.g., "contoso_entity" -> "contoso") + var parts = schemaName.Split('_', 2); + + if (parts.Length == 2) + { + var prefix = parts[0]; + + // Find publisher by matching prefix + foreach (var publisher in publisherLookup.Values) + { + if (publisher.Prefix.Equals(prefix, StringComparison.OrdinalIgnoreCase)) + { + return (publisher.Name, publisher.Prefix); + } + } + } + + // Default to Microsoft if no prefix or prefix not found + return ("Microsoft", ""); + } + } +} From 9492036f22f1874e87c2bd21bf1c6f665dde9e08 Mon Sep 17 00:00:00 2001 From: Lucki2g Date: Sat, 22 Nov 2025 00:34:05 +0100 Subject: [PATCH 04/23] feat: new generator logic to show inclusion type and visually the same as the make portal --- Generator/DTO/Attributes/Attribute.cs | 4 +- Generator/DTO/Relationship.cs | 4 +- Generator/DataverseService.cs | 198 +++++---- .../RelationshipExtensions.cs | 34 ++ Generator/Program.cs | 3 +- Generator/Services/AttributeMappingService.cs | 4 +- Generator/Services/EntityMetadataService.cs | 20 +- Generator/Services/RecordMappingService.cs | 21 +- Generator/Services/RelationshipService.cs | 167 +++----- .../Services/SolutionComponentService.cs | 380 ++++++++++++++++++ Generator/Services/SolutionService.cs | 35 +- .../datamodelview/Relationships.tsx | 38 +- Website/lib/Types.ts | 29 +- 13 files changed, 668 insertions(+), 269 deletions(-) create mode 100644 Generator/ExtensionMethods/RelationshipExtensions.cs create mode 100644 Generator/Services/SolutionComponentService.cs diff --git a/Generator/DTO/Attributes/Attribute.cs b/Generator/DTO/Attributes/Attribute.cs index 221ce63..6ebed81 100644 --- a/Generator/DTO/Attributes/Attribute.cs +++ b/Generator/DTO/Attributes/Attribute.cs @@ -1,4 +1,5 @@ -using Microsoft.Xrm.Sdk.Metadata; +using Generator.Services; +using Microsoft.Xrm.Sdk.Metadata; namespace Generator.DTO.Attributes; @@ -9,6 +10,7 @@ public abstract class Attribute public bool IsPrimaryId { get; set; } public bool IsPrimaryName { get; set; } public List AttributeUsages { get; set; } = new List(); + public ComponentInclusionType InclusionType { get; set; } public string DisplayName { get; } public string SchemaName { get; } public string Description { get; } diff --git a/Generator/DTO/Relationship.cs b/Generator/DTO/Relationship.cs index c26986a..2a2d356 100644 --- a/Generator/DTO/Relationship.cs +++ b/Generator/DTO/Relationship.cs @@ -1,4 +1,5 @@ -using Microsoft.Xrm.Sdk.Metadata; +using Generator.Services; +using Microsoft.Xrm.Sdk.Metadata; namespace Generator.DTO; @@ -10,4 +11,5 @@ public record Relationship( string LookupDisplayName, string RelationshipSchema, bool IsManyToMany, + ComponentInclusionType InclusionType, CascadeConfiguration? CascadeConfiguration); diff --git a/Generator/DataverseService.cs b/Generator/DataverseService.cs index c37c882..42c8fe8 100644 --- a/Generator/DataverseService.cs +++ b/Generator/DataverseService.cs @@ -1,12 +1,12 @@ using Generator.DTO; using Generator.DTO.Attributes; using Generator.DTO.Warnings; +using Generator.ExtensionMethods; using Generator.Queries; using Generator.Services; using Generator.Services.Plugins; using Generator.Services.PowerAutomate; using Generator.Services.WebResources; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk.Metadata; @@ -16,29 +16,26 @@ namespace Generator { internal class DataverseService { - private readonly ServiceClient client; - private readonly IConfiguration configuration; private readonly ILogger logger; private readonly EntityMetadataService entityMetadataService; private readonly SolutionService solutionService; private readonly SecurityRoleService securityRoleService; private readonly EntityIconService entityIconService; private readonly RecordMappingService recordMappingService; + private readonly SolutionComponentService solutionComponentService; private readonly List analyzerRegistrations; public DataverseService( ServiceClient client, - IConfiguration configuration, ILogger logger, EntityMetadataService entityMetadataService, SolutionService solutionService, SecurityRoleService securityRoleService, EntityIconService entityIconService, - RecordMappingService recordMappingService) + RecordMappingService recordMappingService, + SolutionComponentService solutionComponentService) { - this.client = client; - this.configuration = configuration; this.logger = logger; this.entityMetadataService = entityMetadataService; this.solutionService = solutionService; @@ -62,42 +59,49 @@ public DataverseService( solutionIds => client.GetWebResourcesAsync(solutionIds), "WebResources") }; + this.solutionComponentService = solutionComponentService; } public async Task<(IEnumerable, IEnumerable, IEnumerable)> GetFilteredMetadata() { - var warnings = new List(); // used to collect warnings for the insights dashboard + // used to collect warnings for the insights dashboard + var warnings = new List(); var (solutionIds, solutionEntities) = await solutionService.GetSolutionIds(); - var solutionComponents = await solutionService.GetSolutionComponents(solutionIds); // (id, type, rootcomponentbehavior, solutionid) - var entitiesInSolution = solutionComponents.Where(x => x.ComponentType == 1).Select(x => x.ObjectId).Distinct().ToList(); - var entityRootBehaviour = solutionComponents - .Where(x => x.ComponentType == 1) - .GroupBy(x => x.ObjectId) - .ToDictionary(g => g.Key, g => - { - // If any solution includes all attributes (0), use that, otherwise use the first occurrence - var behaviors = g.Select(x => x.RootComponentBehavior).ToList(); - return behaviors.Contains(0) ? 0 : behaviors.First(); - }); - var attributesInSolution = solutionComponents.Where(x => x.ComponentType == 2).Select(x => x.ObjectId).ToHashSet(); - var rolesInSolution = solutionComponents.Where(x => x.ComponentType == 20).Select(x => x.ObjectId).Distinct().ToList(); - - var entitiesInSolutionMetadata = await entityMetadataService.GetEntityMetadataByObjectIds(entitiesInSolution); - - var logicalNameToKeys = entitiesInSolutionMetadata.ToDictionary( - entity => entity.LogicalName, - entity => entity.Keys.Select(key => new Key( - key.DisplayName.UserLocalizedLabel?.Label ?? key.DisplayName.LocalizedLabels.First().Label, - key.LogicalName, - key.KeyAttributes) - ).ToList()); + /// SOLUTIONS + IEnumerable solutionComponents; + try + { + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponents()"); + solutionComponents = solutionComponentService.GetAllSolutionComponents(solutionIds); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {solutionComponents.Count()} solution components"); + } + catch (Exception ex) + { + logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to get solution components"); + throw; + } + var inclusionMap = solutionComponents.ToDictionary(s => s.ObjectId, s => s.InclusionType); - var logicalNameToSecurityRoles = await securityRoleService.GetSecurityRoles(rolesInSolution, entitiesInSolutionMetadata.ToDictionary(x => x.LogicalName, x => x.Privileges)); + /// ENTITIES + var set = solutionComponents.Select(c => c.ObjectId).ToHashSet(); + IEnumerable entitiesInSolutionMetadata; + IEnumerable entitiesInSolution; + try + { + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling entityMetadataService.GetEntityMetadataByObjectIds()"); + entitiesInSolution = solutionComponents.Where(c => c.ComponentType is 1).DistinctBy(comp => comp.ObjectId); + entitiesInSolutionMetadata = await entityMetadataService.GetEntityMetadataByObjectIds(entitiesInSolution.Select(e => e.ObjectId)); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {entitiesInSolutionMetadata.Count()} entity metadata"); + } + catch (Exception ex) + { + logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to get entity metadata"); + throw; + } var entityLogicalNamesInSolution = entitiesInSolutionMetadata.Select(e => e.LogicalName).ToHashSet(); - - logger.LogInformation("There are {Count} entities in the solution.", entityLogicalNamesInSolution.Count); - // Collect all referenced entities from attributes and add (needed for lookup attributes) + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {entityLogicalNamesInSolution.Count} unique entities"); + var entityIconMap = await entityIconService.GetEntityIconMap(entitiesInSolutionMetadata); var relatedEntityLogicalNames = new HashSet(); foreach (var entity in entitiesInSolutionMetadata) { @@ -110,69 +114,72 @@ public DataverseService( } logger.LogInformation("There are {Count} entities referenced outside the solution.", relatedEntityLogicalNames.Count); var referencedEntityMetadata = await entityMetadataService.GetEntityMetadataByLogicalNames(relatedEntityLogicalNames.ToList()); - var allEntityMetadata = entitiesInSolutionMetadata.Concat(referencedEntityMetadata).ToList(); var logicalToSchema = allEntityMetadata.ToDictionary(x => x.LogicalName, x => new ExtendedEntityInformation { Name = x.SchemaName, IsInSolution = entitiesInSolutionMetadata.Any(e => e.LogicalName == x.LogicalName) }); + + /// SECURITY ROLES + var rolesInSolution = solutionComponents.Where(x => x.ComponentType == 20).Select(x => x.ObjectId).Distinct().ToList(); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {rolesInSolution.Count} roles"); + var logicalNameToSecurityRoles = await securityRoleService.GetSecurityRoles(rolesInSolution, entitiesInSolutionMetadata.ToDictionary(x => x.LogicalName, x => x.Privileges)); + + /// ATTRIBUTES + var attributesInSolution = solutionComponents.Where(x => x.ComponentType == 2).Select(x => x.ObjectId).ToHashSet(); + var rootBehaviourEntities = entitiesInSolution.Where(ent => ent.RootComponentBehaviour is 0).Select(e => e.ObjectId).ToHashSet(); + var attributesAllExplicitlyAdded = entitiesInSolutionMetadata.Where(e => rootBehaviourEntities.Contains(e.MetadataId!.Value)).SelectMany(e => e.Attributes.Select(a => a.MetadataId!.Value)); + foreach (var attr in attributesAllExplicitlyAdded) attributesInSolution.Add(attr); + + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {attributesInSolution.Count} attributes"); var attributeLogicalToSchema = allEntityMetadata.ToDictionary(x => x.LogicalName, x => x.Attributes?.ToDictionary(attr => attr.LogicalName, attr => attr.DisplayName.UserLocalizedLabel?.Label ?? attr.SchemaName) ?? []); - var entityIconMap = await entityIconService.GetEntityIconMap(allEntityMetadata); - // Processes analysis - var attributeUsages = new Dictionary>>(); + /// ENTITY RELATIONSHIPS + var relationshipsInSolution = solutionComponents.Where(x => x.ComponentType == 10).Select(x => x.ObjectId).ToHashSet(); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {relationshipsInSolution.Count} relations"); - // Run all registered analyzers, passing entity metadata + /// KEYS + var logicalNameToKeys = entitiesInSolutionMetadata.ToDictionary( + entity => entity.LogicalName, + entity => entity.Keys.Select(key => new Key( + key.DisplayName.UserLocalizedLabel?.Label ?? key.DisplayName.LocalizedLabels.First().Label, + key.LogicalName, + key.KeyAttributes) + ).ToList()); + + /// PROCESS ANALYSERS + var attributeUsages = new Dictionary>>(); foreach (var registration in analyzerRegistrations) - { await registration.RunAnalysisAsync(solutionIds, attributeUsages, warnings, logger, entitiesInSolutionMetadata.ToList()); - } + var records = entitiesInSolutionMetadata - .Select(x => new + .Select(entMeta => { - EntityMetadata = x, - RelevantAttributes = - x.GetRelevantAttributes(attributesInSolution, entityRootBehaviour) - .Where(x => x.DisplayName.UserLocalizedLabel?.Label != null) - .ToList(), - RelevantManyToMany = - x.ManyToManyRelationships - .Where(r => entityLogicalNamesInSolution.Contains(r.Entity1LogicalName) && entityLogicalNamesInSolution.Contains(r.Entity2LogicalName)) - .ToList(), - }) - .Where(x => x.EntityMetadata.DisplayName.UserLocalizedLabel?.Label != null) - .ToList(); - - // Warn about attributes that were used in processes, but the entity could not be resolved from e.g. JavaScript file name or similar - var hash = entitiesInSolutionMetadata.SelectMany(r => [r.LogicalCollectionName?.ToLower() ?? "", r.LogicalName.ToLower()]).ToHashSet(); - warnings.AddRange(attributeUsages.Keys - .Where(k => !hash.Contains(k.ToLower())) - .SelectMany(entityKey => attributeUsages.GetValueOrDefault(entityKey)! - .SelectMany(attributeDict => attributeDict.Value - .Select(usage => - new AttributeWarning($"{attributeDict.Key} was used inside a {usage.ComponentType} component [{usage.Name}]. However, the entity {entityKey} could not be resolved in the provided solutions."))))); + var relevantAttributes = entMeta.Attributes.Where(attr => attributesInSolution.Contains(attr.MetadataId!.Value)).ToList(); + var relevantManyToManyRelations = entMeta.ManyToManyRelationships.Where(rel => relationshipsInSolution.Contains(rel.MetadataId!.Value)).ConvertToRelationship(entMeta.LogicalName, inclusionMap); + var relevantOneToManyRelations = entMeta.OneToManyRelationships.Where(rel => relationshipsInSolution.Contains(rel.MetadataId!.Value)).ConvertToRelationship(entMeta.LogicalName, attributeLogicalToSchema, inclusionMap); + var relevantManyToOneRelations = entMeta.ManyToOneRelationships.Where(rel => relationshipsInSolution.Contains(rel.MetadataId!.Value)).ConvertToRelationship(entMeta.LogicalName, attributeLogicalToSchema, inclusionMap); + var relevantRelationships = relevantManyToManyRelations.Concat(relevantManyToOneRelations).Concat(relevantOneToManyRelations).ToList(); - // Create solutions with their components - var solutions = await solutionService.CreateSolutions(solutionEntities, solutionComponents, allEntityMetadata); - - return (records - .Select(x => - { - logicalNameToSecurityRoles.TryGetValue(x.EntityMetadata.LogicalName, out var securityRoles); - logicalNameToKeys.TryGetValue(x.EntityMetadata.LogicalName, out var keys); + logicalNameToSecurityRoles.TryGetValue(entMeta.LogicalName, out var securityRoles); + logicalNameToKeys.TryGetValue(entMeta.LogicalName, out var keys); return recordMappingService.CreateRecord( - x.EntityMetadata, - x.RelevantAttributes, - x.RelevantManyToMany, + entMeta, + relevantAttributes, + relevantRelationships, logicalToSchema, - attributeLogicalToSchema, securityRoles ?? [], keys ?? [], entityIconMap, - attributeUsages); - }), - warnings, - solutions); + attributeUsages, + inclusionMap); + }) + .ToList(); + + var solutions = await solutionService.CreateSolutions(solutionEntities, solutionComponents, entitiesInSolutionMetadata); + + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] GetFilteredMetadata completed - returning empty results"); + return (records, warnings, solutions); } } @@ -215,20 +222,41 @@ public async Task RunAnalysisAsync( ILogger logger, List entityMetadata) { + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Starting {componentTypeName} analysis"); var stopwatch = Stopwatch.StartNew(); - var components = await queryFunc(solutionIds); - var componentList = components.ToList(); + IEnumerable components; + try + { + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Querying {componentTypeName} from Dataverse"); + components = await queryFunc(solutionIds); + } + catch (Exception ex) + { + logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to query {componentTypeName}"); + throw; + } - logger.LogInformation($"There are {componentList.Count} {componentTypeName} in the environment."); + var componentList = components.ToList(); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] There are {componentList.Count} {componentTypeName} in the environment."); + int processedCount = 0; foreach (var component in componentList) { - await analyzer.AnalyzeComponentAsync(component, attributeUsages, warnings, entityMetadata); + try + { + await analyzer.AnalyzeComponentAsync(component, attributeUsages, warnings, entityMetadata); + processedCount++; + } + catch (Exception ex) + { + logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to analyze {componentTypeName} component (processed {processedCount}/{componentList.Count})"); + // Continue with next component instead of throwing + } } stopwatch.Stop(); - logger.LogInformation($"{componentTypeName} analysis took {stopwatch.ElapsedMilliseconds} ms."); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] {componentTypeName} analysis completed - processed {processedCount}/{componentList.Count} components in {stopwatch.ElapsedMilliseconds} ms"); } } } diff --git a/Generator/ExtensionMethods/RelationshipExtensions.cs b/Generator/ExtensionMethods/RelationshipExtensions.cs new file mode 100644 index 0000000..7fe8acb --- /dev/null +++ b/Generator/ExtensionMethods/RelationshipExtensions.cs @@ -0,0 +1,34 @@ +using Generator.DTO; +using Generator.Services; +using Microsoft.Xrm.Sdk.Metadata; + +namespace Generator.ExtensionMethods; + +public static class RelationshipExtensions +{ + public static IEnumerable ConvertToRelationship(this IEnumerable relationships, string entityLogicalName, Dictionary inclusionMap) + { + return relationships.Select(rel => new Relationship( + rel.IsCustomRelationship ?? false, + $"{rel.Entity1AssociatedMenuConfiguration.Label.UserLocalizedLabel.Label} ⟷ {rel.Entity2AssociatedMenuConfiguration.Label.UserLocalizedLabel.Label}", + entityLogicalName, + "-", + rel.SchemaName, + rel.RelationshipType is RelationshipType.ManyToManyRelationship, + inclusionMap[rel.MetadataId!.Value], + null)); + } + + public static IEnumerable ConvertToRelationship(this IEnumerable relationships, string entityLogicalName, Dictionary> attributeMapping, Dictionary inclusionMap) + { + return relationships.Select(rel => new Relationship( + rel.IsCustomRelationship ?? false, + rel.ReferencingEntityNavigationPropertyName ?? rel.ReferencedEntity, + entityLogicalName, + attributeMapping[rel.ReferencingEntity][rel.ReferencingAttribute], + rel.SchemaName, + rel.RelationshipType is not RelationshipType.ManyToManyRelationship, + inclusionMap[rel.MetadataId!.Value], + rel.CascadeConfiguration)); + } +} diff --git a/Generator/Program.cs b/Generator/Program.cs index ee0508d..4a3c1c7 100644 --- a/Generator/Program.cs +++ b/Generator/Program.cs @@ -13,7 +13,7 @@ .AddEnvironmentVariables() .AddJsonFile("appsettings.local.json", optional: true) .Build(); -var verbose = configuration.GetValue("Verbosity", LogLevel.Information); +var verbose = configuration.GetValue("Verbosity", LogLevel.Warning); // Set up dependency injection var services = new ServiceCollection(); @@ -57,6 +57,7 @@ services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); +services.AddSingleton(); // Build service provider var serviceProvider = services.BuildServiceProvider(); diff --git a/Generator/Services/AttributeMappingService.cs b/Generator/Services/AttributeMappingService.cs index 7c93d3f..cae2d5d 100644 --- a/Generator/Services/AttributeMappingService.cs +++ b/Generator/Services/AttributeMappingService.cs @@ -25,7 +25,8 @@ public Attribute MapAttribute( AttributeMetadata metadata, EntityMetadata entity, Dictionary logicalToSchema, - Dictionary>> attributeUsages) + Dictionary>> attributeUsages, + Dictionary inclusionMap) { Attribute attr = metadata switch { @@ -50,6 +51,7 @@ public Attribute MapAttribute( var pluralname = attributeUsages.GetValueOrDefault(entity.LogicalCollectionName)?.GetValueOrDefault(metadata.LogicalName) ?? []; attr.AttributeUsages = [.. schemaname, .. pluralname]; + attr.InclusionType = inclusionMap.GetValueOrDefault(metadata.MetadataId!.Value, ComponentInclusionType.Implicit); attr.IsStandardFieldModified = MetadataExtensions.StandardFieldHasChanged(metadata, entity.DisplayName.UserLocalizedLabel?.Label ?? string.Empty, entity.IsCustomEntity ?? false); return attr; diff --git a/Generator/Services/EntityMetadataService.cs b/Generator/Services/EntityMetadataService.cs index 266f33a..761f27d 100644 --- a/Generator/Services/EntityMetadataService.cs +++ b/Generator/Services/EntityMetadataService.cs @@ -1,4 +1,3 @@ -using Generator.Queries; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk.Metadata; using System.Collections.Concurrent; @@ -20,9 +19,10 @@ public EntityMetadataService(ServiceClient client) /// /// Retrieves entity metadata by object IDs /// - public async Task> GetEntityMetadataByObjectIds(List entityObjectIds) + public async Task> GetEntityMetadataByObjectIds(IEnumerable entityObjectIds) { ConcurrentBag metadata = new(); + ConcurrentBag failedIds = new(); // Disable affinity cookie client.EnableAffinityCookie = false; @@ -37,9 +37,23 @@ await Parallel.ForEachAsync( parallelOptions: parallelOptions, async (objectId, token) => { - metadata.Add(await client.RetrieveEntityAsync(objectId, token)); + try + { + metadata.Add(await client.RetrieveEntityAsync(objectId, token)); + } + catch (Exception ex) + { + // Entity doesn't exist or cannot be retrieved - log and continue + Console.WriteLine($"Warning: Failed to retrieve entity with ID {objectId}: {ex.Message}"); + failedIds.Add(objectId); + } }); + if (failedIds.Any()) + { + Console.WriteLine($"Warning: Failed to retrieve {failedIds.Count} entities out of {entityObjectIds.Count()}. IDs: {string.Join(", ", failedIds)}"); + } + return metadata; } diff --git a/Generator/Services/RecordMappingService.cs b/Generator/Services/RecordMappingService.cs index aeead52..aaece8d 100644 --- a/Generator/Services/RecordMappingService.cs +++ b/Generator/Services/RecordMappingService.cs @@ -14,9 +14,6 @@ internal class RecordMappingService { private readonly AttributeMappingService attributeMappingService; private readonly RelationshipService relationshipService; - private readonly IConfiguration configuration; - private readonly ILogger logger; - private readonly ILogger relationshipLogger; public RecordMappingService( AttributeMappingService attributeMappingService, @@ -27,9 +24,6 @@ public RecordMappingService( { this.attributeMappingService = attributeMappingService; this.relationshipService = relationshipService; - this.configuration = configuration; - this.logger = logger; - this.relationshipLogger = relationshipLogger; } /// @@ -38,24 +32,21 @@ public RecordMappingService( public Record CreateRecord( EntityMetadata entity, List relevantAttributes, - List relevantManyToMany, + List relevantRelationships, Dictionary logicalToSchema, - Dictionary> attributeLogicalToSchema, List securityRoles, List keys, Dictionary entityIconMap, - Dictionary>> attributeUsages) + Dictionary>> attributeUsages, + Dictionary inclusionMap) { var attributes = relevantAttributes - .Select(metadata => attributeMappingService.MapAttribute(metadata, entity, logicalToSchema, attributeUsages)) + .Select(metadata => attributeMappingService.MapAttribute(metadata, entity, logicalToSchema, attributeUsages, inclusionMap)) .Where(x => !string.IsNullOrEmpty(x.DisplayName)) .ToList(); - var oneToMany = relationshipService.MapOneToManyRelationships(entity, logicalToSchema, attributeLogicalToSchema); - var manyToMany = relationshipService.MapManyToManyRelationships(entity, relevantManyToMany, logicalToSchema); - - var tablegroups = relationshipService.ParseTableGroups(relationshipLogger); + var tablegroups = relationshipService.ParseTableGroups(); var (group, description) = relationshipService.GetGroupAndDescription(entity, tablegroups); entityIconMap.TryGetValue(entity.LogicalName, out string? iconBase64); @@ -71,7 +62,7 @@ public Record CreateRecord( entity.OwnershipType ?? OwnershipTypes.UserOwned, entity.HasNotes ?? false, attributes, - oneToMany.Concat(manyToMany).ToList(), + relevantRelationships, securityRoles, keys, iconBase64); diff --git a/Generator/Services/RelationshipService.cs b/Generator/Services/RelationshipService.cs index 0bfb3f5..e9d5553 100644 --- a/Generator/Services/RelationshipService.cs +++ b/Generator/Services/RelationshipService.cs @@ -1,133 +1,80 @@ -using Generator.DTO; -using Generator.DTO.Attributes; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Xrm.Sdk.Metadata; -namespace Generator.Services +namespace Generator.Services; + +/// +/// Service responsible for mapping entity relationships +/// +internal class RelationshipService { + private readonly IConfiguration configuration; + private readonly ILogger logger; + + public RelationshipService(IConfiguration configuration, ILogger logger) + { + this.configuration = configuration; + this.logger = logger; + } + /// - /// Service responsible for mapping entity relationships + /// Parses table groups from configuration /// - internal class RelationshipService + public Dictionary ParseTableGroups() { - private readonly IConfiguration configuration; - - public RelationshipService(IConfiguration configuration) - { - this.configuration = configuration; - } - - /// - /// Maps one-to-many relationships for an entity - /// - public List MapOneToManyRelationships( - EntityMetadata entity, - Dictionary logicalToSchema, - Dictionary> attributeLogicalToSchema) - { - return (entity.OneToManyRelationships ?? Enumerable.Empty()) - .Where(x => logicalToSchema.ContainsKey(x.ReferencingEntity) && logicalToSchema[x.ReferencingEntity].IsInSolution && attributeLogicalToSchema[x.ReferencingEntity].ContainsKey(x.ReferencingAttribute)) - .Select(x => new Relationship( - x.IsCustomRelationship ?? false, - x.ReferencingEntityNavigationPropertyName, - logicalToSchema[x.ReferencingEntity].Name, - attributeLogicalToSchema[x.ReferencingEntity][x.ReferencingAttribute], - x.SchemaName, - IsManyToMany: false, - x.CascadeConfiguration)) - .ToList(); - } - - /// - /// Maps many-to-many relationships for an entity - /// - public List MapManyToManyRelationships( - EntityMetadata entity, - List relevantManyToMany, - Dictionary logicalToSchema) - { - return relevantManyToMany - .Where(x => logicalToSchema.ContainsKey(x.Entity1LogicalName) && logicalToSchema[x.Entity1LogicalName].IsInSolution) - .Select(x => - { - var useEntity1 = x.Entity1LogicalName == entity.LogicalName; - - var label = !useEntity1 - ? x.Entity1AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity1NavigationPropertyName - : x.Entity2AssociatedMenuConfiguration.Label.UserLocalizedLabel?.Label ?? x.Entity2NavigationPropertyName; - - return new DTO.Relationship( - x.IsCustomRelationship ?? false, - label ?? x.SchemaName, // Fallback to schema name if no localized label is available, this is relevant for some Default/System Many to Many relationships. - logicalToSchema[!useEntity1 ? x.Entity1LogicalName : x.Entity2LogicalName].Name, - "-", - x.SchemaName, - IsManyToMany: true, - null - ); - }) - .ToList(); - } - - /// - /// Parses table groups from configuration - /// - public Dictionary ParseTableGroups(ILogger logger) + Dictionary tablegroups = []; // logicalname -> group + var tablegroupstring = configuration["TableGroups"]; + if (tablegroupstring?.Length > 0) { - Dictionary tablegroups = []; // logicalname -> group - var tablegroupstring = configuration["TableGroups"]; - if (tablegroupstring?.Length > 0) + var groupEntries = tablegroupstring.Split(';', StringSplitOptions.RemoveEmptyEntries); + foreach (var g in groupEntries) { - var groupEntries = tablegroupstring.Split(';', StringSplitOptions.RemoveEmptyEntries); - foreach (var g in groupEntries) + var tables = g.Split(':'); + if (tables.Length != 2) { - var tables = g.Split(':'); - if (tables.Length != 2) + logger.LogError($"Invalid format for tablegroup entry: ({g})"); + continue; + } + + var logicalNames = tables[1].Split(',', StringSplitOptions.RemoveEmptyEntries); + foreach (var logicalName in logicalNames) + if (!tablegroups.TryAdd(logicalName.Trim().ToLower(), tables[0].Trim())) { - logger.LogError($"Invalid format for tablegroup entry: ({g})"); + logger.LogWarning($"Dublicate logicalname detected: {logicalName} (already in tablegroup '{tablegroups[logicalName]}', dublicate found in group '{g}')"); continue; } - - var logicalNames = tables[1].Split(',', StringSplitOptions.RemoveEmptyEntries); - foreach (var logicalName in logicalNames) - if (!tablegroups.TryAdd(logicalName.Trim().ToLower(), tables[0].Trim())) - { - logger.LogWarning($"Dublicate logicalname detected: {logicalName} (already in tablegroup '{tablegroups[logicalName]}', dublicate found in group '{g}')"); - continue; - } - } } - return tablegroups; } + return tablegroups; + } - /// - /// Extracts group and description from entity metadata - /// - public (string? Group, string? Description) GetGroupAndDescription(EntityMetadata entity, IDictionary tableGroups) + /// + /// Extracts group and description from entity metadata + /// + public (string? Group, string? Description) GetGroupAndDescription(EntityMetadata entity, IDictionary tableGroups) + { + var description = entity.Description.UserLocalizedLabel?.Label ?? string.Empty; + if (!description.StartsWith("#")) { - var description = entity.Description.UserLocalizedLabel?.Label ?? string.Empty; - if (!description.StartsWith("#")) - { - if (tableGroups.TryGetValue(entity.LogicalName, out var tablegroup)) - return (tablegroup, description); - return (null, description); - } + if (tableGroups.TryGetValue(entity.LogicalName, out var tablegroup)) + return (tablegroup, description); + return (null, description); + } - var newlineIndex = description.IndexOf("\n"); - if (newlineIndex != -1) - { - var group = description.Substring(1, newlineIndex - 1).Trim(); - description = description.Substring(newlineIndex + 1); - return (group, description); - } + var newlineIndex = description.IndexOf("\n"); + if (newlineIndex != -1) + { + var group = description.Substring(1, newlineIndex - 1).Trim(); + description = description.Substring(newlineIndex + 1); + return (group, description); + } - var withoutHashtag = description.Substring(1).Trim(); - var firstSpace = withoutHashtag.IndexOf(" "); - if (firstSpace != -1) - return (withoutHashtag.Substring(0, firstSpace), withoutHashtag.Substring(firstSpace + 1)); + var withoutHashtag = description.Substring(1).Trim(); + var firstSpace = withoutHashtag.IndexOf(" "); + if (firstSpace != -1) + return (withoutHashtag.Substring(0, firstSpace), withoutHashtag.Substring(firstSpace + 1)); - return (withoutHashtag, null); - } + return (withoutHashtag, null); } } diff --git a/Generator/Services/SolutionComponentService.cs b/Generator/Services/SolutionComponentService.cs new file mode 100644 index 0000000..d8797cc --- /dev/null +++ b/Generator/Services/SolutionComponentService.cs @@ -0,0 +1,380 @@ +using Microsoft.Crm.Sdk.Messages; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Messages; +using Microsoft.Xrm.Sdk.Query; + +namespace Generator.Services; + +public class ComponentInfo +{ + public int ComponentType { get; set; } + public Guid ObjectId { get; set; } + public Guid? SolutionComponentId { get; set; } + public ComponentInclusionType InclusionType { get; set; } + public int RootComponentBehaviour { get; set; } + public Guid SolutionId { get; set; } + + public override bool Equals(object? obj) + { + return obj is ComponentInfo info && + ComponentType == info.ComponentType && + ObjectId == info.ObjectId; + } + + public override int GetHashCode() + { + return HashCode.Combine(ComponentType, ObjectId); + } +} + +public record SolutionComponentInfo( + Guid ObjectId, + Guid SolutionComponentId, + int ComponentType, + int RootComponentBehaviour, + EntityReference SolutionId + ); + +public record DependencyInfo( + Guid DependencyId, + int DependencyType, + Guid DependentComponentNodeId, + Guid RequiredComponentNodeId + ); + +public record ComponentNodeInfo( + Guid NodeId, + int ComponentType, + Guid ObjectId, + EntityReference SolutionId + ); + +public enum ComponentInclusionType +{ + Explicit = 0, // Explicitly added to the solution + Implicit = 1, // Implicitly included (subcomponent) + Required = 2 // Required dependency +} + +public class SolutionComponentService +{ + private readonly ServiceClient _client; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public SolutionComponentService(ServiceClient client, IConfiguration configuration, ILogger logger) + { + _client = client; + _configuration = configuration; + _logger = logger; + } + + public IEnumerable GetAllSolutionComponents(List solutionIds) + { + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Getting all solution components for solutions: {string.Join(", ", solutionIds)}"); + var allComponents = new HashSet(); + + // Get explicit components from solution + IEnumerable explicitComponents; + try + { + explicitComponents = GetExplicitComponents(solutionIds); + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {explicitComponents.Count()} explicit components"); + } + catch (Exception ex) + { + _logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to get explicit components"); + throw; + } + + // Add explicit components - determine if explicit or implicit based on RootComponentBehaviour + foreach (var comp in explicitComponents) + { + // RootComponentBehaviour: + // 0 (IncludeSubcomponents) = Explicitly added with all subcomponents + // 1 (DoNotIncludeSubcomponents) = Explicitly added without subcomponents + // 2 (IncludeAsShellOnly) = Only shell/definition included + // If not present or other values, treat as explicit + var isExplicitlyAdded = comp.RootComponentBehaviour == 0 || comp.RootComponentBehaviour == 1 || comp.RootComponentBehaviour == 2; + + allComponents.Add(new ComponentInfo + { + ComponentType = comp.ComponentType, + ObjectId = comp.ObjectId, + SolutionComponentId = comp.SolutionComponentId, + RootComponentBehaviour = comp.RootComponentBehaviour, + InclusionType = isExplicitlyAdded ? ComponentInclusionType.Explicit : ComponentInclusionType.Implicit, + SolutionId = comp.SolutionId.Id + }); + } + + // Get required dependencies + try + { + var dependencies = GetRequiredComponents(explicitComponents); + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {dependencies.Count} dependency relationships"); + + // Get unique component node IDs + var requiredNodeIds = dependencies.Select(d => d.RequiredComponentNodeId).Distinct().ToList(); + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving component information for {requiredNodeIds.Count} dependency nodes"); + + // Retrieve component node information + var componentNodes = GetComponentNodes(requiredNodeIds); + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {componentNodes.Count} component nodes"); + + // Add required components + foreach (var node in componentNodes) + { + var componentInfo = new ComponentInfo + { + ComponentType = node.ComponentType, + ObjectId = node.ObjectId, + SolutionComponentId = null, + RootComponentBehaviour = -1, + InclusionType = ComponentInclusionType.Required, + SolutionId = node.SolutionId.Id + }; + + // Only add if not already present as explicit component + if (!allComponents.Contains(componentInfo)) + { + allComponents.Add(componentInfo); + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to get required components, continuing without them"); + } + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Total components found: {allComponents.Count} (Explicit: {allComponents.Count(c => c.InclusionType == ComponentInclusionType.Explicit)}, Implicit: {allComponents.Count(c => c.InclusionType == ComponentInclusionType.Implicit)}, Required: {allComponents.Count(c => c.InclusionType == ComponentInclusionType.Required)})"); + return allComponents; + } + + private IEnumerable GetExplicitComponents(List solutionIds) + { + var query = new QueryExpression("solutioncomponent") + { + ColumnSet = new ColumnSet("objectid", "componenttype", "solutioncomponentid", "rootcomponentbehavior", "solutionid"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("componenttype", ConditionOperator.In, new List() { 1, 2, 10, 20 }), // entity, attribute, 1:N relationship, role, workflow/flow, N:N relationship, sdkpluginstep (https://learn.microsoft.com/en-us/power-apps/developer/data-platform/reference/entities/solutioncomponent) + new ConditionExpression("solutionid", ConditionOperator.In, solutionIds) + } + } + }; + + return _client.RetrieveMultiple(query).Entities + .Select(e => new SolutionComponentInfo( + e.GetAttributeValue("objectid"), + e.GetAttributeValue("solutioncomponentid"), + e.GetAttributeValue("componenttype").Value, + e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, + e.GetAttributeValue("solutionid"))); + } + + private List GetComponentNodes(List nodeIds) + { + if (!nodeIds.Any()) + { + return new List(); + } + + var results = new List(); + const int batchSize = 500; // Query in batches to avoid URL length limits + + for (int i = 0; i < nodeIds.Count; i += batchSize) + { + var batch = nodeIds.Skip(i).Take(batchSize).ToList(); + + var query = new QueryExpression("dependencynode") + { + ColumnSet = new ColumnSet("dependencynodeid", "componenttype", "objectid", "solutionid"), + Criteria = new FilterExpression(LogicalOperator.And) + { + Conditions = + { + new ConditionExpression("dependencynodeid", ConditionOperator.In, batch.Cast().ToArray()) + } + } + }; + + try + { + var response = _client.RetrieveMultiple(query); + foreach (var entity in response.Entities) + { + results.Add(new ComponentNodeInfo( + entity.GetAttributeValue("dependencynodeid"), + entity.GetAttributeValue("componenttype").Value, + entity.GetAttributeValue("objectid"), + entity.GetAttributeValue("solutionid") + )); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to retrieve component nodes for batch starting at index {i}"); + } + } + + return results; + } + + private HashSet GetDependentComponents(IEnumerable components) + { + var results = new HashSet(); + var componentsList = components.ToList(); + int totalCount = componentsList.Count; + int processedCount = 0; + int errorCount = 0; + const int batchSize = 100; // Dataverse recommends batches of 100-1000 + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving dependent components for {totalCount} components in batches of {batchSize}"); + + for (int i = 0; i < totalCount; i += batchSize) + { + var batch = componentsList.Skip(i).Take(batchSize).ToList(); + var batchNum = (i / batchSize) + 1; + var totalBatches = (int)Math.Ceiling((double)totalCount / batchSize); + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Processing batch {batchNum}/{totalBatches} ({batch.Count} components)"); + + var executeMultiple = new ExecuteMultipleRequest + { + Settings = new ExecuteMultipleSettings + { + ContinueOnError = true, + ReturnResponses = true + }, + Requests = new OrganizationRequestCollection() + }; + + foreach (var component in batch) + { + executeMultiple.Requests.Add(new RetrieveDependentComponentsRequest + { + ComponentType = component.ComponentType, + ObjectId = component.ObjectId + }); + } + + try + { + var response = (ExecuteMultipleResponse)_client.Execute(executeMultiple); + + for (int j = 0; j < response.Responses.Count; j++) + { + var item = response.Responses[j]; + + if (item.Fault != null) + { + errorCount++; + var component = batch[j]; + _logger.LogWarning($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to retrieve dependents for component type {component.ComponentType}, ObjectId {component.ObjectId}: {item.Fault.Message}"); + continue; + } + + var dependentResponse = (RetrieveDependentComponentsResponse)item.Response; + foreach (var dep in dependentResponse.EntityCollection.Entities) + { + results.Add(new DependencyInfo( + dep.GetAttributeValue("dependencyid"), + dep.GetAttributeValue("dependencytype").Value, + dep.GetAttributeValue("dependentcomponentnodeid").Id, + dep.GetAttributeValue("requiredcomponentnodeid").Id)); + } + processedCount++; + } + } + catch (Exception ex) + { + errorCount += batch.Count; + _logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to execute batch {batchNum}. Continuing..."); + } + } + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Dependent components: Processed {processedCount}/{totalCount} components, {errorCount} errors, {results.Count} dependencies found"); + return results; + } + + private HashSet GetRequiredComponents(IEnumerable components) + { + var results = new HashSet(); + var componentsList = components.ToList(); + int totalCount = componentsList.Count; + int processedCount = 0; + int errorCount = 0; + const int batchSize = 100; // Dataverse recommends batches of 100-1000 + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving required components for {totalCount} components in batches of {batchSize}"); + + for (int i = 0; i < totalCount; i += batchSize) + { + var batch = componentsList.Skip(i).Take(batchSize).ToList(); + var batchNum = (i / batchSize) + 1; + var totalBatches = (int)Math.Ceiling((double)totalCount / batchSize); + + var executeMultiple = new ExecuteMultipleRequest + { + Settings = new ExecuteMultipleSettings + { + ContinueOnError = true, + ReturnResponses = true + }, + Requests = new OrganizationRequestCollection() + }; + + foreach (var component in batch) + { + executeMultiple.Requests.Add(new RetrieveRequiredComponentsRequest + { + ComponentType = component.ComponentType, + ObjectId = component.ObjectId + }); + } + + try + { + var response = (ExecuteMultipleResponse)_client.Execute(executeMultiple); + + for (int j = 0; j < response.Responses.Count; j++) + { + var item = response.Responses[j]; + + if (item.Fault != null) + { + errorCount++; + var component = batch[j]; + _logger.LogWarning($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to retrieve required components for component type {component.ComponentType}, ObjectId {component.ObjectId}: {item.Fault.Message}"); + continue; + } + + var requiredResponse = (RetrieveRequiredComponentsResponse)item.Response; + foreach (var dep in requiredResponse.EntityCollection.Entities) + { + results.Add(new DependencyInfo( + dep.GetAttributeValue("dependencyid"), + dep.GetAttributeValue("dependencytype").Value, + dep.GetAttributeValue("dependentcomponentnodeid").Id, + dep.GetAttributeValue("requiredcomponentnodeid").Id)); + } + processedCount++; + } + } + catch (Exception ex) + { + errorCount += batch.Count; + _logger.LogError(ex, $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Failed to execute batch {batchNum}. Continuing..."); + } + } + + _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Required components: Processed {processedCount}/{totalCount} components, {errorCount} errors, {results.Count} dependencies found"); + return results; + } +} \ No newline at end of file diff --git a/Generator/Services/SolutionService.cs b/Generator/Services/SolutionService.cs index 2f6bb8e..d4dd741 100644 --- a/Generator/Services/SolutionService.cs +++ b/Generator/Services/SolutionService.cs @@ -51,38 +51,13 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg return (resp.Entities.Select(e => e.GetAttributeValue("solutionid")).ToList(), resp.Entities.ToList()); } - /// - /// Retrieves solution components for given solution IDs - /// - public async Task> GetSolutionComponents(List solutionIds) - { - var entityQuery = new QueryExpression("solutioncomponent") - { - ColumnSet = new ColumnSet("objectid", "componenttype", "rootcomponentbehavior", "solutionid"), - Criteria = new FilterExpression(LogicalOperator.And) - { - Conditions = - { - new ConditionExpression("componenttype", ConditionOperator.In, new List() { 1, 2, 20, 29, 92 }), // entity, attribute, role, workflow/flow, sdkpluginstep (https://learn.microsoft.com/en-us/power-apps/developer/data-platform/reference/entities/solutioncomponent) - new ConditionExpression("solutionid", ConditionOperator.In, solutionIds) - } - } - }; - - return - (await client.RetrieveMultipleAsync(entityQuery)) - .Entities - .Select(e => (e.GetAttributeValue("objectid"), e.GetAttributeValue("componenttype").Value, e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, e.GetAttributeValue("solutionid"))) - .ToList(); - } - /// /// Creates Solution DTOs with their components /// public async Task> CreateSolutions( List solutionEntities, - IEnumerable<(Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId)> solutionComponents, - List allEntityMetadata) + IEnumerable solutionComponents, + IEnumerable allEntityMetadata) { var solutions = new List(); @@ -116,7 +91,7 @@ public async Task> CreateSolutions( )); // Group components by solution - var componentsBySolution = solutionComponents.GroupBy(c => c.SolutionId).ToDictionary(g => g.Key.Id, g => g); + var componentsBySolution = solutionComponents.GroupBy(c => c.SolutionId).ToDictionary(g => g.Key, g => g); // Process ALL solutions from configuration, not just those with components foreach (var solutionEntity in solutionEntities) @@ -160,9 +135,9 @@ public async Task> CreateSolutions( /// Creates a solution component DTO from component metadata /// private SolutionComponent? CreateSolutionComponent( - (Guid ObjectId, int ComponentType, int RootComponentBehavior, EntityReference SolutionId) component, + ComponentInfo component, Dictionary entityLookup, - List allEntityMetadata, + IEnumerable allEntityMetadata, Dictionary publisherLookup) { try diff --git a/Website/components/datamodelview/Relationships.tsx b/Website/components/datamodelview/Relationships.tsx index 243626d..6466872 100644 --- a/Website/components/datamodelview/Relationships.tsx +++ b/Website/components/datamodelview/Relationships.tsx @@ -6,8 +6,8 @@ import { useState, useEffect } from "react" import { useDatamodelView, useDatamodelViewDispatch } from "@/contexts/DatamodelViewContext" import React from "react" import { highlightMatch } from "../datamodelview/List"; -import { Button, FormControl, InputLabel, MenuItem, Select, Table, TableBody, TableCell, TableHead, TableRow, TextField, InputAdornment, Box, Typography, useTheme } from "@mui/material" -import { SearchRounded, ClearRounded, ArrowUpwardRounded, ArrowDownwardRounded } from "@mui/icons-material" +import { Button, FormControl, InputLabel, MenuItem, Select, Table, TableBody, TableCell, TableHead, TableRow, TextField, InputAdornment, Box, Typography, useTheme, Tooltip } from "@mui/material" +import { SearchRounded, ClearRounded, ArrowUpwardRounded, ArrowDownwardRounded, Visibility, VisibilityOff } from "@mui/icons-material" type SortDirection = 'asc' | 'desc' | null type SortColumn = 'name' | 'tableSchema' | 'lookupField' | 'type' | 'behavior' | 'schemaName' | null @@ -23,6 +23,7 @@ export const Relationships = ({ entity, search = "", onVisibleCountChange }: IRe const [sortDirection, setSortDirection] = useState("asc") const [typeFilter, setTypeFilter] = useState("all") const [searchQuery, setSearchQuery] = useState("") + const [hideStandardRelationships, setHideStandardRelationships] = useState(true) const theme = useTheme(); @@ -47,17 +48,17 @@ export const Relationships = ({ entity, search = "", onVisibleCountChange }: IRe const getSortedRelationships = () => { let filteredRelationships = entity.Relationships - + if (typeFilter !== "all") { - filteredRelationships = filteredRelationships.filter(rel => - (typeFilter === "many-to-many" && rel.IsManyToMany) || + filteredRelationships = filteredRelationships.filter(rel => + (typeFilter === "many-to-many" && rel.IsManyToMany) || (typeFilter === "one-to-many" && !rel.IsManyToMany) ) } - + if (searchQuery) { const query = searchQuery.toLowerCase() - filteredRelationships = filteredRelationships.filter(rel => + filteredRelationships = filteredRelationships.filter(rel => rel.Name.toLowerCase().includes(query) || rel.TableSchema.toLowerCase().includes(query) || rel.LookupDisplayName.toLowerCase().includes(query) || @@ -68,7 +69,7 @@ export const Relationships = ({ entity, search = "", onVisibleCountChange }: IRe // Also filter by parent search prop if provided if (search && search.length >= 3) { const query = search.toLowerCase() - filteredRelationships = filteredRelationships.filter(rel => + filteredRelationships = filteredRelationships.filter(rel => rel.Name.toLowerCase().includes(query) || rel.TableSchema.toLowerCase().includes(query) || rel.LookupDisplayName.toLowerCase().includes(query) || @@ -76,6 +77,8 @@ export const Relationships = ({ entity, search = "", onVisibleCountChange }: IRe ) } + if (hideStandardRelationships) filteredRelationships = filteredRelationships.filter(rel => rel.IsInSolution); + if (!sortColumn || !sortDirection) return filteredRelationships return [...filteredRelationships].sort((a, b) => { @@ -181,7 +184,7 @@ export const Relationships = ({ entity, search = "", onVisibleCountChange }: IRe }} /> - + Filter by type + + + {(searchQuery || typeFilter !== "all") && (