diff --git a/KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs b/KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs index 0bdc8347..dc7a8e9f 100644 --- a/KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs +++ b/KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs @@ -187,9 +187,8 @@ public async Task Handle(MigratePagesCommand request, Cancellatio ? (Guid?)null : spoiledGuidContext.EnsureNodeGuid(ksNodeParent); - DataClassInfo targetClass = null!; var classMapping = classMappingProvider.GetMapping(ksNodeClass.ClassName); - targetClass = classMapping != null + var targetClass = classMapping != null ? DataClassInfoProvider.ProviderObject.Get(classMapping.TargetClassName) : DataClassInfoProvider.ProviderObject.Get(ksNodeClass.ClassGUID); @@ -210,6 +209,7 @@ public async Task Handle(MigratePagesCommand request, Cancellatio var commonDataInfos = new List(); foreach (var umtModel in results) { + logger.LogTrace("UMT-M: {UMT}", umtModel.PrintMe()); switch (await importer.ImportAsync(umtModel)) { case { Success: false } result: @@ -233,9 +233,23 @@ public async Task Handle(MigratePagesCommand request, Cancellatio webPageItemInfo = wp; break; } - + case {} importResult: + { + logger.LogTrace("Unexpected state {UMT} Result: {Result}", umtModel.PrintMe(), new + { + importResult.Success, + importResult.PrimaryKey, + importResult.Imported, + importResult.Exception, + validationResults = JsonConvert.SerializeObject(importResult.ModelValidationResults ?? []) + }); + break; + } default: + { + logger.LogTrace("Unexpected state {UMT}", umtModel.PrintMe()); break; + } } } @@ -270,7 +284,14 @@ await MigratePageUrlPaths(ksSite.SiteGUID, } else { - logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath); + if (nodeParentGuid is {} npg && ContentItemInfo.Provider.Get(npg) is {ContentItemIsReusable:true}) + { + logger.LogTrace("No webpage item produced for '{NodeAliasPath}' - parent is reusable, possibly converted with mapping?", ksNode.NodeAliasPath); + } + else + { + logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath); + } } } diff --git a/KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs b/KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs index eab7a346..73823934 100644 --- a/KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs +++ b/KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs @@ -66,7 +66,7 @@ protected override IEnumerable MapInternal(CmsTreeMapperSource source var sourceNodeClass = modelFacade.SelectById(cmsTree.NodeClassID) ?? throw new InvalidOperationException($"Fatal: node class is missing, class id '{cmsTree.NodeClassID}'"); var mapping = classMappingProvider.GetMapping(sourceNodeClass.ClassName); var targetClassGuid = sourceNodeClass.ClassGUID; - DataClassInfo targetClassInfo = null; + var targetClassInfo = DataClassInfoProvider.ProviderObject.Get(sourceNodeClass.ClassName); if (mapping != null) { targetClassInfo = DataClassInfoProvider.ProviderObject.Get(mapping.TargetClassName) ?? throw new InvalidOperationException($"Unable to find target class '{mapping.TargetClassName}'"); @@ -74,9 +74,14 @@ protected override IEnumerable MapInternal(CmsTreeMapperSource source } bool migratedAsContentFolder = sourceNodeClass.ClassName.Equals("cms.folder", StringComparison.InvariantCultureIgnoreCase) && !configuration.UseDeprecatedFolderPageType.GetValueOrDefault(false); - + var contentItemGuid = spoiledGuidContext.EnsureNodeGuid(cmsTree.NodeGUID, cmsTree.NodeSiteID, cmsTree.NodeID); bool isMappedTypeReusable = (targetClassInfo?.ClassContentTypeType is ClassContentTypeType.REUSABLE) || configuration.ClassNamesConvertToContentHub.Contains(sourceNodeClass.ClassName); + if (isMappedTypeReusable) + { + logger.LogTrace("Target is reusable {Info}", new { cmsTree.NodeAliasPath, targetClassInfo?.ClassName }); + } + yield return new ContentItemModel { ContentItemGUID = contentItemGuid, @@ -670,7 +675,7 @@ IClassMapping mapping !configuration.MigrateMediaToMediaLibrary) { var mediaLinkService = mediaLinkServiceFactory.Create(); - var htmlProcessor = new HtmlProcessor(html, mediaLinkService); + var htmlProcessor = new HtmlProcessor(html, mediaLinkService, logger); target[targetColumnName] = await htmlProcessor.ProcessHtml(site.SiteID, async (result, original) => { diff --git a/KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs b/KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs index 64452916..bfdbfb45 100644 --- a/KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs +++ b/KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs @@ -26,7 +26,7 @@ public class MediaFileMigrator( PrimaryKeyMappingContext primaryKeyMappingContext, EntityIdentityFacade entityIdentityFacade, IProtocol protocol - ) : IMediaFileMigrator +) : IMediaFileMigrator { public async Task Handle(MigrateMediaLibrariesCommand request, CancellationToken cancellationToken) { @@ -84,6 +84,7 @@ SELECT LibraryName FROM Media_Library GROUP BY LibraryName HAVING COUNT(*) > 1 } catch (Exception ex) { + logger.LogTrace($"Failed {ksMediaLibrary}"); protocol.Append(HandbookReferences .ErrorCreatingTargetInstance(ex) .NeedsManualAction() @@ -136,70 +137,77 @@ private void RequireMigratedMediaFiles(List<(IMediaLibrary sourceLibrary, ICmsSi foreach (var ksMediaFile in ksMediaFiles) { - protocol.FetchedSource(ksMediaFile); - - bool found = false; - IUploadedFile? uploadedFile = null; - string? fullMediaPath = ""; - if (loadMediaFileData) + try { - (found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType); - if (!found) + protocol.FetchedSource(ksMediaFile); + + bool found = false; + IUploadedFile? uploadedFile = null; + string? fullMediaPath = ""; + if (loadMediaFileData) { - // report missing file (currently reported in mapper) + (found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType); + if (!found) + { + // report missing file (currently reported in mapper) + } } - } - string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath); + string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath); - (bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile); - if (isFixed) - { - logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid); - } - - var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid); + (bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile); + if (isFixed) + { + logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid); + } - protocol.FetchedTarget(kxoMediaFile); + var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid); - var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null, - librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid); - var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile); - protocol.MappedTarget(mapped); + protocol.FetchedTarget(kxoMediaFile); - if (mapped is { Success: true } result) - { - (var mf, bool newInstance) = result; - ArgumentNullException.ThrowIfNull(mf, nameof(mf)); + var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null, + librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid); + var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile); + protocol.MappedTarget(mapped); - try + if (mapped is { Success: true } result) { - if (newInstance) + (var mf, bool newInstance) = result; + ArgumentNullException.ThrowIfNull(mf, nameof(mf)); + + try { - mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID); - } + if (newInstance) + { + mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID); + } - mediaFileFacade.SetMediaFile(mf, newInstance); + mediaFileFacade.SetMediaFile(mf, newInstance); - protocol.Success(ksMediaFile, mf, mapped); - logger.LogEntitySetAction(newInstance, mf); - } - catch (Exception ex) - { - protocol.Append(HandbookReferences - .ErrorCreatingTargetInstance(ex) - .NeedsManualAction() - .WithIdentityPrint(mf) + protocol.Success(ksMediaFile, mf, mapped); + logger.LogEntitySetAction(newInstance, mf); + } + catch (Exception ex) + { + protocol.Append(HandbookReferences + .ErrorCreatingTargetInstance(ex) + .NeedsManualAction() + .WithIdentityPrint(mf) + ); + logger.LogEntitySetError(ex, newInstance, mf); + continue; + } + + primaryKeyMappingContext.SetMapping( + r => r.FileID, + ksMediaFile.FileID, + mf.FileID ); - logger.LogEntitySetError(ex, newInstance, mf); - continue; } - - primaryKeyMappingContext.SetMapping( - r => r.FileID, - ksMediaFile.FileID, - mf.FileID - ); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed {MediaFile}", ksMediaFile); } } } diff --git a/Migration.Tool.Common/Commands.cs b/Migration.Tool.Common/Commands.cs index 1393020e..801ab8a1 100644 --- a/Migration.Tool.Common/Commands.cs +++ b/Migration.Tool.Common/Commands.cs @@ -86,7 +86,7 @@ public record MigratePageTypesCommand : IRequest, ICommand public record MigratePagesCommand : IRequest, ICommand { - public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank; + public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank + MigrateMediaLibrariesCommand.Rank; public static string Moniker => "pages"; public static string MonikerFriendly => "Pages"; diff --git a/Migration.Tool.Common/Helpers/HtmlProcessor.cs b/Migration.Tool.Common/Helpers/HtmlProcessor.cs index cd51d49c..dfadf227 100644 --- a/Migration.Tool.Common/Helpers/HtmlProcessor.cs +++ b/Migration.Tool.Common/Helpers/HtmlProcessor.cs @@ -1,16 +1,19 @@ using HtmlAgilityPack; +using Microsoft.Extensions.Logging; namespace Migration.Tool.Common.Helpers; public class HtmlProcessor { private readonly MediaLinkService mediaLinkService; + private readonly ILogger? logger; private readonly HtmlDocument document; private readonly string html; - public HtmlProcessor(string html, MediaLinkService mediaLinkService) + public HtmlProcessor(string html, MediaLinkService mediaLinkService, ILogger? logger) { this.mediaLinkService = mediaLinkService; + this.logger = logger; var doc = new HtmlDocument(); doc.LoadHtml(html); @@ -25,7 +28,20 @@ public IEnumerable GetImages(int currentSiteId) { if (imgNode?.Attributes["src"].Value is { } src) { - yield return mediaLinkService.MatchMediaLink(src, currentSiteId); + MatchMediaLinkResult? mediaLinkMatch = null; + try + { + mediaLinkMatch = mediaLinkService.MatchMediaLink(src, currentSiteId); + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed to match media link for value '{SourceValue}'", src); + } + + if (mediaLinkMatch != null) + { + yield return mediaLinkMatch; + } } } } diff --git a/Migration.Tool.Common/Services/CommandParser.cs b/Migration.Tool.Common/Services/CommandParser.cs index f5009468..d5f07490 100644 --- a/Migration.Tool.Common/Services/CommandParser.cs +++ b/Migration.Tool.Common/Services/CommandParser.cs @@ -120,7 +120,7 @@ public List Parse(Queue args, ref bool bypassDependencyCheck, } } - return commands; + return commands.OrderBy(c => c.Rank).ToList(); } private void PrintCommandDescriptions() diff --git a/Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs b/Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs index 48e9644e..10fef2ac 100644 --- a/Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs +++ b/Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs @@ -57,8 +57,20 @@ public async Task MigrateValue(object? sourceValue, FieldM List mfis = []; bool hasMigratedAsset = false; - if (sourceValue is string link && - mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID) is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result) + MatchMediaLinkResult? mediaLinkMatch = null; + try + { + if (sourceValue is string link) + { + mediaLinkMatch = mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to match media link for value '{SourceValue}'", sourceValue); + } + + if (mediaLinkMatch is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result) { if (mediaLinkKind == MediaLinkKind.Path) { @@ -107,10 +119,21 @@ public async Task MigrateValue(object? sourceValue, FieldM { if (configuration.MigrateMediaToMediaLibrary) { - if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x) + if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi) { - mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }]; - hasMigratedAsset = true; + if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf) + { + mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }]; + hasMigratedAsset = true; + } + else + { + logger.LogError("Media file {MediaFileIdentity} not found", mfi); + } + } + else + { + logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile); } } else @@ -165,10 +188,21 @@ public async Task MigrateValue(object? sourceValue, FieldM { if (configuration.MigrateMediaToMediaLibrary) { - if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x) + if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi) { - mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }]; - hasMigratedAsset = true; + if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf) + { + mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }]; + hasMigratedAsset = true; + } + else + { + logger.LogError("Media file {MediaFileIdentity} not found", mfi); + } + } + else + { + logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile); } } else diff --git a/Migration.Tool.Tests/HtmlProcessorTests.cs b/Migration.Tool.Tests/HtmlProcessorTests.cs index 31ac2933..d1e0aa36 100644 --- a/Migration.Tool.Tests/HtmlProcessorTests.cs +++ b/Migration.Tool.Tests/HtmlProcessorTests.cs @@ -48,7 +48,7 @@ public void TestHtmlFragment1() } ); - var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService); + var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService, null); var actual = processor.GetImages(2).ToArray(); Assert.Collection(actual, diff --git a/Migration.Tool.Tests/MediaHelperTest.cs b/Migration.Tool.Tests/MediaHelperTest.cs index a57718e4..0b1173d3 100644 --- a/Migration.Tool.Tests/MediaHelperTest.cs +++ b/Migration.Tool.Tests/MediaHelperTest.cs @@ -293,5 +293,108 @@ public void ParseMediaLinkDirectPath() Assert.Equal(3, a.LinkSiteId); Assert.Equal("MediaLibraryS3", a.LibraryDir); } + + { + var a = mediaLinkService.MatchMediaLink("http://localhost:5003/Site3/media/MediaLibraryS3/some sub dir/myfile.jpg", 3); + Assert.True(a.Success); + Assert.Equal("/MediaLibraryS3/some sub dir/myfile.jpg", a.Path); + Assert.Null(a.MediaGuid); + Assert.Equal(MediaKind.MediaFile, a.MediaKind); + Assert.Equal(MediaLinkKind.DirectMediaPath, a.LinkKind); + Assert.Equal(3, a.LinkSiteId); + Assert.Equal("MediaLibraryS3/some sub dir", a.LibraryDir); + } + } + + // [Fact] + // public void ParseMediaLinkDirectPath_SpacesInPath() + // { + // var mediaLinkService = new MediaLinkService( + // [ + // (1, "Site1", "http://localhost:5001"), // site with custom global dir + // (2, "Site2", "http://localhost:5002/SiteSubPath"), // site with custom global dir & subpath + // (3, "Site3", "http://localhost:5003"), // site without custom global media library dir + // (4, "Site4", "http://localhost:5004"), // site with custom global media library dir & without media sites folder + // // add site with live site url "localhost" - that was valid in K11 + // ], + // [ + // // (null, null), not set globally + // (1, "Site1MediaFolder"), + // (2, "Site2MediaFolder"), + // // (3, null) not set for site 3 + // (4, "Site4MediaFolder"), + // ], + // [ + // // (null, null) not set globally + // (1, "False"), + // (2, "False"), + // (3, "True"), + // (4, "True"), + // ], + // new Dictionary> + // { + // {1, new (["MediaLibraryS1"],StringComparer.InvariantCultureIgnoreCase)}, + // {2, new (["MediaLibraryS2"],StringComparer.InvariantCultureIgnoreCase)}, + // {3, new (["MediaLibraryS3"],StringComparer.InvariantCultureIgnoreCase)}, + // {4, new (["MediaLibraryS4"],StringComparer.InvariantCultureIgnoreCase)} + // } + // ); + // + // { + // var a = mediaLinkService.MatchMediaLink("http://localhost:5003/Site3/media/MediaLibraryS3/some sub dir/myfile.jpg", 3); + // Assert.True(a.Success); + // Assert.Equal("/MediaLibraryS3/some sub dir/myfile.jpg", a.Path); + // Assert.Null(a.MediaGuid); + // Assert.Equal(MediaKind.MediaFile, a.MediaKind); + // Assert.Equal(MediaLinkKind.DirectMediaPath, a.LinkKind); + // Assert.Equal(3, a.LinkSiteId); + // Assert.Equal("MediaLibraryS3/some sub dir", a.LibraryDir); + // } + // } + + [Fact] + public void ParseMediaLinkDirectPath_SpacesInPath_CorrectlyEscaped() + { + var mediaLinkService = new MediaLinkService( + [ + (1, "Site1", "http://localhost:5001"), // site with custom global dir + (2, "Site2", "http://localhost:5002/SiteSubPath"), // site with custom global dir & subpath + (3, "Site3", "http://localhost:5003"), // site without custom global media library dir + (4, "Site4", "http://localhost:5004"), // site with custom global media library dir & without media sites folder + // add site with live site url "localhost" - that was valid in K11 + ], + [ + // (null, null), not set globally + (1, "Site1MediaFolder"), + (2, "Site2MediaFolder"), + // (3, null) not set for site 3 + (4, "Site4MediaFolder"), + ], + [ + // (null, null) not set globally + (1, "False"), + (2, "False"), + (3, "True"), + (4, "True"), + ], + new Dictionary> + { + {1, new (["MediaLibraryS1"],StringComparer.InvariantCultureIgnoreCase)}, + {2, new (["MediaLibraryS2"],StringComparer.InvariantCultureIgnoreCase)}, + {3, new (["MediaLibraryS3"],StringComparer.InvariantCultureIgnoreCase)}, + {4, new (["MediaLibraryS4"],StringComparer.InvariantCultureIgnoreCase)} + } + ); + + { + var a = mediaLinkService.MatchMediaLink("http://localhost:5003/Site3/media/MediaLibraryS3/some%20sub%20dir/myfile.jpg", 3); + Assert.True(a.Success); + Assert.Equal("/MediaLibraryS3/some sub dir/myfile.jpg", a.Path); + Assert.Null(a.MediaGuid); + Assert.Equal(MediaKind.MediaFile, a.MediaKind); + Assert.Equal(MediaLinkKind.DirectMediaPath, a.LinkKind); + Assert.Equal(3, a.LinkSiteId); + Assert.Equal("MediaLibraryS3", a.LibraryDir); + } } }