Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/302 issues #309

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Fix/302 issues #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions KVA/Migration.Tool.Source/Handlers/MigratePagesCommandHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,8 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
? (Guid?)null
: spoiledGuidContext.EnsureNodeGuid(ksNodeParent);

DataClassInfo targetClass = null!;
var classMapping = classMappingProvider.GetMapping(ksNodeClass.ClassName);
targetClass = classMapping != null
var targetClass = classMapping != null
? DataClassInfoProvider.ProviderObject.Get(classMapping.TargetClassName)
: DataClassInfoProvider.ProviderObject.Get(ksNodeClass.ClassGUID);

Expand All @@ -210,6 +209,7 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
var commonDataInfos = new List<ContentItemCommonDataInfo>();
foreach (var umtModel in results)
{
logger.LogTrace("UMT-M: {UMT}", umtModel.PrintMe());
switch (await importer.ImportAsync(umtModel))
{
case { Success: false } result:
Expand All @@ -233,9 +233,23 @@ public async Task<CommandResult> Handle(MigratePagesCommand request, Cancellatio
webPageItemInfo = wp;
break;
}

case {} importResult:
{
logger.LogTrace("Unexpected state {UMT} Result: {Result}", umtModel.PrintMe(), new
{
importResult.Success,
importResult.PrimaryKey,
importResult.Imported,
importResult.Exception,
validationResults = JsonConvert.SerializeObject(importResult.ModelValidationResults ?? [])
});
break;
}
default:
{
logger.LogTrace("Unexpected state {UMT}", umtModel.PrintMe());
break;
}
}
}

Expand Down Expand Up @@ -270,7 +284,14 @@ await MigratePageUrlPaths(ksSite.SiteGUID,
}
else
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath);
if (nodeParentGuid is {} npg && ContentItemInfo.Provider.Get(npg) is {ContentItemIsReusable:true})
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}' - parent is reusable, possibly converted with mapping?", ksNode.NodeAliasPath);
}
else
{
logger.LogTrace("No webpage item produced for '{NodeAliasPath}'", ksNode.NodeAliasPath);
}
}
}

Expand Down
11 changes: 8 additions & 3 deletions KVA/Migration.Tool.Source/Mappers/ContentItemMapper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,22 @@ protected override IEnumerable<IUmtModel> MapInternal(CmsTreeMapperSource source
var sourceNodeClass = modelFacade.SelectById<ICmsClass>(cmsTree.NodeClassID) ?? throw new InvalidOperationException($"Fatal: node class is missing, class id '{cmsTree.NodeClassID}'");
var mapping = classMappingProvider.GetMapping(sourceNodeClass.ClassName);
var targetClassGuid = sourceNodeClass.ClassGUID;
DataClassInfo targetClassInfo = null;
var targetClassInfo = DataClassInfoProvider.ProviderObject.Get(sourceNodeClass.ClassName);
if (mapping != null)
{
targetClassInfo = DataClassInfoProvider.ProviderObject.Get(mapping.TargetClassName) ?? throw new InvalidOperationException($"Unable to find target class '{mapping.TargetClassName}'");
targetClassGuid = targetClassInfo.ClassGUID;
}

bool migratedAsContentFolder = sourceNodeClass.ClassName.Equals("cms.folder", StringComparison.InvariantCultureIgnoreCase) && !configuration.UseDeprecatedFolderPageType.GetValueOrDefault(false);

var contentItemGuid = spoiledGuidContext.EnsureNodeGuid(cmsTree.NodeGUID, cmsTree.NodeSiteID, cmsTree.NodeID);
bool isMappedTypeReusable = (targetClassInfo?.ClassContentTypeType is ClassContentTypeType.REUSABLE) || configuration.ClassNamesConvertToContentHub.Contains(sourceNodeClass.ClassName);
if (isMappedTypeReusable)
{
logger.LogTrace("Target is reusable {Info}", new { cmsTree.NodeAliasPath, targetClassInfo?.ClassName });
}

yield return new ContentItemModel
{
ContentItemGUID = contentItemGuid,
Expand Down Expand Up @@ -670,7 +675,7 @@ IClassMapping mapping
!configuration.MigrateMediaToMediaLibrary)
{
var mediaLinkService = mediaLinkServiceFactory.Create();
var htmlProcessor = new HtmlProcessor(html, mediaLinkService);
var htmlProcessor = new HtmlProcessor(html, mediaLinkService, logger);

target[targetColumnName] = await htmlProcessor.ProcessHtml(site.SiteID, async (result, original) =>
{
Expand Down
108 changes: 58 additions & 50 deletions KVA/Migration.Tool.Source/Services/MediaFileMigrator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class MediaFileMigrator(
PrimaryKeyMappingContext primaryKeyMappingContext,
EntityIdentityFacade entityIdentityFacade,
IProtocol protocol
) : IMediaFileMigrator
) : IMediaFileMigrator
{
public async Task<CommandResult> Handle(MigrateMediaLibrariesCommand request, CancellationToken cancellationToken)
{
Expand Down Expand Up @@ -84,6 +84,7 @@ SELECT LibraryName FROM Media_Library GROUP BY LibraryName HAVING COUNT(*) > 1
}
catch (Exception ex)
{
logger.LogTrace($"Failed {ksMediaLibrary}");
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
Expand Down Expand Up @@ -136,70 +137,77 @@ private void RequireMigratedMediaFiles(List<(IMediaLibrary sourceLibrary, ICmsSi

foreach (var ksMediaFile in ksMediaFiles)
{
protocol.FetchedSource(ksMediaFile);

bool found = false;
IUploadedFile? uploadedFile = null;
string? fullMediaPath = "<uninitialized>";
if (loadMediaFileData)
try
{
(found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType);
if (!found)
protocol.FetchedSource(ksMediaFile);

bool found = false;
IUploadedFile? uploadedFile = null;
string? fullMediaPath = "<uninitialized>";
if (loadMediaFileData)
{
// report missing file (currently reported in mapper)
(found, uploadedFile, fullMediaPath) = LoadMediaFileBinary(sourceMediaLibraryPath, ksMediaFile.FilePath, ksMediaFile.FileMimeType);
if (!found)
{
// report missing file (currently reported in mapper)
}
}
}

string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath);
string? librarySubfolder = Path.GetDirectoryName(ksMediaFile.FilePath);

(bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile);
if (isFixed)
{
logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid);
}

var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid);
(bool isFixed, var safeMediaFileGuid) = entityIdentityFacade.Translate(ksMediaFile);
if (isFixed)
{
logger.LogWarning("MediaFile {File} has non-unique guid, new guid {Guid} was required", new { ksMediaFile.FileGUID, ksMediaFile.FileName, ksMediaFile.FileSiteID }, safeMediaFileGuid);
}

protocol.FetchedTarget(kxoMediaFile);
var kxoMediaFile = mediaFileFacade.GetMediaFile(safeMediaFileGuid);

var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null,
librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid);
var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile);
protocol.MappedTarget(mapped);
protocol.FetchedTarget(kxoMediaFile);

if (mapped is { Success: true } result)
{
(var mf, bool newInstance) = result;
ArgumentNullException.ThrowIfNull(mf, nameof(mf));
var source = new MediaFileInfoMapperSource(fullMediaPath, ksMediaFile, targetMediaLibrary.LibraryID, found ? uploadedFile : null,
librarySubfolder, toolConfiguration.MigrateOnlyMediaFileInfo.GetValueOrDefault(false), safeMediaFileGuid);
var mapped = mediaFileInfoMapper.Map(source, kxoMediaFile);
protocol.MappedTarget(mapped);

try
if (mapped is { Success: true } result)
{
if (newInstance)
(var mf, bool newInstance) = result;
ArgumentNullException.ThrowIfNull(mf, nameof(mf));

try
{
mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID);
}
if (newInstance)
{
mediaFileFacade.EnsureMediaFilePathExistsInLibrary(mf, targetMediaLibrary.LibraryID);
}

mediaFileFacade.SetMediaFile(mf, newInstance);
mediaFileFacade.SetMediaFile(mf, newInstance);

protocol.Success(ksMediaFile, mf, mapped);
logger.LogEntitySetAction(newInstance, mf);
}
catch (Exception ex)
{
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
.WithIdentityPrint(mf)
protocol.Success(ksMediaFile, mf, mapped);
logger.LogEntitySetAction(newInstance, mf);
}
catch (Exception ex)
{
protocol.Append(HandbookReferences
.ErrorCreatingTargetInstance<MediaLibraryInfo>(ex)
.NeedsManualAction()
.WithIdentityPrint(mf)
);
logger.LogEntitySetError(ex, newInstance, mf);
continue;
}

primaryKeyMappingContext.SetMapping<MediaFileInfo>(
r => r.FileID,
ksMediaFile.FileID,
mf.FileID
);
logger.LogEntitySetError(ex, newInstance, mf);
continue;
}

primaryKeyMappingContext.SetMapping<MediaFileInfo>(
r => r.FileID,
ksMediaFile.FileID,
mf.FileID
);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed {MediaFile}", ksMediaFile);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Common/Commands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public record MigratePageTypesCommand : IRequest<CommandResult>, ICommand

public record MigratePagesCommand : IRequest<CommandResult>, ICommand
{
public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank;
public static readonly int Rank = 1 + MigrateSitesCommand.Rank + MigrateUsersCommand.Rank + MigratePageTypesCommand.Rank + MigrateMediaLibrariesCommand.Rank;

public static string Moniker => "pages";
public static string MonikerFriendly => "Pages";
Expand Down
20 changes: 18 additions & 2 deletions Migration.Tool.Common/Helpers/HtmlProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;

namespace Migration.Tool.Common.Helpers;

public class HtmlProcessor
{
private readonly MediaLinkService mediaLinkService;
private readonly ILogger? logger;
private readonly HtmlDocument document;
private readonly string html;

public HtmlProcessor(string html, MediaLinkService mediaLinkService)
public HtmlProcessor(string html, MediaLinkService mediaLinkService, ILogger? logger)
{
this.mediaLinkService = mediaLinkService;
this.logger = logger;

var doc = new HtmlDocument();
doc.LoadHtml(html);
Expand All @@ -25,7 +28,20 @@ public IEnumerable<MatchMediaLinkResult> GetImages(int currentSiteId)
{
if (imgNode?.Attributes["src"].Value is { } src)
{
yield return mediaLinkService.MatchMediaLink(src, currentSiteId);
MatchMediaLinkResult? mediaLinkMatch = null;
try
{
mediaLinkMatch = mediaLinkService.MatchMediaLink(src, currentSiteId);
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed to match media link for value '{SourceValue}'", src);
}

if (mediaLinkMatch != null)
{
yield return mediaLinkMatch;
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Common/Services/CommandParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public List<ICommand> Parse(Queue<string> args, ref bool bypassDependencyCheck,
}
}

return commands;
return commands.OrderBy(c => c.Rank).ToList();
}

private void PrintCommandDescriptions()
Expand Down
50 changes: 42 additions & 8 deletions Migration.Tool.Extensions/DefaultMigrations/AssetMigration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,20 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM

List<object> mfis = [];
bool hasMigratedAsset = false;
if (sourceValue is string link &&
mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID) is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result)
MatchMediaLinkResult? mediaLinkMatch = null;
try
{
if (sourceValue is string link)
{
mediaLinkMatch = mediaLinkServiceFactory.Create().MatchMediaLink(link, cmsSite.SiteID);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to match media link for value '{SourceValue}'", sourceValue);
}

if (mediaLinkMatch is (true, var mediaLinkKind, var mediaKind, var path, var mediaGuid, _, _) result)
{
if (mediaLinkKind == MediaLinkKind.Path)
{
Expand Down Expand Up @@ -107,10 +119,21 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM
{
if (configuration.MigrateMediaToMediaLibrary)
{
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x)
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi)
{
mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }];
hasMigratedAsset = true;
if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf)
{
mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }];
hasMigratedAsset = true;
}
else
{
logger.LogError("Media file {MediaFileIdentity} not found", mfi);
}
}
else
{
logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile);
}
}
else
Expand Down Expand Up @@ -165,10 +188,21 @@ public async Task<FieldMigrationResult> MigrateValue(object? sourceValue, FieldM
{
if (configuration.MigrateMediaToMediaLibrary)
{
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mf && mediaFileFacade.GetMediaFile(mf.Identity) is { } x)
if (entityIdentityFacade.Translate(sourceMediaFile) is { } mfi)
{
mfis = [new AssetRelatedItem { Identifier = x.FileGUID, Dimensions = new AssetDimensions { Height = x.FileImageHeight, Width = x.FileImageWidth }, Name = x.FileName, Size = x.FileSize }];
hasMigratedAsset = true;
if (mediaFileFacade.GetMediaFile(mfi.Identity) is { } mf)
{
mfis = [new AssetRelatedItem { Identifier = mf.FileGUID, Dimensions = new AssetDimensions { Height = mf.FileImageHeight, Width = mf.FileImageWidth }, Name = mf.FileName, Size = mf.FileSize }];
hasMigratedAsset = true;
}
else
{
logger.LogError("Media file {MediaFileIdentity} not found", mfi);
}
}
else
{
logger.LogError("Unable to determine identity for {MediaFile}", sourceMediaFile);
}
}
else
Expand Down
2 changes: 1 addition & 1 deletion Migration.Tool.Tests/HtmlProcessorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public void TestHtmlFragment1()
}
);

var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService);
var processor = new HtmlProcessor(HtmlFragmentSample1, mediaLinkService, null);

var actual = processor.GetImages(2).ToArray();
Assert.Collection(actual,
Expand Down
Loading