Fix for greek letters (#2712)

This pull request refactors the `TfsWorkItemMigrationProcessor` class by
removing unused code, improving validation logic, and enhancing logging
and telemetry. The changes aim to simplify the codebase, improve
maintainability, and provide better insights during execution. Below are
the most important changes grouped by theme:

### Code Simplification and Cleanup:
* Removed the `ProgressTimer` class and associated logic, replacing it
with simpler calculations for average and remaining time during work
item processing.
(`src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs`)
[[1]](diffhunk://#diff-b0a23408b76051c7078de3664c94b03ef26ca367ede9f084067d8722393f130eL40-L70)
[[2]](diffhunk://#diff-b0a23408b76051c7078de3664c94b03ef26ca367ede9f084067d8722393f130eL181)
[[3]](diffhunk://#diff-b0a23408b76051c7078de3664c94b03ef26ca367ede9f084067d8722393f130eL643-R689)
* Removed unused imports from `TfsValidateRequiredFieldTool.cs` to clean
up dependencies.
(`src/MigrationTools.Clients.TfsObjectModel/Tools/TfsValidateRequiredFieldTool.cs`)

### Validation Improvements:
* Enhanced the `ValiddateWorkItemTypesExistInTarget` method to include
detailed logging for missing work item types, suggestions for mapping
based on character similarity, and explicit error messages for unmapped
types.
(`src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs`)
* Fixed a typo in the method name `ValiddateWorkItemTypesExistInTarget`
to maintain consistency.
(`src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs`)

### Logging and Telemetry Enhancements:
* Improved exception logging in `ValidatingRequiredField` to include the
work item type name for better debugging.
(`src/MigrationTools.Clients.TfsObjectModel/Tools/TfsValidateRequiredFieldTool.cs`)
* Added telemetry tags and end time tracking to activities in
`ProcessWorkItemAsync` for better observability.
(`src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs`)
[[1]](diffhunk://#diff-b0a23408b76051c7078de3664c94b03ef26ca367ede9f084067d8722393f130eR404)
[[2]](diffhunk://#diff-b0a23408b76051c7078de3664c94b03ef26ca367ede9f084067d8722393f130eL533-R577)

### Configuration Update:
* Updated the `launchSettings.json` file to use a specific configuration
file path for execution, likely for testing or debugging purposes.
(`src/MigrationTools.ConsoleFull/Properties/launchSettings.json`)

Author: MrHinsh

src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs

@@ -37,37 +37,6 @@ namespace MigrationTools.Processors
     /// <processingtarget>Work Items</processingtarget>
     public class TfsWorkItemMigrationProcessor : TfsProcessor
     {
-        private class ProgressTimer
-        {
-            private int _totalCount;
-            private int _processedCount;
-            private TimeSpan _totalProcessedTime = TimeSpan.Zero;
-
-            public ProgressTimer(int totalCount)
-            {
-                _totalCount = totalCount;
-            }
-
-            public int TotalCount => _totalCount;
-            public int ProcessedCount => _processedCount;
-            public TimeSpan TotalProcessedTime => _totalProcessedTime;
-
-            public void AddProcessedItem(TimeSpan processDuration, bool addToTotal)
-            {
-                if (addToTotal)
-                {
-                    _totalCount++;
-                }
-                _processedCount++;
-                _totalProcessedTime += processDuration;
-            }
-
-            public TimeSpan AverageTime => ProcessedCount > 0
-                ? TimeSpan.FromSeconds(TotalProcessedTime.TotalSeconds / ProcessedCount)
-                : TotalProcessedTime;
-
-            public TimeSpan RemainingTime => TimeSpan.FromTicks(AverageTime.Ticks * (TotalCount - ProcessedCount));
-        }
 
         private static int _count = 0;
         private static int _current = 0;
@@ -143,8 +112,8 @@ protected override void InternalExecute()
                     sourceWorkItems.Count);
 
                 //////////////////////////////////////////////////
-                ValidateAllWorkItemTypesHaveReflectedWorkItemIdField(sourceWorkItems);
                 ValiddateWorkItemTypesExistInTarget(sourceWorkItems);
+                ValidateAllWorkItemTypesHaveReflectedWorkItemIdField(sourceWorkItems);
                 CommonTools.NodeStructure.ValidateAllNodesExistOrAreMapped(this, sourceWorkItems, Source.WorkItems.Project.Name, Target.WorkItems.Project.Name);
                 ValidateAllUsersExistOrAreMapped(sourceWorkItems);
                 //////////////////////////////////////////////////
@@ -178,7 +147,6 @@ protected override void InternalExecute()
                 _current = 1;
                 _count = sourceWorkItems.Count;
                 _totalWorkItem = sourceWorkItems.Count;
-                ProgressTimer progressTimer = new ProgressTimer(_totalWorkItem);
                 ProcessorActivity.SetTag("source_workitems_to_process", sourceWorkItems.Count);
                 foreach (WorkItemData sourceWorkItemData in sourceWorkItems)
                 {
@@ -195,7 +163,7 @@ protected override void InternalExecute()
                     {
                         try
                         {
-                            ProcessWorkItemAsync(sourceWorkItemData, progressTimer, Options.WorkItemCreateRetryLimit).Wait();
+                            ProcessWorkItemAsync(sourceWorkItemData, Options.WorkItemCreateRetryLimit).Wait();
 
                             stopwatch.Stop();
                             var processingTime = stopwatch.Elapsed.TotalMilliseconds;
@@ -241,6 +209,7 @@ protected override void InternalExecute()
                 {
                     contextLog.Warning("The following items could not be migrated: {ItemIds}", string.Join(", ", _itemsInError));
                 }
+
             }
         }
 
@@ -297,44 +266,111 @@ private void ValidateAllWorkItemTypesHaveReflectedWorkItemIdField(List<WorkItemD
         private void ValiddateWorkItemTypesExistInTarget(List<WorkItemData> sourceWorkItems)
         {
             contextLog.Information("Validating::Check that all work item types needed in the Target exist or are mapped");
-            // get list of all work item types
-            List<String> sourceWorkItemTypes = sourceWorkItems.SelectMany(x => x.Revisions.Values)
-            //.Where(x => x.Fields[fieldName].Value.ToString().Contains("\\"))
-            .Select(x => x.Type)
-            .Distinct()
-            .ToList();
 
-            Log.LogDebug("Validating::WorkItemTypes::sourceWorkItemTypes: {count} WorkItemTypes in the full source history {sourceWorkItemTypesString}", sourceWorkItemTypes.Count(), string.Join(",", sourceWorkItemTypes));
+            var sourceWorkItemTypes = sourceWorkItems
+                .SelectMany(x => x.Revisions.Values)
+                .Select(x => x.Type)
+                .Distinct()
+                .ToList();
+
+            Log.LogDebug("Validating::WorkItemTypes::sourceWorkItemTypes: {count} WorkItemTypes in the full source history: {sourceWorkItemTypesString}",
+                sourceWorkItemTypes.Count, string.Join(",", sourceWorkItemTypes));
+
+            var targetWorkItemTypes = Target.WorkItems.Project
+                .ToProject()
+                .WorkItemTypes
+                .Cast<WorkItemType>()
+                .Select(x => x.Name)
+                .Distinct()
+                .ToList();
+
+
+
+            Log.LogDebug("Validating::WorkItemTypes::targetWorkItemTypes: {count} WorkItemTypes in Target process: {targetWorkItemTypesString}",
+                targetWorkItemTypes.Count, string.Join(",", targetWorkItemTypes));
+
+            var missingWorkItemTypes = new List<string>();
 
-            var targetWorkItemTypes = Target.WorkItems.Project.ToProject().WorkItemTypes.Cast<WorkItemType>().Select(x => x.Name);
-            Log.LogDebug("Validating::WorkItemTypes::targetWorkItemTypes::{count} WorkItemTypes in Target process: {targetWorkItemTypesString}", targetWorkItemTypes.Count(), string.Join(",", targetWorkItemTypes));
+            foreach (var sourceWit in sourceWorkItemTypes)
+            {
+                var witToFind = sourceWit;
+                if (CommonTools.WorkItemTypeMapping.Mappings.ContainsKey(sourceWit))
+                {
+                    witToFind = CommonTools.WorkItemTypeMapping.Mappings[sourceWit];
+                    Log.LogDebug("Validating::WorkItemTypes::sourceWit: `{sourceWit}` is mapped to `{witToFind}`", sourceWit, witToFind);
+                }
+                if (!targetWorkItemTypes.Contains(witToFind, StringComparer.OrdinalIgnoreCase))
+                {
+                    missingWorkItemTypes.Add(witToFind);
+                }
+            }
 
-            var missingWorkItemTypes = sourceWorkItemTypes.Where(sourceWit => !targetWorkItemTypes.Contains(sourceWit)); // the real one
-            if (missingWorkItemTypes.Count() > 0)
+            if (missingWorkItemTypes.Count > 0)
             {
-                Log.LogWarning("Validating::WorkItemTypes::targetWorkItemTypes::There are {count} WorkItemTypes that are used in the history of the Source and that do not exist in the Target. These will all need mapped using `WorkItemTypeDefinition` in the config. ", missingWorkItemTypes.Count());
+                Log.LogWarning("There are {count} WorkItemTypes used in source history that do not exist in the target. These may need to be mapped.", missingWorkItemTypes.Count);
 
-                bool allTypesMapped = true;
-                foreach (var missingWorkItemType in missingWorkItemTypes)
+                foreach (var sourceWit in missingWorkItemTypes)
                 {
-                    bool thisTypeMapped = true;
-                    if (!CommonTools.WorkItemTypeMapping.Mappings.ContainsKey(missingWorkItemType))
+                    Log.LogWarning("Missing Source WIT: '{sourceWit}' (Unicode: {unicode})",
+                        sourceWit,
+                        string.Join(" ", sourceWit.Select(c => $"U+{(int)c:X4}")));
+
+                    // Try to suggest a match based on character similarity
+                    foreach (var targetWit in targetWorkItemTypes)
                     {
-                        thisTypeMapped = false;
+                        if (AreVisuallySimilar(sourceWit, targetWit))
+                        {
+                            Log.LogWarning("→ Suggested mapping: \"{0}\": \"{1}\"", sourceWit, targetWit);
+                        }
+                    }
+
+                    if (!CommonTools.WorkItemTypeMapping.Mappings.ContainsKey(sourceWit))
+                    {
+                        Log.LogWarning("WorkItemType '{0}' is not mapped in your config. Add to WorkItemTypeDefinition if needed.", sourceWit);
                     }
-                    Log.LogWarning("Validating::WorkItemTypes::targetWorkItemTypes::{missingWorkItemType}::Mapped? {thisTypeMapped}", missingWorkItemType, thisTypeMapped.ToString());
-                    allTypesMapped &= thisTypeMapped;
                 }
-                if (!allTypesMapped)
+
+                var ex = new Exception("Some Work Item Types in the source do not exist in the target. Either map or filter them.");
+                Log.LogError(ex, "Validation failed: unmapped or missing work item types.");
+                Environment.Exit(-1);
+            }
+        }
+
+
+        private static bool AreVisuallySimilar(string a, string b)
+        {
+            if (a.Equals(b, StringComparison.OrdinalIgnoreCase)) return false; // Already matched normally
+
+            if (a.Length != b.Length) return false;
+
+            int similarCount = 0;
+
+            for (int i = 0; i < a.Length; i++)
+            {
+                var aChar = a[i];
+                var bChar = b[i];
+
+                if (aChar == bChar)
                 {
-                    var ex = new Exception(
-                       "Not all WorkItemTypes present in the Source are present in the Target or mapped! Filter them from the query, or map the to target types.");
-                    Log.LogError(ex, "Not all WorkItemTypes present in the Source are present in the Target or mapped using `WorkItemTypeDefinition` in the config.");
-                    Environment.Exit(-1);
+                    similarCount++;
+                    continue;
+                }
+
+                // Check known lookalike characters (expandable)
+                if ((aChar == '\u0399' && bChar == 'I') || (aChar == 'I' && bChar == '\u0399'))
+                {
+                    similarCount++;
+                }
+                else
+                {
+                    return false;
                 }
             }
+
+            return similarCount == a.Length;
         }
 
+
         private void ValidatePatTokenRequirement()
         {
             string collUrl = Target.Options.Collection.ToString();
@@ -365,6 +401,7 @@ private WorkItemData CreateWorkItem_Shell(ProjectData destProject, WorkItemData
                 activity?.SetTagsFromOptions(Options);
                 activity?.SetTag("http.request.method", "GET");
                 activity?.SetTag("migrationtools.client", "TfsObjectModel");
+                activity?.SetEndTime(activity.StartTimeUtc.AddSeconds(10));
 
                 WorkItem newwit;
                 if (destProject.ToProject().WorkItemTypes.Contains(destType))
@@ -443,7 +480,7 @@ private void PopulateWorkItem(WorkItemData oldWorkItemData, WorkItemData newWork
                     newWorkItem.Fields["Microsoft.VSTS.Common.ClosedDate"].Value = oldWorkItem.Fields["Microsoft.VSTS.Common.ClosedDate"].Value;
                 }
             }
-            catch (FieldDefinitionNotExistException)
+            catch (FieldDefinitionNotExistException ex)
             {
                 // Eat exception coz the TFS API Sucks
             }
@@ -530,13 +567,14 @@ private void ProcessWorkItemEmbeddedLinks(WorkItemData sourceWorkItem, WorkItemD
             }
         }
 
-        private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, ProgressTimer progressTimer, int retryLimit = 5, int retries = 0)
+        private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, int retryLimit = 5, int retries = 0)
         {
             using (var activity = ActivitySourceProvider.ActivitySource.StartActivity("ProcessWorkItemAsync", ActivityKind.Client))
             {
                 activity?.SetTagsFromOptions(Options);
                 activity?.SetTag("http.request.method", "GET");
                 activity?.SetTag("migrationtools.client", "TfsObjectModel");
+                activity?.SetEndTime(activity.StartTimeUtc.AddSeconds(10));
                 activity?.SetTag("SourceURL", Source.Options.Collection.ToString());
                 activity?.SetTag("SourceWorkItem", sourceWorkItem.Id);
                 activity?.SetTag("TargetURL", Target.Options.Collection.ToString());
@@ -624,7 +662,7 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, ProgressTim
                             {"Retrys", retries },
                             {"RetryLimit", retryLimit }
                             });
-                        await ProcessWorkItemAsync(sourceWorkItem, progressTimer, retryLimit, retries);
+                        await ProcessWorkItemAsync(sourceWorkItem, retryLimit, retries);
                     }
                     else
                     {
@@ -640,18 +678,15 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, ProgressTim
                     Telemetry.TrackException(ex, activity.Tags);
                     throw ex;
                 }
-                activity?.Stop();
-                progressTimer.AddProcessedItem(activity.Duration, retries > 0);
+                var average = new TimeSpan(0, 0, 0, 0, (int)(activity.Duration.TotalMilliseconds / _current));
+                var remaining = new TimeSpan(0, 0, 0, 0, (int)(average.TotalMilliseconds * _count));
                 TraceWriteLine(LogEventLevel.Information,
-                    "Processed {processedItemsCount} items from {totalItemsCount} with average process time {average:%s}.{average:%fff} s (total processing time is {totalProcessedTime:h\\:mm\\:ss}). "
-                    + "Estimated time to completion is {remaining:%h} hours {remaining:%m} minutes {remaining:%s} seconds.",
+                    "Average time of {average:%s}.{average:%fff} per work item and {remaining:%h} hours {remaining:%m} minutes {remaining:%s}.{remaining:%fff} seconds estimated to completion",
                     new Dictionary<string, object>() {
-                        { "processedItemsCount", progressTimer.ProcessedCount},
-                        { "totalItemsCount", progressTimer.TotalCount },
-                        { "totalProcessedTime", progressTimer.TotalProcessedTime },
-                        { "average", progressTimer.AverageTime },
-                        { "remaining", progressTimer.RemainingTime }
+                    {"average", average},
+                    {"remaining", remaining}
                     });
+                activity?.Stop();
                 activity?.SetStatus(ActivityStatusCode.Error);
                 activity?.SetTag("http.response.status_code", "200");
 

src/MigrationTools.Clients.TfsObjectModel/Tools/TfsEmbededImagesTool.cs

@@ -1,22 +1,17 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq.Expressions;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text.RegularExpressions;
-using System.Windows.Forms;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.TeamFoundation.Client;
 using Microsoft.TeamFoundation.WorkItemTracking.Client;
 using Microsoft.TeamFoundation.WorkItemTracking.WebApi;
-using MigrationTools._EngineV1.Configuration;
 using MigrationTools.DataContracts;
 using MigrationTools.Endpoints;
-using MigrationTools.Processors;
 using MigrationTools.Processors.Infrastructure;
 using MigrationTools.Tools.Infrastructure;
 
@@ -28,8 +23,8 @@ public class TfsEmbededImagesTool : EmbededImagesRepairToolBase<TfsEmbededImages
         private const string RegexPatternForImageFileName = "(?<=FileName=)[^=]*";
         private const string TargetDummyWorkItemTitle = "***** DELETE THIS - Migration Tool Generated Dummy Work Item For TfsEmbededImagesTool *****";
 
-        private  Project _targetProject;
-        private  TfsTeamProjectEndpointOptions _targetConfig;
+        private Project _targetProject;
+        private TfsTeamProjectEndpointOptions _targetConfig;
 
         private readonly IDictionary<string, string> _cachedUploadedUrisBySourceValue;
 
@@ -50,7 +45,7 @@ public int FixEmbededImages(TfsProcessor processor, WorkItemData sourceWorkItem,
             return 0;
         }
 
-        public  void ProcessorExecutionEnd(TfsProcessor processor)
+        public void ProcessorExecutionEnd(TfsProcessor processor)
         {
             _processor = processor;
             if (_targetDummyWorkItem != null)
@@ -159,9 +154,10 @@ private string UploadedAndRetrieveAttachmentLinkUrl(string matchedSourceUri, str
 
                 return attachRef.Url;
             }
-            catch (Exception ex) {
+            catch (Exception ex)
+            {
                 throw ex;
-            }            
+            }
             finally
             {
                 if (File.Exists(fullImageFilePath))
@@ -200,13 +196,13 @@ private Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models.AttachmentRefere
             });
 
             var dummyWi = GetDummyWorkItem(wi.Type);
-            var wii = httpClient.UpdateWorkItemAsync(payload, dummyWi.Id).GetAwaiter().GetResult();
+            var wii = httpClient.UpdateWorkItemAsync(payload, dummyWi.Id, bypassRules: true).GetAwaiter().GetResult();
             if (wii != null)
             {
                 payload[0].Operation = Microsoft.VisualStudio.Services.WebApi.Patch.Operation.Remove;
                 payload[0].Path = "/relations/" + (wii.Relations.Count - 1);
                 payload[0].Value = null;
-                wii = httpClient.UpdateWorkItemAsync(payload, dummyWi.Id).GetAwaiter().GetResult();
+                wii = httpClient.UpdateWorkItemAsync(payload, dummyWi.Id, bypassRules: true).GetAwaiter().GetResult();
             }
             else
             {
@@ -265,7 +261,8 @@ private WorkItem GetDummyWorkItem(WorkItemType type = null)
                 if (_targetDummyWorkItem.Id == 0)
                 {
                     throw new Exception("The Dummy work Item cant be created due to a save failure. This is likley due to required fields on the Task or First work items type.");
-                } else
+                }
+                else
                 {
                     Log.LogDebug("TfsEmbededImagesTool: Dummy workitem {id} created on the target collection.", _targetDummyWorkItem.Id);
                     //_targetProject.Store.DestroyWorkItems(new List<int> { _targetDummyWorkItem.Id });

src/MigrationTools.Clients.TfsObjectModel/Tools/TfsValidateRequiredFieldTool.cs

@@ -4,11 +4,8 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Microsoft.TeamFoundation.TestManagement.WebApi;
 using Microsoft.TeamFoundation.WorkItemTracking.Client;
 using MigrationTools.DataContracts;
-using MigrationTools.Enrichers;
-using MigrationTools.Processors;
 using MigrationTools.Processors.Infrastructure;
 using MigrationTools.Tools.Infrastructure;
 using MigrationTools.Tools.Interfaces;
@@ -56,7 +53,7 @@ public bool ValidatingRequiredField(TfsProcessor processor, string fieldToFind,
                 }
                 catch (WorkItemTypeDeniedOrNotExistException ex)
                 {
-                    Log.LogWarning(ex, "ValidatingRequiredField: Unable to validate one of the work items as its returned by TFS but has been deleted");
+                    Log.LogWarning(ex, "ValidatingRequiredField: Unable to validate work item type {name} as its returned by the source but does not exist in the target", sourceWorkItemType.Name);
                 }
 
             }
@@ -81,4 +78,4 @@ public bool ValidatingRequiredField(TfsProcessor processor, string fieldToFind,
         //    }
         //}
     }
-}
+}