Fix pagination in Content Delivery API Index Helper (#19606)

* Refactor descendant enumeration in DeliveryApiContentIndexHelper

Improved loop condition to allow for processing of more than 10.000 descendants for indexing.

* Add failing test for original issue.

* Renamed variable for clarity.

---------

Co-authored-by: Brynjar Þorsteinsson <brynjar@vettvangur.is>
Co-authored-by: Andy Butland <abutland73@gmail.com>

src/Umbraco.Cms.Api.Delivery/Services/RoutingServiceBase.cs

@@ -36,7 +36,7 @@ internal abstract class RoutingServiceBase
     }
 
     protected static string GetContentRoute(DomainAndUri domainAndUri, Uri contentRoute)
-        => $"{domainAndUri.ContentId}{DomainUtilities.PathRelativeToDomain(domainAndUri.Uri, contentRoute.AbsolutePath)}";
+        => $"{domainAndUri.ContentId}{DomainUtilities.PathRelativeToDomain(domainAndUri.Uri, contentRoute.LocalPath)}"; // Use LocalPath over AbsolutePath to keep the path decoded.
 
     protected DomainAndUri? GetDomainAndUriForRoute(Uri contentUrl)
     {

src/Umbraco.Cms.Api.Delivery/Umbraco.Cms.Api.Delivery.csproj

@@ -13,6 +13,9 @@
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
       <_Parameter1>Umbraco.Tests.UnitTests</_Parameter1>
     </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests.Integration</_Parameter1>
+    </AssemblyAttribute>
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
       <_Parameter1>DynamicProxyGenAssembly2</_Parameter1>
     </AssemblyAttribute>

src/Umbraco.Core/Models/ContentEditing/PropertyTypeBasic.cs

@@ -25,6 +25,7 @@ public class PropertyTypeBasic
 
     [Required]
     [RegularExpression(@"^([a-zA-Z]\w.*)$", ErrorMessage = "Invalid alias")]
+    [MaxLength(255, ErrorMessage = "Alias is too long")]
     [DataMember(Name = "alias")]
     public string Alias { get; set; } = null!;
 

src/Umbraco.Core/Services/ContentService.cs

@@ -3627,6 +3627,7 @@ public class ContentService : RepositoryService, IContentService
 
     private static readonly string?[] ArrayOfOneNullString = { null };
 
+    /// <inheritdoc />
     public IContent CreateContentFromBlueprint(IContent blueprint, string name, int userId = Constants.Security.SuperUserId)
     {
         if (blueprint == null)

src/Umbraco.Core/Services/IContentService.cs

@@ -56,6 +56,9 @@ public interface IContentService : IContentServiceBase<IContent>
     /// <summary>
     ///     Creates a new content item from a blueprint.
     /// </summary>
+    /// <remarks>Warning: If you intend to save the resulting <c>IContent</c> as a content node, you must trigger a
+    /// <see cref="Notifications.ContentScaffoldedNotification"/> notification to ensure that the block ids are regenerated.
+    /// Failing to do so could lead to caching issues.</remarks>
     IContent CreateContentFromBlueprint(IContent blueprint, string name, int userId = Constants.Security.SuperUserId);
 
     /// <summary>

src/Umbraco.Core/Services/MediaService.cs

@@ -418,7 +418,7 @@ namespace Umbraco.Cms.Core.Services
             }
 
             using ICoreScope scope = ScopeProvider.CreateCoreScope(autoComplete: true);
-            scope.ReadLock(Constants.Locks.ContentTree);
+            scope.ReadLock(Constants.Locks.MediaTree);
             return _mediaRepository.GetPage(Query<IMedia>()?.Where(x => x.ContentTypeId == contentTypeId), pageIndex, pageSize, out totalRecords, filter, ordering);
         }
 
@@ -441,7 +441,7 @@ namespace Umbraco.Cms.Core.Services
             }
 
             using ICoreScope scope = ScopeProvider.CreateCoreScope(autoComplete: true);
-            scope.ReadLock(Constants.Locks.ContentTree);
+            scope.ReadLock(Constants.Locks.MediaTree);
             return _mediaRepository.GetPage(
                 Query<IMedia>()?.Where(x => contentTypeIds.Contains(x.ContentTypeId)), pageIndex, pageSize, out totalRecords, filter, ordering);
         }

src/Umbraco.Infrastructure/DeliveryApi/ApiRichTextElementParser.cs

@@ -1,13 +1,10 @@
-using HtmlAgilityPack;
-using Microsoft.Extensions.DependencyInjection;
+using HtmlAgilityPack;
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.DeliveryApi;
-using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models.Blocks;
 using Umbraco.Cms.Core.Models.DeliveryApi;
 using Umbraco.Cms.Core.PublishedCache;
-using Umbraco.Cms.Core.Routing;
 using Umbraco.Cms.Infrastructure.Extensions;
 using Umbraco.Extensions;
 
@@ -101,9 +98,9 @@ internal sealed class ApiRichTextElementParser : ApiRichTextParserBase, IApiRich
         // - non-#comment nodes
         // - non-#text nodes
         // - non-empty #text nodes
-        // - empty #text between inline elements (see #17037)
+        // - empty #text between inline elements (see #17037) but not #text with only newlines (see #19388)
         HtmlNode[] childNodes = element.ChildNodes
-            .Where(c => c.Name != CommentNodeName && (c.Name != TextNodeName || c.NextSibling is not null || string.IsNullOrWhiteSpace(c.InnerText) is false))
+            .Where(c => c.Name != CommentNodeName && (c.Name != TextNodeName || IsNonEmptyElement(c)))
             .ToArray();
 
         var tag = TagName(element);
@@ -124,6 +121,9 @@ internal sealed class ApiRichTextElementParser : ApiRichTextParserBase, IApiRich
         return createElement(tag, attributes, childElements);
     }
 
+    private static bool IsNonEmptyElement(HtmlNode htmlNode) =>
+        string.IsNullOrWhiteSpace(htmlNode.InnerText) is false || htmlNode.InnerText.Any(c => c != '\n' && c != '\r');
+
     private string TagName(HtmlNode htmlNode) => htmlNode.Name;
 
     private void ReplaceLocalLinks(IPublishedSnapshot publishedSnapshot, Dictionary<string, object> attributes)

src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs

@@ -1,4 +1,4 @@
-using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Querying;
@@ -28,21 +28,28 @@ internal sealed class DeliveryApiContentIndexHelper : IDeliveryApiContentIndexHe
     public void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform)
     {
         const int pageSize = 10000;
-        var pageIndex = 0;
+        EnumerateApplicableDescendantsForContentIndex(rootContentId, actionToPerform, pageSize);
+    }
+
+    internal void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform, int pageSize)
+    {
+        var itemIndex = 0;
+        long total;
+
+        IQuery<IContent> query = _umbracoDatabaseFactory.SqlContext.Query<IContent>().Where(content => content.Trashed == false);
 
         IContent[] descendants;
-        IQuery<IContent> query = _umbracoDatabaseFactory.SqlContext.Query<IContent>().Where(content => content.Trashed == false);
         do
         {
             descendants = _contentService
-                .GetPagedDescendants(rootContentId, pageIndex, pageSize, out _, query, Ordering.By("Path"))
+                .GetPagedDescendants(rootContentId, itemIndex / pageSize, pageSize, out total, query, Ordering.By("Path"))
                 .Where(descendant => _deliveryApiSettings.IsAllowedContentType(descendant.ContentType.Alias))
                 .ToArray();
 
-            actionToPerform(descendants.ToArray());
+            actionToPerform(descendants);
 
-            pageIndex++;
+            itemIndex += pageSize;
         }
-        while (descendants.Length == pageSize);
+        while (descendants.Length > 0 && itemIndex < total);
     }
 }

src/Umbraco.Infrastructure/Persistence/Repositories/Implement/TagRepository.cs

@@ -128,10 +128,13 @@ internal class TagRepository : EntityRepositoryBase<int, ITag>, ITagRepository
         var group = SqlSyntax.GetQuotedColumnName("group");
 
         // insert tags
+        // - Note we are checking in the subquery for the existence of the tag, so we don't insert duplicates, using a case-insensitive comparison (the
+        //   LOWER keyword is consistent across SQLite and SQLServer). This ensures consistent behavior across databases as by default, SQLServer will
+        //   perform a case-insensitive comparison, while SQLite will not.
         var sql1 = $@"INSERT INTO cmsTags (tag, {group}, languageId)
 SELECT tagSet.tag, tagSet.{group}, tagSet.languageId
 FROM {tagSetSql}
-LEFT OUTER JOIN cmsTags ON (tagSet.tag = cmsTags.tag AND tagSet.{group} = cmsTags.{group} AND COALESCE(tagSet.languageId, -1) = COALESCE(cmsTags.languageId, -1))
+LEFT OUTER JOIN cmsTags ON (LOWER(tagSet.tag) = LOWER(cmsTags.tag) AND LOWER(tagSet.{group}) = LOWER(cmsTags.{group}) AND COALESCE(tagSet.languageId, -1) = COALESCE(cmsTags.languageId, -1))
 WHERE cmsTags.id IS NULL";
 
         Database.Execute(sql1);
@@ -142,7 +145,7 @@ SELECT {contentId}, {propertyTypeId}, tagSet2.Id
 FROM (
     SELECT t.Id
     FROM {tagSetSql}
-    INNER JOIN cmsTags as t ON (tagSet.tag = t.tag AND tagSet.{group} = t.{group} AND COALESCE(tagSet.languageId, -1) = COALESCE(t.languageId, -1))
+    INNER JOIN cmsTags as t ON (LOWER(tagSet.tag) = LOWER(t.tag) AND LOWER(tagSet.{group}) = LOWER(t.{group}) AND COALESCE(tagSet.languageId, -1) = COALESCE(t.languageId, -1))
 ) AS tagSet2
 LEFT OUTER JOIN cmsTagRelationship r ON (tagSet2.id = r.tagId AND r.nodeId = {contentId} AND r.propertyTypeID = {propertyTypeId})
 WHERE r.tagId IS NULL";
@@ -245,14 +248,18 @@ WHERE r.tagId IS NULL";
     {
         public bool Equals(ITag? x, ITag? y) =>
             ReferenceEquals(x, y) // takes care of both being null
-            || (x != null && y != null && x.Text == y.Text && x.Group == y.Group && x.LanguageId == y.LanguageId);
+            || (x != null &&
+                y != null &&
+                string.Equals(x.Text, y.Text, StringComparison.OrdinalIgnoreCase) &&
+                string.Equals(x.Group, y.Group, StringComparison.OrdinalIgnoreCase) &&
+            x.LanguageId == y.LanguageId);
 
         public int GetHashCode(ITag obj)
         {
             unchecked
             {
-                var h = obj.Text.GetHashCode();
-                h = (h * 397) ^ obj.Group.GetHashCode();
+                var h = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Text);
+                h = (h * 397) ^ StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Group);
                 h = (h * 397) ^ (obj.LanguageId?.GetHashCode() ?? 0);
                 return h;
             }

src/Umbraco.Infrastructure/PropertyEditors/RichTextPropertyIndexValueFactory.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using System.Text.RegularExpressions;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Models.Blocks;
@@ -50,9 +51,11 @@ internal class RichTextPropertyIndexValueFactory : NestedPropertyIndexValueFacto
                 : null;
 
         // index the stripped HTML values combined with "blocks values resume" value
+        var richTextWithoutMarkup = StripHtmlForIndexing(richTextEditorValue.Markup);
+
         yield return new KeyValuePair<string, IEnumerable<object?>>(
             property.Alias,
-            new object[] { $"{richTextEditorValue.Markup.StripHtml()} {blocksIndexValuesResume}" });
+            new object[] { $"{richTextWithoutMarkup} {blocksIndexValuesResume}" });
 
         // store the raw value
         yield return new KeyValuePair<string, IEnumerable<object?>>(
@@ -75,4 +78,28 @@ internal class RichTextPropertyIndexValueFactory : NestedPropertyIndexValueFacto
 
     protected override IEnumerable<BlockItemData> GetDataItems(RichTextEditorValue input)
         => input.Blocks?.ContentData ?? new List<BlockItemData>();
+
+    /// <summary>
+    /// Strips HTML tags from content while preserving whitespace from line breaks.
+    /// This addresses the issue where &lt;br&gt; tags don't create word boundaries when HTML is stripped.
+    /// </summary>
+    /// <param name="html">The HTML content to strip</param>
+    /// <returns>Plain text with proper word boundaries</returns>
+    private static string StripHtmlForIndexing(string html)
+    {
+        if (string.IsNullOrWhiteSpace(html))
+        {
+            return string.Empty;
+        }
+
+        // Replace <br> and <br/> tags (with any amount of whitespace and attributes) with spaces
+        // This regex matches:
+        // - <br> (with / without spaces or attributes)
+        // - <br /> (with / without spaces or attributes)
+        html = Regex.Replace(html, @"<br\b[^>]*/?>\s*", " ", RegexOptions.IgnoreCase);
+
+        // Use the existing Microsoft StripHtml function for everything else
+        return html.StripHtml();
+    }
+
 }

src/Umbraco.PublishedCache.NuCache/ContentStore.cs

@@ -321,17 +321,12 @@ public class ContentStore
     {
         if (_writeLock.CurrentCount != 0)
         {
-            throw new InvalidOperationException("Write lock must be acquried.");
+            throw new InvalidOperationException("Write lock must be acquired.");
         }
     }
 
     private void Lock(WriteLockInfo lockInfo, bool forceGen = false)
     {
-        if (_writeLock.CurrentCount == 0)
-        {
-            throw new InvalidOperationException("Recursive locks not allowed");
-        }
-
         if (_writeLock.Wait(_monitorTimeout))
         {
             lockInfo.Taken = true;

src/Umbraco.Web.BackOffice/Controllers/AuthenticationController.cs

@@ -131,17 +131,12 @@ public class AuthenticationController : UmbracoApiControllerBase
         AuthorizationPolicies.BackOfficeAccess)] // Needed to enforce the principle set on the request, if one exists.
     public IDictionary<string, object> GetPasswordConfig(int userId)
     {
-        if (HttpContext.HasActivePasswordResetFlowSession(userId))
-        {
-            return _passwordConfiguration.GetConfiguration();
-        }
-
         Attempt<int> currentUserId =
             _backofficeSecurityAccessor.BackOfficeSecurity?.GetUserId() ?? Attempt<int>.Fail();
-
-        return currentUserId.Success
-            ? _passwordConfiguration.GetConfiguration(currentUserId.Result != userId)
-            : new Dictionary<string, object>();
+        return _passwordConfiguration.GetConfiguration(
+            currentUserId.Success
+                ? currentUserId.Result != userId
+                : true);
     }
 
     /// <summary>
@@ -422,8 +417,6 @@ public class AuthenticationController : UmbracoApiControllerBase
     [Authorize(Policy = AuthorizationPolicies.DenyLocalLoginIfConfigured)]
     public async Task<ActionResult<UserDetail?>> PostLogin(LoginModel loginModel)
     {
-        HttpContext.EndPasswordResetFlowSession();
-
         // Start a timed scope to ensure failed responses return is a consistent time
         var loginDuration = Math.Max(_loginDurationAverage ?? _securitySettings.UserDefaultFailedLoginDurationInMilliseconds, _securitySettings.UserMinimumFailedLoginDurationInMilliseconds);
         await using var timedScope = new TimedScope(loginDuration, HttpContext.RequestAborted);
@@ -497,8 +490,6 @@ public class AuthenticationController : UmbracoApiControllerBase
             return BadRequest();
         }
 
-        HttpContext.EndPasswordResetFlowSession();
-
         BackOfficeIdentityUser? identityUser = await _userManager.FindByEmailAsync(model.Email);
 
         await Task.Delay(RandomNumberGenerator.GetInt32(400, 2500)); // To randomize response time preventing user enumeration
@@ -655,8 +646,6 @@ public class AuthenticationController : UmbracoApiControllerBase
     [AllowAnonymous]
     public async Task<IActionResult> PostSetPassword(SetPasswordModel model)
     {
-        HttpContext.EndPasswordResetFlowSession();
-
         BackOfficeIdentityUser? identityUser =
             await _userManager.FindByIdAsync(model.UserId.ToString(CultureInfo.InvariantCulture));
             if (identityUser is null)

src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs

@@ -402,11 +402,6 @@ public class BackOfficeController : UmbracoController
 
         var result = await _userManager.VerifyUserTokenAsync(user, "Default", "ResetPassword", resetCode);
 
-        if (result)
-        {
-            HttpContext.StartPasswordResetFlowSession(userId);
-        }
-
         return result ?
 
             // Redirect to login with userId and resetCode

src/Umbraco.Web.BackOffice/Extensions/HttpContextExtensions.cs

@@ -5,20 +5,9 @@ namespace Umbraco.Extensions;
 
 public static class HttpContextExtensions
 {
-    private const string PasswordResetFlowSessionKey = nameof(PasswordResetFlowSessionKey);
-
     public static void SetExternalLoginProviderErrors(this HttpContext httpContext, BackOfficeExternalLoginProviderErrors errors)
         => httpContext.Items[nameof(BackOfficeExternalLoginProviderErrors)] = errors;
 
     public static BackOfficeExternalLoginProviderErrors? GetExternalLoginProviderErrors(this HttpContext httpContext)
         => httpContext.Items[nameof(BackOfficeExternalLoginProviderErrors)] as BackOfficeExternalLoginProviderErrors;
-
-    internal static void StartPasswordResetFlowSession(this HttpContext httpContext, int userId)
-        => httpContext.Session.SetInt32(PasswordResetFlowSessionKey, userId);
-
-    internal static void EndPasswordResetFlowSession(this HttpContext httpContext)
-        => httpContext.Session.Remove(PasswordResetFlowSessionKey);
-
-    internal static bool HasActivePasswordResetFlowSession(this HttpContext httpContext, int userId)
-        => httpContext.Session.GetInt32(PasswordResetFlowSessionKey) == userId;
 }

src/Umbraco.Web.UI.Client/src/views/components/umb-locked-field.html

@@ -25,7 +25,8 @@
                title="{{ngModel}}"
                focus-when="{{!locked}}"
                umb-select-when="{{!locked}}"
-               ng-blur="lock()" />
+               ng-blur="lock()"
+               ng-maxlength="255" />
 
     </div>
 
@@ -46,6 +47,11 @@
              ng-if="serverValidationField.length > 0"
              ng-message="valServerField">{{lockedFieldForm.lockedField.errorMsg}}
         </div>
+        <div class="umb-validation-label"
+             ng-class="{ '-left': validationPosition === 'left', '-right': validationPosition === 'right' }"
+             ng-if="ngModel.length > 255">
+            <localize key="general_invalid">Invalid</localize> <localize key="content_alias">alias</localize>
+        </div>
     </div>
 
 </ng-form>

tests/Umbraco.Tests.Integration/Umbraco.Core/DeliveryApi/RequestRoutingServiceTests.cs

@@ -0,0 +1,74 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Cms.Api.Delivery.Services;
+using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.DeliveryApi;
+using Umbraco.Cms.Core.PublishedCache;
+using Umbraco.Cms.Core.Routing;
+using Umbraco.Cms.Tests.Common.Testing;
+using Umbraco.Cms.Tests.Integration.Testing;
+
+namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.DeliveryApi;
+
+[TestFixture]
+[UmbracoTest(
+    Database = UmbracoTestOptions.Database.NewSchemaPerFixture,
+    WithApplication = true)]
+public class RequestRoutingServiceTests : UmbracoIntegrationTest
+{
+    private IRequestRoutingService RequestRoutingService => GetRequiredService<IRequestRoutingService>();
+
+    protected override void CustomTestSetup(IUmbracoBuilder builder)
+    {
+        builder.Services.AddUnique<IRequestRoutingService, RequestRoutingService>();
+
+        var elementCache = new FastDictionaryAppCache();
+        var snapshotCache = new FastDictionaryAppCache();
+
+        var domainCacheMock = new Mock<IDomainCache>();
+        domainCacheMock.Setup(x => x.GetAll(It.IsAny<bool>()))
+            .Returns(
+            [
+                new Domain(1, "localhost/en", 1000, "en-us", false, 0),
+                new Domain(2, "localhost/jp", 1000, "ja-jp", false, 1),
+            ]);
+        var publishedSnapshotMock = new Mock<IPublishedSnapshot>();
+        publishedSnapshotMock.SetupGet(p => p.ElementsCache).Returns(elementCache);
+        publishedSnapshotMock.SetupGet(p => p.SnapshotCache).Returns(snapshotCache);
+        publishedSnapshotMock.SetupGet(p => p.Domains).Returns(domainCacheMock.Object);
+
+        var publishedSnapshot = publishedSnapshotMock.Object;
+        var publishedSnapshotAccessor = new Mock<IPublishedSnapshotAccessor>();
+        publishedSnapshotAccessor.Setup(p => p.TryGetPublishedSnapshot(out publishedSnapshot)).Returns(true);
+        builder.Services.AddSingleton(provider => publishedSnapshotAccessor.Object);
+    }
+
+    [TestCase(null, "")]
+    [TestCase("", "")]
+    [TestCase("/", "/")]
+    [TestCase("/en/test/", "1000/test/")]       // Verifies matching a domain.
+    [TestCase("/da/test/", "/da/test/")]        // Verifies that with no matching domain, so route will be returned as is.
+    [TestCase("/jp/オフィス/", "1000/オフィス/")] // Verifies that with a URL segment containing special characters, the route remains decoded.
+    public void GetContentRoute_ReturnsExpectedRoute(string? requestedRoute, string expectedResult)
+    {
+        if (!string.IsNullOrEmpty(requestedRoute))
+        {
+            var httpContextAccessor = GetRequiredService<IHttpContextAccessor>();
+
+            httpContextAccessor.HttpContext = new DefaultHttpContext
+            {
+                Request =
+                {
+                    Scheme = "https",
+                    Host = new HostString("localhost"),
+                    Path = requestedRoute,
+                },
+            };
+        }
+
+        var result = RequestRoutingService.GetContentRoute(requestedRoute);
+        Assert.AreEqual(expectedResult, result);
+    }
+}

tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelperTests.cs

@@ -0,0 +1,120 @@
+using Microsoft.Extensions.Options;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Infrastructure.Examine;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Tests.Common.Builders;
+using Umbraco.Cms.Tests.Common.Testing;
+using Umbraco.Cms.Tests.Integration.Testing;
+
+namespace Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Examine;
+
+[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
+[TestFixture]
+public class DeliveryApiContentIndexHelperTests : UmbracoIntegrationTestWithContent
+{
+    public override void CreateTestData()
+    {
+        base.CreateTestData();
+
+        // Save an extra, published content item of a different type to those created via the base class,
+        // that we'll use to test filtering out disallowed content types.
+        var template = TemplateBuilder.CreateTextPageTemplate("textPage2");
+        FileService.SaveTemplate(template);
+
+        var contentType = ContentTypeBuilder.CreateSimpleContentType("umbTextpage2", "Textpage2", defaultTemplateId: template.Id);
+        contentType.Key = Guid.NewGuid();
+        ContentTypeService.Save(contentType);
+
+        ContentType.AllowedContentTypes =
+        [
+            new ContentTypeSort(ContentType.Id, 0),
+            new ContentTypeSort(contentType.Id, 1),
+        ];
+        ContentTypeService.Save(ContentType);
+
+        var subpage = ContentBuilder.CreateSimpleContent(contentType, "Alternate Text Page 4", Textpage.Id);
+        ContentService.Save(subpage);
+
+        // And then add some more of the first type, so the one we'll filter out in tests isn't in the last page.
+        for (int i = 0; i < 5; i++)
+        {
+            subpage = ContentBuilder.CreateSimpleContent(ContentType, $"Text Page {5 + i}", Textpage.Id);
+            ContentService.Save(subpage);
+        }
+    }
+
+    [Test]
+    public void Can_Enumerate_Descendants_For_Content_Index()
+    {
+        var sut = CreateDeliveryApiContentIndexHelper();
+
+        var expectedNumberOfContentItems = GetExpectedNumberOfContentItems();
+
+        var contentEnumerated = 0;
+        Action<IContent[]> actionToPerform = content =>
+        {
+            contentEnumerated += content.Length;
+        };
+
+        const int pageSize = 3;
+        sut.EnumerateApplicableDescendantsForContentIndex(
+            Cms.Core.Constants.System.Root,
+            actionToPerform,
+            pageSize);
+
+        Assert.AreEqual(expectedNumberOfContentItems, contentEnumerated);
+    }
+
+    [Test]
+    public void Can_Enumerate_Descendants_For_Content_Index_With_Disallowed_Content_Type()
+    {
+        var sut = CreateDeliveryApiContentIndexHelper(["umbTextPage2"]);
+
+        var expectedNumberOfContentItems = GetExpectedNumberOfContentItems();
+
+        var contentEnumerated = 0;
+        Action<IContent[]> actionToPerform = content =>
+        {
+            contentEnumerated += content.Length;
+        };
+
+        const int pageSize = 3;
+        sut.EnumerateApplicableDescendantsForContentIndex(
+            Cms.Core.Constants.System.Root,
+            actionToPerform,
+            pageSize);
+
+        Assert.AreEqual(expectedNumberOfContentItems - 1, contentEnumerated);
+    }
+
+    private DeliveryApiContentIndexHelper CreateDeliveryApiContentIndexHelper(string[]? disallowedContentTypeAliases = null)
+    {
+        return new DeliveryApiContentIndexHelper(
+            ContentService,
+            GetRequiredService<IUmbracoDatabaseFactory>(),
+            GetDeliveryApiSettings(disallowedContentTypeAliases ?? []));
+    }
+
+    private IOptionsMonitor<DeliveryApiSettings> GetDeliveryApiSettings(string[] disallowedContentTypeAliases)
+    {
+        var deliveryApiSettings = new DeliveryApiSettings
+        {
+            DisallowedContentTypeAliases = disallowedContentTypeAliases,
+        };
+
+        var optionsMonitorMock = new Mock<IOptionsMonitor<DeliveryApiSettings>>();
+        optionsMonitorMock.Setup(o => o.CurrentValue).Returns(deliveryApiSettings);
+        return optionsMonitorMock.Object;
+    }
+
+    private int GetExpectedNumberOfContentItems()
+    {
+        var result = ContentService.GetAllPublished().Count();
+        Assert.AreEqual(10, result);
+        return result;
+    }
+}

tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/TagRepositoryTest.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
-using System.Linq;
 using Microsoft.Extensions.Logging;
 using NUnit.Framework;
 using Umbraco.Cms.Core.Cache;
@@ -1047,6 +1046,98 @@ public class TagRepositoryTest : UmbracoIntegrationTest
         }
     }
 
+    [Test]
+    public void Can_Create_Tag_Relations_With_Mixed_Casing_For_Tag()
+    {
+        var provider = ScopeProvider;
+        using (var scope = ScopeProvider.CreateScope())
+        {
+            (IContentType contentType, IContent content1, IContent content2) = CreateContentForCreateTagTests();
+
+            var repository = CreateRepository(provider);
+
+            // Note two tags are applied, but they differ only in case for the tag.
+            Tag[] tags1 = { new() { Text = "tag1", Group = "test" }, new() { Text = "Tag1", Group = "test" } };
+            repository.Assign(
+                content1.Id,
+                contentType.PropertyTypes.First().Id,
+                tags1,
+                false);
+
+            // Note the casing is different from the tag in tags1, but both should be considered equivalent.
+            Tag[] tags2 = { new() { Text = "TAG1", Group = "test" } };
+            repository.Assign(
+                content2.Id,
+                contentType.PropertyTypes.First().Id,
+                tags2,
+                false);
+
+            // Only one tag should have been saved.
+            var tagCount = scope.Database.ExecuteScalar<int>(
+                "SELECT COUNT(*) FROM cmsTags WHERE [group] = 'test'");
+            Assert.AreEqual(1, tagCount);
+
+            // Both content items should be found as tagged by the tag, even though one was assigned with the tag differing in case.
+            Assert.AreEqual(2, repository.GetTaggedEntitiesByTag(TaggableObjectTypes.Content, "tag1").Count());
+        }
+    }
+
+    [Test]
+    public void Can_Create_Tag_Relations_With_Mixed_Casing_For_Group()
+    {
+        var provider = ScopeProvider;
+        using (var scope = ScopeProvider.CreateScope())
+        {
+            (IContentType contentType, IContent content1, IContent content2) = CreateContentForCreateTagTests();
+
+            var repository = CreateRepository(provider);
+
+            // Note two tags are applied, but they differ only in case for the group.
+            Tag[] tags1 = { new() { Text = "tag1", Group = "group1" }, new() { Text = "tag1", Group = "Group1" } };
+            repository.Assign(
+                content1.Id,
+                contentType.PropertyTypes.First().Id,
+                tags1,
+                false);
+
+            // Note the casing is different from the group in tags1, but both should be considered equivalent.
+            Tag[] tags2 = { new() { Text = "tag1", Group = "GROUP1" } };
+            repository.Assign(
+                content2.Id,
+                contentType.PropertyTypes.First().Id,
+                tags2,
+                false);
+
+            // Only one tag/group should have been saved.
+            var tagCount = scope.Database.ExecuteScalar<int>(
+                "SELECT COUNT(*) FROM cmsTags WHERE [tag] = 'tag1'");
+            Assert.AreEqual(1, tagCount);
+
+            var groupCount = scope.Database.ExecuteScalar<int>(
+                "SELECT COUNT(*) FROM cmsTags WHERE [group] = 'group1'");
+            Assert.AreEqual(1, groupCount);
+
+            // Both content items should be found as tagged by the tag, even though one was assigned with the group differing in case.
+            Assert.AreEqual(2, repository.GetTaggedEntitiesByTagGroup(TaggableObjectTypes.Content, "group1").Count());
+        }
+    }
+
+    private (IContentType ContentType, IContent Content1, IContent Content2) CreateContentForCreateTagTests()
+    {
+        var template = TemplateBuilder.CreateTextPageTemplate();
+        FileService.SaveTemplate(template);
+
+        var contentType = ContentTypeBuilder.CreateSimpleContentType("test", "Test", defaultTemplateId: template.Id);
+        ContentTypeRepository.Save(contentType);
+
+        var content1 = ContentBuilder.CreateSimpleContent(contentType);
+        var content2 = ContentBuilder.CreateSimpleContent(contentType);
+        DocumentRepository.Save(content1);
+        DocumentRepository.Save(content2);
+
+        return (contentType, content1, content2);
+    }
+
     private TagRepository CreateRepository(IScopeProvider provider) =>
         new((IScopeAccessor)provider, AppCaches.Disabled, LoggerFactory.CreateLogger<TagRepository>());
 }

tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/RichTextParserTests.cs

@@ -1,4 +1,4 @@
-using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging;
 using Moq;
 using NUnit.Framework;
 using Umbraco.Cms.Core;
@@ -357,16 +357,71 @@ public class RichTextParserTests : PropertyValueConverterTests
         Assert.IsEmpty(blockLevelBlock.Elements);
     }
 
+    private const string TestParagraph = "What follows from <strong>here</strong> <em>is</em> <a href=\"#\">just</a> a bunch of text.";
+
     [Test]
     public void ParseElement_CanHandleWhitespaceAroundInlineElemements()
     {
         var parser = CreateRichTextElementParser();
 
-        var element = parser.Parse("<p>What follows from <strong>here</strong> <em>is</em> <a href=\"#\">just</a> a bunch of text.</p>") as RichTextRootElement;
+        var element = parser.Parse($"<p>{TestParagraph}</p>") as RichTextRootElement;
         Assert.IsNotNull(element);
         var paragraphElement = element.Elements.Single() as RichTextGenericElement;
         Assert.IsNotNull(paragraphElement);
 
+        AssertTestParagraph(paragraphElement);
+    }
+
+    [TestCase(1, "\n")]
+    [TestCase(2, "\n")]
+    [TestCase(1, "\r")]
+    [TestCase(2, "\r")]
+    [TestCase(1, "\r\n")]
+    [TestCase(2, "\r\n")]
+    public void ParseElement_RemovesNewLinesAroundHtmlStructuralElements(int numberOfNewLineCharacters, string newlineCharacter)
+    {
+        var parser = CreateRichTextElementParser();
+
+        var newLineSeparator = string.Concat(Enumerable.Repeat(newlineCharacter, numberOfNewLineCharacters));
+        var element = parser.Parse($"<table>{newLineSeparator}<tr>{newLineSeparator}<td>{TestParagraph}</td>{newLineSeparator}</tr>{newLineSeparator}</table>") as RichTextRootElement;
+        Assert.IsNotNull(element);
+        var tableElement = element.Elements.Single() as RichTextGenericElement;
+        Assert.IsNotNull(tableElement);
+
+        var rowElement = tableElement.Elements.Single() as RichTextGenericElement;
+        Assert.IsNotNull(rowElement);
+
+        var cellElement = rowElement.Elements.Single() as RichTextGenericElement;
+        Assert.IsNotNull(cellElement);
+
+        AssertTestParagraph(cellElement);
+    }
+
+    [TestCase(1, "\n")]
+    [TestCase(2, "\n")]
+    [TestCase(1, "\r")]
+    [TestCase(2, "\r")]
+    [TestCase(1, "\r\n")]
+    [TestCase(2, "\r\n")]
+    public void ParseElement_RemovesNewLinesAroundHtmlContentElements(int numberOfNewLineCharacters, string newlineCharacter)
+    {
+        var parser = CreateRichTextElementParser();
+
+        var newLineSeparator = string.Concat(Enumerable.Repeat(newlineCharacter, numberOfNewLineCharacters));
+        var element = parser.Parse($"<div><p>{TestParagraph}</p>{newLineSeparator}<p></p>{newLineSeparator}<p>&nbsp;</p>{newLineSeparator}<p>{TestParagraph}</p></div>") as RichTextRootElement;
+        Assert.IsNotNull(element);
+        var divElement = element.Elements.Single() as RichTextGenericElement;
+        Assert.IsNotNull(divElement);
+
+        var paragraphELements = divElement.Elements;
+        Assert.AreEqual(4, paragraphELements.Count());
+
+        AssertTestParagraph(paragraphELements.First() as RichTextGenericElement);
+        AssertTestParagraph(paragraphELements.Last() as RichTextGenericElement);
+    }
+
+    private static void AssertTestParagraph(RichTextGenericElement paragraphElement)
+    {
         var childElements = paragraphElement.Elements.ToArray();
         Assert.AreEqual(7, childElements.Length);
 

tests/Umbraco.Tests.UnitTests/Umbraco.Core/PropertyEditors/RichTextPropertyIndexValueFactoryTests.cs

@@ -0,0 +1,82 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.PropertyEditors;
+using Umbraco.Cms.Core.Serialization;
+using Umbraco.Cms.Core.Services;
+
+namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.PropertyEditors;
+
+/// <summary>
+/// Tests for <see cref="RichTextPropertyIndexValueFactory"/> to ensure it correctly creates index values from rich text properties. 
+/// </summary>
+public class RichTextPropertyIndexValueFactoryTests
+{
+    /// <summary>
+    /// Tests that the factory can create index values from a rich text property with valid content
+    /// </summary>
+    /// <param name="testContent"></param>
+    /// <param name="expected"></param>
+    [Test]
+    [TestCase("<p>Sample text</p>", "Sample text")]
+    [TestCase("<p>John Smith<br>Company ABC<br>London</p>", "John Smith Company ABC London")]
+    [TestCase("<p>John Smith<break>Company ABC<break>London</p>", "John SmithCompany ABCLondon")]
+    [TestCase("<p>John Smith<br>Company ABC<branything>London</p>", "John Smith Company ABCLondon")]
+    [TestCase("<p>Another sample text with <strong>bold</strong> content</p>", "Another sample text with bold content")]
+    [TestCase("<p>Text with <a href=\"https://example.com\">link</a></p>", "Text with link")]
+    [TestCase("<p>Text with <img src=\"image.jpg\" alt=\"image\" /></p>", "Text with")]
+    [TestCase("<p>Text with <span style=\"color: red;\">styled text</span></p>", "Text with styled text")]
+    [TestCase("<p>Text with <em>emphasized</em> content</p>", "Text with emphasized content")]
+    [TestCase("<p>Text with <u>underlined</u> content</p>", "Text with underlined content")]
+    [TestCase("<p>Text with <code>inline code</code></p>", "Text with inline code")]
+    [TestCase("<p>Text with <pre><code>code block</code></pre></p>", "Text with code block")]
+    [TestCase("<p>Text with <blockquote>quoted text</blockquote></p>", "Text with quoted text")]
+    [TestCase("<p>Text with <ul><li>list item 1</li><li>list item 2</li></ul></p>",
+        "Text with list item 1list item 2")]
+    [TestCase("<p>Text with <ol><li>ordered item 1</li><li>ordered item 2</li></ol></p>",
+        "Text with ordered item 1ordered item 2")]
+    [TestCase("<p>Text with <div class=\"class-name\">div content</div></p>", "Text with div content")]
+    [TestCase("<p>Text with <span class=\"class-name\">span content</span></p>", "Text with span content")]
+    [TestCase("<p>Text with <strong>bold</strong> and <em>italic</em> content</p>",
+        "Text with bold and italic content")]
+    [TestCase("<p>Text with <a href=\"https://example.com\" target=\"_blank\">external link</a></p>",
+        "Text with external link")]
+    [TestCase("<p>John Smith<br class=\"test\">Company ABC<br>London</p>", "John Smith Company ABC London")]
+    [TestCase("<p>John Smith<br \r\n />Company ABC<br>London</p>", "John Smith Company ABC London")]
+    public void Can_Create_Index_Values_From_RichText_Property(string testContent, string expected)
+    {
+        var propertyEditorCollection = new PropertyEditorCollection(new DataEditorCollection(() => null));
+        var jsonSerializer = Mock.Of<IJsonSerializer>();
+        var indexingSettings = Mock.Of<IOptionsMonitor<IndexingSettings>>();
+        Mock.Get(indexingSettings).Setup(x => x.CurrentValue).Returns(new IndexingSettings { });
+        var contentTypeService = Mock.Of<IContentTypeService>();
+        var logger = Mock.Of<ILogger<RichTextPropertyIndexValueFactory>>();
+        string alias = "richText";
+
+        var factory = new RichTextPropertyIndexValueFactory(
+            propertyEditorCollection,
+            jsonSerializer,
+            indexingSettings,
+            contentTypeService,
+            logger);
+
+        // create a mock property with the rich text value
+        var property = Mock.Of<IProperty>(p => p.Alias == alias
+                                               && (string)p.GetValue(It.IsAny<string>(), It.IsAny<string>(),
+                                                   It.IsAny<bool>()) == testContent);
+
+        // get the index value for the property
+        var indexValue = factory
+            .GetIndexValues(property, null, null, true, [], new Dictionary<Guid, IContentType>())
+            .FirstOrDefault(kvp => kvp.Key == alias);
+        Assert.IsNotNull(indexValue);
+
+        // assert that index the value is created correctly (it might contain a trailing whitespace, but that's OK)
+        var expectedIndexValue = indexValue.Value.SingleOrDefault() as string;
+        Assert.IsNotNull(expectedIndexValue);
+        Assert.AreEqual(expected, expectedIndexValue.TrimEnd());
+    }
+}

version.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/main/src/NerdBank.GitVersioning/version.schema.json",
-  "version": "13.9.2",
+  "version": "13.10.0-rc",
   "assemblyVersion": {
     "precision": "build"
   },