Allow marks after NFC normalization

Includes regression test for the Tamil example from #3618
This commit is contained in:
Vyr Cossont 2025-01-27 09:41:10 -08:00
commit 13a2573ca2
5 changed files with 29 additions and 22 deletions

View file

@ -19,19 +19,16 @@ package text
import "unicode"
func isPlausiblyInHashtag(r rune) bool {
// Marks are allowed during parsing
// prior to normalization, but not after,
// since they may be combined into letters
// during normalization.
return unicode.IsMark(r) ||
isPermittedInHashtag(r)
}
func isPermittedInHashtag(r rune) bool {
return unicode.IsLetter(r) ||
unicode.IsNumber(r) ||
r == '_'
isPermittedIfNotEntireHashtag(r)
}
// isPermittedIfNotEntireHashtag is true for characters that may be in a hashtag
// but are not allowed to be the only characters making up the hashtag.
func isPermittedIfNotEntireHashtag(r rune) bool {
return unicode.IsMark(r) || r == '_'
}
// isHashtagBoundary returns true if rune r