15#include "string_base.h"
31# include <unicode/ustring.h>
36#if defined(WITH_COCOA)
54void strecpy(std::span<char> dst, std::string_view src)
57 if (std::empty(dst) || std::size(src) >= std::size(dst) - 1U) {
58#if defined(STRGEN) || defined(SETTINGSGEN)
59 FatalError(
"String too long for destination buffer");
61 Debug(misc, 0,
"String too long for destination buffer");
62 src = src.substr(0, std::size(dst) - 1U);
66 auto it = std::copy(std::begin(src), std::end(src), std::begin(dst));
54void strecpy(std::span<char> dst, std::string_view src) {
…}
78 str.reserve(data.size() * 2 + 1);
81 format_append(str,
"{:02X}", b);
95 case SCC_RECORD_SEPARATOR:
116template <
class Builder>
122 if (!c.has_value()) {
129 if ((IsPrintable(*c) && (*c < SCC_SPRITE_START || *c > SCC_SPRITE_END)) ||
138 builder.PutChar(
' ');
141 builder.PutChar(
'?');
172 if (str.empty())
return;
209 if (!c.has_value())
return false;
210 if (*c == 0)
return true;
211 if (!IsPrintable(*c) || (*c >= SCC_SPRITE_START && *c <= SCC_SPRITE_END)) {
229 if (first_pos == std::string::npos) {
233 str.erase(0, first_pos);
236 str.erase(last_pos + 1);
239std::string_view StrTrimView(std::string_view str, std::string_view characters_to_trim)
241 size_t first_pos = str.find_first_not_of(characters_to_trim);
242 if (first_pos == std::string::npos) {
243 return std::string_view{};
245 size_t last_pos = str.find_last_not_of(characters_to_trim);
246 return str.substr(first_pos, last_pos - first_pos + 1);
257 if (str.size() < prefix.size())
return false;
263 static bool eq(
char c1,
char c2) {
return toupper(c1) == toupper(c2); }
264 static bool ne(
char c1,
char c2) {
return toupper(c1) != toupper(c2); }
265 static bool lt(
char c1,
char c2) {
return toupper(c1) < toupper(c2); }
267 static int compare(
const char *s1,
const char *s2,
size_t n)
270 if (toupper(*s1) < toupper(*s2))
return -1;
271 if (toupper(*s1) > toupper(*s2))
return 1;
277 static const char *find(
const char *s,
size_t n,
char a)
279 for (; n > 0; --n, ++s) {
280 if (toupper(*s) == toupper(a))
return s;
297 if (str.size() < suffix.size())
return false;
312 return ci_str1.compare(ci_str2);
323 if (str1.size() != str2.size())
return false;
338 return ci_str.find(ci_value) != ci_str.npos;
350 return std::distance(view.begin(), view.end());
353bool strtolower(std::string &str, std::string::size_type offs)
355 bool changed =
false;
356 for (
auto ch = str.begin() + offs; ch != str.end(); ++ch) {
357 auto new_ch =
static_cast<char>(tolower(
static_cast<unsigned char>(*ch)));
358 changed |= new_ch != *ch;
375 case CS_NUMERAL:
return (key >=
'0' && key <=
'9');
378 case CS_ALPHA:
return IsPrintable(key) && !(key >=
'0' && key <=
'9');
379 case CS_HEXADECIMAL:
return (key >=
'0' && key <=
'9') || (key >=
'a' && key <=
'f') || (key >=
'A' && key <=
'F');
380 default: NOT_REACHED();
391 if (c >=
'0' && c <=
'9')
return false;
392 if (c >=
'A' && c <=
'Z')
return false;
393 if (c >=
'a' && c <=
'z')
return false;
394 if (c >= SCC_CONTROL_START && c <= SCC_CONTROL_END)
return true;
395 if (c >= 0xC0 && c <= 0x10FFFF)
return false;
411 auto it = view.begin();
412 const auto end = view.end();
414 return str.substr(it.GetByteOffset());
427 if (ignore_garbage_at_front) {
434 UErrorCode status = U_ZERO_ERROR;
435 int result =
_current_collator->compareUTF8(icu::StringPiece(s1.data(), s1.size()), icu::StringPiece(s2.data(), s2.size()), status);
436 if (U_SUCCESS(status))
return result;
440#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
441 int res = OTTDStringCompare(s1, s2);
442 if (res != 0)
return res - 2;
445#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
447 if (res != 0)
return res - 2;
456#include <unicode/stsearch.h>
466static int ICUStringContains(std::string_view str, std::string_view value,
bool case_insensitive)
469 std::unique_ptr<icu::RuleBasedCollator> coll(
dynamic_cast<icu::RuleBasedCollator *
>(
_current_collator->clone()));
471 UErrorCode status = U_ZERO_ERROR;
472 coll->setStrength(case_insensitive ? icu::Collator::SECONDARY : icu::Collator::TERTIARY);
473 coll->setAttribute(UCOL_NUMERIC_COLLATION, UCOL_OFF, status);
475 auto u_str = icu::UnicodeString::fromUTF8(icu::StringPiece(str.data(), str.size()));
476 auto u_value = icu::UnicodeString::fromUTF8(icu::StringPiece(value.data(), value.size()));
477 icu::StringSearch u_searcher(u_value, u_str, coll.get(),
nullptr, status);
478 if (U_SUCCESS(status)) {
479 auto pos = u_searcher.first(status);
480 if (U_SUCCESS(status))
return pos != USEARCH_DONE ? 1 : 0;
500 if (res_u >= 0)
return res_u > 0;
503#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
505 if (res >= 0)
return res > 0;
508#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
510 if (res >= 0)
return res > 0;
513 return str.find(value) != std::string_view::npos;
527 if (res_u >= 0)
return res_u > 0;
530#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
532 if (res >= 0)
return res > 0;
535#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
537 if (res >= 0)
return res > 0;
542 return ci_str.find(ci_value) != CaseInsensitiveStringView::npos;
553 if (c >=
'0' && c <=
'9')
return c -
'0';
554 if (c >=
'A' && c <=
'F')
return c + 10 -
'A';
555 if (c >=
'a' && c <=
'f')
return c + 10 -
'a';
572 if (bytes.size() != hex.size() / 2) {
577 if (hex.size() % 2 != 0) {
581 for (
size_t i = 0; i < hex.size() / 2; i++) {
585 if (hi < 0 || lo < 0) {
589 bytes[i] = (hi << 4) | lo;
599 return std::make_unique<UniscribeStringIterator>();
602#elif defined(WITH_ICU_I18N)
604#include <unicode/utext.h>
605#include <unicode/brkiter.h>
619 UErrorCode status = U_ZERO_ERROR;
623 this->utf16_str.push_back(
'\0');
624 this->utf16_to_utf8.push_back(0);
639 this->utf16_str.clear();
640 this->utf16_to_utf8.clear();
643 for (
auto it = view.begin(), end = view.end(); it != end; ++it) {
644 size_t idx = it.GetByteOffset();
647 this->utf16_str.push_back((UChar)c);
650 this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
651 this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
652 this->utf16_to_utf8.push_back(idx);
654 this->utf16_to_utf8.push_back(idx);
656 this->utf16_str.push_back(
'\0');
657 this->utf16_to_utf8.push_back(s.size());
659 UText text = UTEXT_INITIALIZER;
660 UErrorCode status = U_ZERO_ERROR;
661 utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
662 this->char_itr->setText(&text, status);
663 this->word_itr->setText(&text, status);
664 this->char_itr->first();
665 this->word_itr->first();
672 for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
673 if (this->utf16_to_utf8[i] == pos) {
682 this->char_itr->isBoundary(utf16_pos);
683 return this->utf16_to_utf8[this->char_itr->current()];
691 pos = this->char_itr->next();
695 pos = this->word_itr->following(this->char_itr->current());
699 while (pos != icu::BreakIterator::DONE &&
701 int32_t new_pos = this->word_itr->next();
704 if (new_pos == icu::BreakIterator::DONE)
break;
708 this->char_itr->isBoundary(pos);
715 return pos == icu::BreakIterator::DONE ?
END : this->utf16_to_utf8[pos];
723 pos = this->char_itr->previous();
727 pos = this->word_itr->preceding(this->char_itr->current());
731 while (pos != icu::BreakIterator::DONE &&
733 int32_t new_pos = this->word_itr->previous();
736 if (new_pos == icu::BreakIterator::DONE)
break;
740 this->char_itr->isBoundary(pos);
747 return pos == icu::BreakIterator::DONE ?
END : this->utf16_to_utf8[pos];
753 return std::make_unique<IcuStringIterator>();
765 void SetString(std::string_view s)
override
768 this->cur_pos = this->
string.begin();
773 this->cur_pos = this->
string.GetIterAtByte(pos);
774 return this->cur_pos.GetByteOffset();
777 size_t Next(IterType what)
override
779 const auto end = this->
string.end();
781 if (this->cur_pos >= end)
return END;
786 return this->cur_pos.GetByteOffset();
790 while (this->cur_pos != end && !
IsWhitespace(*this->cur_pos)) {
794 while (this->cur_pos != end &&
IsWhitespace(*this->cur_pos)) {
797 return this->cur_pos.GetByteOffset();
806 size_t Prev(IterType what)
override
808 const auto begin = this->
string.begin();
810 if (this->cur_pos == begin)
return END;
815 return this->cur_pos.GetByteOffset();
821 }
while (this->cur_pos != begin &&
IsWhitespace(*this->cur_pos));
823 while (this->cur_pos != begin && !
IsWhitespace(*this->cur_pos)) {
828 return this->cur_pos.GetByteOffset();
838#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
841 std::unique_ptr<StringIterator> i = OSXStringIterator::Create();
842 if (i !=
nullptr)
return i;
844 return std::make_unique<DefaultStringIterator>();
849 return std::make_unique<DefaultStringIterator>();
860std::optional<std::string_view>
GetEnv(
const char *variable)
862 auto val = std::getenv(variable);
863 if (val ==
nullptr || *val ==
'\0')
return std::nullopt;
860std::optional<std::string_view>
GetEnv(
const char *variable) {
…}
void PutChar(char c)
Append 8-bit char.
String iterator using ICU as a backend.
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
void SetString(std::string_view s) override
Set a new iteration string.
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
std::vector< UChar > utf16_str
UTF-16 copy of the string.
icu::BreakIterator * char_itr
ICU iterator for characters.
icu::BreakIterator * word_itr
ICU iterator for words.
bool AnyBytesUnused() const noexcept
Check whether any unused bytes are left between the Builder and Consumer position.
size_type GetBytesWritten() const noexcept
Get number of already written bytes.
Compose data into a fixed size buffer, which is consumed at the same time.
InPlaceBuilder builder
Builder into shared buffer.
StringConsumer consumer
Consumer from shared buffer.
Compose data into a growing std::string.
Parse data from a string / buffer.
bool AnyBytesLeft() const noexcept
Check whether any bytes left to read.
bool PeekCharIf(char c) const
Check whether the next 8-bit char matches 'c'.
static const std::string_view WHITESPACE_NO_NEWLINE
ASCII whitespace characters, excluding new-line.
std::optional< char32_t > TryReadUtf8()
Try to read a UTF-8 character, and then advance reader.
void Skip(size_type len)
Discard some bytes.
Class for iterating over different kind of parts of a string.
static const size_t END
Sentinel to indicate end-of-iteration.
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
static std::unique_ptr< StringIterator > Create()
Create a new iterator instance.
IterType
Type of the iterator.
@ ITER_WORD
Iterate over words.
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
virtual void SetString(std::string_view s)=0
Set a new iteration string.
Bidirectional input iterator over codepoints.
Constant span of UTF-8 encoded data.
Control codes that are embedded in the translation strings.
@ SCC_ENCODED
Encoded string marker and sub-string parameter.
@ SCC_ENCODED_NUMERIC
Encoded numeric parameter.
@ SCC_ENCODED_STRING
Encoded string parameter.
@ SCC_ENCODED_INTERNAL
Encoded text from OpenTTD.
Functions related to debugging.
#define Debug(category, level, format_string,...)
Output a line of debugging information.
Error reporting related functions.
fluid_settings_t * settings
FluidSynth settings handle.
Functions related to the gfx engine.
Information about languages and their files.
const LanguageMetadata * _current_language
The currently loaded language.
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
A number of safeguards to prevent using unsafe methods.
Definition of base types and functions in a cross-platform compatible way.
bool ConvertHexToBytes(std::string_view hex, std::span< uint8_t > bytes)
Convert a hex-string to a byte-array, while validating it was actually hex.
bool StrNaturalContainsIgnoreCase(std::string_view str, std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case insensi...
size_t Utf8StringLength(std::string_view str)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
static int ICUStringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
bool StrNaturalContains(std::string_view str, std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case sensiti...
bool IsValidChar(char32_t key, CharSetFilter afilter)
Only allow certain keys.
std::optional< std::string_view > GetEnv(const char *variable)
Get the environment variable using std::getenv and when it is an empty string (or nullptr),...
void StrMakeValidInPlace(char *str, StringValidationSettings settings)
Scans the string for invalid characters and replaces them with a question mark '?' (if not ignored).
void strecpy(std::span< char > dst, std::string_view src)
Copies characters from one buffer to another.
std::string FormatArrayAsHex(std::span< const uint8_t > data)
Format a byte array into a continuous hex string.
bool StrEqualsIgnoreCase(std::string_view str1, std::string_view str2)
Compares two string( view)s for equality, while ignoring the case of the characters.
bool StrEndsWithIgnoreCase(std::string_view str, std::string_view suffix)
Check whether the given string ends with the given suffix, ignoring case.
bool StrValid(std::span< const char > str)
Checks whether the given string is valid, i.e.
static int ConvertHexNibbleToByte(char c)
Convert a single hex-nibble to a byte.
static std::string_view SkipGarbage(std::string_view str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
static bool IsSccEncodedCode(char32_t c)
Test if a character is (only) part of an encoded string.
int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
std::basic_string_view< char, CaseInsensitiveCharTraits > CaseInsensitiveStringView
Case insensitive string view.
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
bool StrStartsWithIgnoreCase(std::string_view str, std::string_view prefix)
Check whether the given string starts with the given prefix, ignoring case.
static void StrMakeValid(Builder &builder, StringConsumer &consumer, StringValidationSettings settings)
Copies the valid (UTF-8) characters from consumer to the builder.
int StrCompareIgnoreCase(std::string_view str1, std::string_view str2)
Compares two string( view)s, while ignoring the case of the characters.
static bool IsGarbageCharacter(char32_t c)
Test if a unicode character is considered garbage to be skipped.
bool StrContainsIgnoreCase(std::string_view str, std::string_view value)
Checks if a string is contained in another string, while ignoring the case of the characters.
Functions related to low-level strings.
char32_t Utf16DecodeChar(const uint16_t *c)
Decode an UTF-16 character.
bool IsWhitespace(char32_t c)
Check whether UNICODE character is whitespace or not, i.e.
Inplace-replacement of textual and binary data.
int MacOSStringCompare(std::string_view s1, std::string_view s2)
Compares two strings using case insensitive natural sort.
int MacOSStringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Functions related to localized text support on OSX.
@ ReplaceWithQuestionMark
Replace the unknown/bad bits with question marks.
@ AllowControlCode
Allow the special control codes.
@ AllowNewline
Allow newlines; replaces '\r ' with ' ' during processing.
@ ReplaceTabCrNlWithSpace
Replace tabs ('\t'), carriage returns ('\r') and newlines (' ') with spaces.
CharSetFilter
Valid filter types for IsValidChar.
@ CS_NUMERAL_SPACE
Only numbers and spaces.
@ CS_HEXADECIMAL
Only hexadecimal characters.
@ CS_NUMERAL
Only numeric ones.
@ CS_NUMERAL_SIGNED
Only numbers and '-' for negative values.
@ CS_ALPHA
Only alphabetic values.
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
Functions related to laying out text on Win32.
Case insensitive implementation of the standard character type traits.
Handling of UTF-8 encoded data.
int Win32StringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
declarations of functions for MS windows systems