Handling of C-type strings (char*). More...

#include "stdafx.h"
#include "debug.h"
#include "core/alloc_func.hpp"
#include "core/math_func.hpp"
#include "error_func.h"
#include "string_func.h"
#include "string_base.h"
#include "table/control_codes.h"
#include <sstream>
#include <iomanip>
#include <unicode/ustring.h>
#include "language.h"
#include "gfx_func.h"
#include "safeguards.h"
#include <unicode/stsearch.h>
#include <unicode/utext.h>
#include <unicode/brkiter.h>

Data Structures
struct	CaseInsensitiveCharTraits
	Case insensitive implementation of the standard character type traits. More...

class	IcuStringIterator
	String iterator using ICU as a backend. More...

Typedefs
typedef std::basic_string_view< char, CaseInsensitiveCharTraits >	CaseInsensitiveStringView
	Case insensitive string view.

Functions
void	strecpy (std::span< char > dst, std::string_view src)
	Copies characters from one buffer to another.

std::string	FormatArrayAsHex (std::span< const uint8_t > data)
	Format a byte array into a continuous hex string.

template<class T >
static void	StrMakeValid (T &dst, const char str, const char last, StringValidationSettings settings)
	Copies the valid (UTF-8) characters from `str` up to `last` to the `dst`.

void	StrMakeValidInPlace (char str, const char last, StringValidationSettings settings)
	Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).

void	StrMakeValidInPlace (char *str, StringValidationSettings settings)
	Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).

std::string	StrMakeValid (std::string_view str, StringValidationSettings settings)
	Copies the valid (UTF-8) characters from `str` to the returned string.

bool	StrValid (std::span< const char > str)
	Checks whether the given string is valid, i.e.

void	StrTrimInPlace (std::string &str)
	Trim the spaces from given string in place, i.e.

std::string_view	StrTrimView (std::string_view str)

bool	StrStartsWithIgnoreCase (std::string_view str, const std::string_view prefix)
	Check whether the given string starts with the given prefix, ignoring case.

bool	StrEndsWithIgnoreCase (std::string_view str, const std::string_view suffix)
	Check whether the given string ends with the given suffix, ignoring case.

int	StrCompareIgnoreCase (const std::string_view str1, const std::string_view str2)
	Compares two string( view)s, while ignoring the case of the characters.

bool	StrEqualsIgnoreCase (const std::string_view str1, const std::string_view str2)
	Compares two string( view)s for equality, while ignoring the case of the characters.

size_t	Utf8StringLength (const char *s)
	Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes that the encoded string contains.

size_t	Utf8StringLength (const std::string &str)
	Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes that the encoded string contains.

bool	strtolower (std::string &str, std::string::size_type offs)

bool	IsValidChar (char32_t key, CharSetFilter afilter)
	Only allow certain keys.

size_t	Utf8Decode (char32_t c, const char s)
	Decode and consume the next UTF-8 encoded character.

template<class T >
size_t	Utf8Encode (T buf, char32_t c)
	Encode a unicode character and place it in the buffer.

size_t	Utf8Encode (char *buf, char32_t c)

size_t	Utf8Encode (std::ostreambuf_iterator< char > &buf, char32_t c)

size_t	Utf8Encode (std::back_insert_iterator< std::string > &buf, char32_t c)

size_t	Utf8TrimString (char *s, size_t maxlen)
	Properly terminate an UTF8 string to some maximum length.

static bool	IsGarbageCharacter (char32_t c)
	Test if a unicode character is considered garbage to be skipped.

static std::string_view	SkipGarbage (std::string_view str)
	Skip some of the 'garbage' in the string that we don't want to use to sort on.

int	StrNaturalCompare (std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
	Compares two strings using case insensitive natural sort.

static int	ICUStringContains (const std::string_view str, const std::string_view value, bool case_insensitive)
	Search if a string is contained in another string using the current locale.

bool	StrNaturalContains (const std::string_view str, const std::string_view value)
	Checks if a string is contained in another string with a locale-aware comparison that is case sensitive.

bool	StrNaturalContainsIgnoreCase (const std::string_view str, const std::string_view value)
	Checks if a string is contained in another string with a locale-aware comparison that is case insensitive.

static int	ConvertHexNibbleToByte (char c)
	Convert a single hex-nibble to a byte.

bool	ConvertHexToBytes (std::string_view hex, std::span< uint8_t > bytes)
	Convert a hex-string to a byte-array, while validating it was actually hex.

Detailed Description

Handling of C-type strings (char*).

Definition in file string.cpp.

Typedef Documentation

◆ CaseInsensitiveStringView

typedef std::basic_string_view<char, CaseInsensitiveCharTraits> CaseInsensitiveStringView

Case insensitive string view.

Definition at line 313 of file string.cpp.

Function Documentation

◆ ConvertHexNibbleToByte()

static int ConvertHexNibbleToByte ( char c )

static

Convert a single hex-nibble to a byte.

Parameters

c	The hex-nibble to convert.

Returns: The byte the hex-nibble represents, or -1 if it is not a valid hex-nibble.

Definition at line 715 of file string.cpp.

Referenced by ConvertHexToBytes().

◆ ConvertHexToBytes()

bool ConvertHexToBytes	(	std::string_view	hex,
		std::span< uint8_t >	bytes
	)

Convert a hex-string to a byte-array, while validating it was actually hex.

Parameters

hex	The hex-string to convert.
bytes	The byte-array to write the result to.

Note: The length of the hex-string has to be exactly twice that of the length of the byte-array, otherwise conversion will fail.

Returns: True iff the hex-string was valid and the conversion succeeded.

Definition at line 734 of file string.cpp.

References ConvertHexNibbleToByte().

Referenced by X25519AuthorizedKeyClientHandler::GetValidSecretKeyAndUpdatePublicKey(), GRFLoadConfig(), PickerLoadConfig(), and ValidateSignature().

◆ FormatArrayAsHex()

std::string FormatArrayAsHex ( std::span< const uint8_t > data )

Format a byte array into a continuous hex string.

Parameters

data	Array to format

Returns: Converted string.

Definition at line 81 of file string.cpp.

Referenced by AddGrfInfo(), CalculateHashV1(), GenerateUid(), X25519AuthenticationHandler::GetPeerPublicKey(), X25519AuthorizedKeyClientHandler::GetValidSecretKeyAndUpdatePublicKey(), HandleSavegameLoadCrash(), IsGoodGRFConfigList(), CommandHelperBase::LogCommandExecution(), MakePNGImage(), NetworkContentListWindow::OpenExternalSearch(), OutputContentState(), ClientNetworkGameSocketHandler::Receive_SERVER_CHECK_NEWGRFS(), and SurveyGrfs().

◆ ICUStringContains()

static int ICUStringContains	(	const std::string_view	str,
		const std::string_view	value,
		bool	case_insensitive
	)

static

Search if a string is contained in another string using the current locale.

Parameters

str	String to search in.
value	String to search for.
case_insensitive	Search case-insensitive.

Returns: 1 if value was found, 0 if it was not found, or -1 if not supported by the OS.

Definition at line 630 of file string.cpp.

References _current_collator.

Referenced by StrNaturalContains(), and StrNaturalContainsIgnoreCase().

◆ IsGarbageCharacter()

static bool IsGarbageCharacter ( char32_t c )

static

Test if a unicode character is considered garbage to be skipped.

Parameters

c	Character to test.

Returns: true iff the character should be skipped.

Definition at line 549 of file string.cpp.

Referenced by SkipGarbage().

◆ IsValidChar()

bool IsValidChar	(	char32_t	key,
		CharSetFilter	afilter
	)

Only allow certain keys.

You can define the filter to be used. This makes sure no invalid keys can get into an editbox, like BELL.

Parameters

key	character to be checked
afilter	the filter to use

Returns: true or false depending if the character is printable/valid or not

Definition at line 396 of file string.cpp.

References CS_ALPHA, CS_ALPHANUMERAL, CS_HEXADECIMAL, CS_NUMERAL, CS_NUMERAL_SIGNED, and CS_NUMERAL_SPACE.

Referenced by IConsoleCmdExec(), Textbuf::InsertString(), OskWindow::OnClick(), VideoDriver_SDL_Base::PollEvent(), TranslateTTDPatchCodes(), and OskWindow::UpdateOskState().

◆ SkipGarbage()

static std::string_view SkipGarbage ( std::string_view str )

static

Skip some of the 'garbage' in the string that we don't want to use to sort on.

This way the alphabetical sorting will work better as we would be actually using those characters instead of some other characters such as spaces and tildes at the begin of the name.

Parameters

str	The string to skip the initial garbage of.

Returns: The string with the garbage skipped.

Definition at line 568 of file string.cpp.

References IsGarbageCharacter(), and Utf8Decode().

Referenced by StrNaturalCompare().

◆ StrCompareIgnoreCase()

int StrCompareIgnoreCase	(	const std::string_view	str1,
		const std::string_view	str2
	)

Compares two string( view)s, while ignoring the case of the characters.

Parameters

str1	The first string.
str2	The second string.

Returns: Less than zero if str1 < str2, zero if str1 == str2, greater than zero if str1 > str2. All ignoring the case of the characters.

Definition at line 334 of file string.cpp.

Referenced by GRFFileScanner::AddFile(), MatchesExtension(), StrEqualsIgnoreCase(), and StrNaturalCompare().

◆ strecpy()

void strecpy	(	std::span< char >	dst,
		std::string_view	src
	)

Copies characters from one buffer to another.

Copies the source string to the destination buffer with respect of the terminating null-character and the size of the destination buffer.

Note: usage: strecpy(dst, src);

Parameters

dst	The destination buffer
src	The buffer containing the string to copy

Definition at line 60 of file string.cpp.

References Debug.

Referenced by FileStringReader::HandlePragma(), and StringReader::ParseFile().

◆ StrEndsWithIgnoreCase()

bool StrEndsWithIgnoreCase	(	std::string_view	str,
		const std::string_view	suffix
	)

Check whether the given string ends with the given suffix, ignoring case.

Parameters

str	The string to look at.
suffix	The suffix to look for.

Returns: True iff the end of the string is the same as the suffix, ignoring case.

Definition at line 321 of file string.cpp.

References StrEqualsIgnoreCase().

Referenced by TextfileWindow::AfterLoadText(), and TextfileWindow::NavigateToFile().

◆ StrEqualsIgnoreCase()

bool StrEqualsIgnoreCase	(	const std::string_view	str1,
		const std::string_view	str2
	)

Compares two string( view)s for equality, while ignoring the case of the characters.

Parameters

str1	The first string.
str2	The second string.

Returns: True iff both strings are equal, barring the case of the characters.

Definition at line 347 of file string.cpp.

References StrCompareIgnoreCase().

Referenced by ChangeWorkingDirectoryToExecutable(), DEF_CONSOLE_CMD(), AIScannerInfo::FindInfo(), GameScannerInfo::FindInfo(), FindKey(), FiosGetSavegameListCallback(), FiosGetScenarioListCallback(), FiosGetTownDataListCallback(), FiosMakeFilename(), FiosNumberedSaveName::FiosNumberedSaveName(), BlitterFactory::GetBlitterFactory(), GetFontByFaceName(), IsSameScript(), ParseCode(), ScriptScanner::RegisterScript(), DriverFactoryBase::SelectDriverImpl(), StrEndsWithIgnoreCase(), StringToContentType(), and StrStartsWithIgnoreCase().

◆ StrMakeValid() [1/2]

std::string StrMakeValid	(	std::string_view	str,
		StringValidationSettings	settings
	)

Copies the valid (UTF-8) characters from str to the returned string.

Depending on the settings invalid characters can be replaced with a question mark, as well as determining what characters are deemed invalid.

Parameters

str	The string to validate.
settings	The settings for the string validation.

Definition at line 205 of file string.cpp.

References settings, and StrMakeValid().

◆ StrMakeValid() [2/2]

template<class T >

static void StrMakeValid	(	T &	dst,
		const char *	str,
		const char *	last,
		StringValidationSettings	settings
	)

static

Copies the valid (UTF-8) characters from str up to last to the dst.

Depending on the settings invalid characters can be replaced with a question mark, as well as determining what characters are deemed invalid.

It is allowed for dst to be the same as src, in which case the string is make valid in place.

Parameters

dst	The destination to write to.
str	The string to validate.
last	The last valid character of str.
settings	The settings for the string validation.

Definition at line 107 of file string.cpp.

References settings, SVS_ALLOW_CONTROL_CODE, SVS_ALLOW_NEWLINE, SVS_REPLACE_TAB_CR_NL_WITH_SPACE, SVS_REPLACE_WITH_QUESTION_MARK, Utf8Decode(), and Utf8EncodedCharLen().

Referenced by FiosFileScanner::AddFile(), ScriptInfo::AddLabels(), ScriptInfo::AddSetting(), DrawNewsString(), ErrorUnknownCallbackResult(), ExtractString(), GetFileTitle(), GetMusicCatEntryName(), GlobalVarChangeInfo(), IConsolePrint(), IniGroup::IniGroup(), IniItem::IniItem(), IniLoadFile::LoadFromDisk(), LoadFromHighScore(), LoadNewGRFSound(), ScriptInstance::LoadObjects(), TextfileWindow::LoadText(), StringSettingDesc::MakeValueValid(), ReadDWordAsString(), Packet::Recv_string(), SanitizeSingleStringHelper(), ShowNewGrfVehicleError(), SlStdString(), StrMakeValid(), and StrMakeValidInPlace().

◆ StrMakeValidInPlace() [1/2]

void StrMakeValidInPlace	(	char *	str,
		const char *	last,
		StringValidationSettings	settings
	)

Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).

Parameters

str	The string to validate.
last	The last valid character of str.
settings	The settings for the string validation.

Definition at line 178 of file string.cpp.

References settings, and StrMakeValid().

Referenced by Textbuf::Assign(), and StrMakeValidInPlace().

◆ StrMakeValidInPlace() [2/2]

void StrMakeValidInPlace	(	char *	str,
		StringValidationSettings	settings
	)

Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).

Only use this function when you are sure the string ends with a '\0'; otherwise use StrMakeValidInPlace(str, last, settings) variant.

Parameters

str	The string (of which you are sure ends with '\0') to validate.

Definition at line 192 of file string.cpp.

References settings, and StrMakeValidInPlace().

◆ StrNaturalCompare()

int StrNaturalCompare	(	std::string_view	s1,
		std::string_view	s2,
		bool	ignore_garbage_at_front
	)

Compares two strings using case insensitive natural sort.

Parameters

s1	First string to compare.
s2	Second string to compare.
ignore_garbage_at_front	Skip punctuation characters in the front

Returns: Less than zero if s1 < s2, zero if s1 == s2, greater than zero if s1 > s2.

Definition at line 589 of file string.cpp.

References _current_collator, MacOSStringCompare(), SkipGarbage(), and StrCompareIgnoreCase().

Referenced by BuildGuiGroupList(), CargoSpecNameSorter(), EngineNameSorter(), GRFSorter(), IndustryDirectoryWindow::IndustryNameSorter(), IndustryTypeNameSorter(), NetworkContentListWindow::NameSorter(), NewGRFWindow::NameSorter(), DropDownString< TBase, TFs, TEnd >::NatSortFunc(), NetworkGameWindow::NGameNameSorter(), FiosItem::operator<(), SignList::SignNameSorter(), CompanyStationsWindow::StationNameSorter(), TownDirectoryWindow::TownNameSorter(), NetworkContentListWindow::TypeSorter(), and VehicleNameSorter().

◆ StrNaturalContains()

bool StrNaturalContains	(	const std::string_view	str,
		const std::string_view	value
	)

Checks if a string is contained in another string with a locale-aware comparison that is case sensitive.

Parameters

str	The string to search in.
value	The string to search for.

Returns: True if a match was found.

Definition at line 660 of file string.cpp.

References ICUStringContains(), MacOSStringContains(), and Win32StringContains().

Referenced by StringFilter::AddLine().

◆ StrNaturalContainsIgnoreCase()

bool StrNaturalContainsIgnoreCase	(	const std::string_view	str,
		const std::string_view	value
	)

Checks if a string is contained in another string with a locale-aware comparison that is case insensitive.

Parameters

str	The string to search in.
value	The string to search for.

Returns: True if a match was found.

Definition at line 687 of file string.cpp.

References ICUStringContains(), MacOSStringContains(), and Win32StringContains().

Referenced by StringFilter::AddLine().

◆ StrStartsWithIgnoreCase()

bool StrStartsWithIgnoreCase	(	std::string_view	str,
		const std::string_view	prefix
	)

Check whether the given string starts with the given prefix, ignoring case.

Parameters

str	The string to look at.
prefix	The prefix to look for.

Returns: True iff the begin of the string is the same as the prefix, ignoring case.

Definition at line 281 of file string.cpp.

References StrEqualsIgnoreCase().

◆ strtolower()

bool strtolower	(	std::string &	str,
		std::string::size_type	offs
	)

Definition at line 378 of file string.cpp.

◆ StrTrimInPlace()

void StrTrimInPlace ( std::string & str )

Trim the spaces from given string in place, i.e.

the string buffer that is passed will be modified whenever spaces exist in the given string. When there are spaces at the begin, the whole string is moved forward and when there are spaces at the back the '\0' termination is moved.

Parameters

str	The string to perform the in place trimming on.

Definition at line 260 of file string.cpp.

Referenced by TarScanner::AddFile(), CheckClientAndServerName(), NetworkValidateClientName(), and NetworkValidateServerName().

◆ StrTrimView()

std::string_view StrTrimView ( std::string_view str )

Definition at line 265 of file string.cpp.

◆ StrValid()

bool StrValid ( std::span< const char > str )

Checks whether the given string is valid, i.e.

contains only valid (printable) characters and is properly terminated.

Note: std::span is used instead of std::string_view as we are validating fixed-length string buffers, and std::string_view's constructor will assume a C-string that ends with a NUL terminator, which is one of the things we are checking.

Parameters

str	Span of chars to validate.

Definition at line 227 of file string.cpp.

References Utf8Decode(), and Utf8EncodedCharLen().

Referenced by LanguagePackHeader::IsValid().

◆ Utf8Decode()

size_t Utf8Decode	(	char32_t *	c,
		const char *	s
	)

Decode and consume the next UTF-8 encoded character.

Parameters

c	Buffer to place decoded character.
s	Character stream to retrieve character from.

Returns: Number of characters in the sequence.

Definition at line 419 of file string.cpp.

References GB(), and HasBit().

Referenced by Textbuf::DeleteChar(), FixSCCEncoded(), FormatString(), GlobalVarChangeInfo(), VideoDriver_SDL_Base::PollEvent(), StringFilter::SetFilterTerm(), SkipGarbage(), StrMakeValid(), StrValid(), and TranslateTTDPatchCodes().

◆ Utf8Encode() [1/4]

size_t Utf8Encode	(	char *	buf,
		char32_t	c
	)

Definition at line 486 of file string.cpp.

◆ Utf8Encode() [2/4]

size_t Utf8Encode	(	std::back_insert_iterator< std::string > &	buf,
		char32_t	c
	)

Definition at line 496 of file string.cpp.

◆ Utf8Encode() [3/4]

size_t Utf8Encode	(	std::ostreambuf_iterator< char > &	buf,
		char32_t	c
	)

Definition at line 491 of file string.cpp.

◆ Utf8Encode() [4/4]

template<class T >

size_t Utf8Encode	(	T	buf,
		char32_t	c
	)

inline

Encode a unicode character and place it in the buffer.

Template Parameters

T	Type of the buffer.

Parameters

buf	Buffer to place character.
c	Unicode character to encode.

Returns: Number of characters in the encoded sequence.

Definition at line 460 of file string.cpp.

References GB().

Referenced by CheckForMissingGlyphs(), CopyFromOldName(), TextfileWindow::FindHyperlinksInMarkdown(), FixSCCEncoded(), UnmappedChoiceList::Flush(), FormatString(), Textbuf::InsertChar(), and TranslateTTDPatchCodes().

◆ Utf8StringLength() [1/2]

size_t Utf8StringLength ( const char * s )

Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes that the encoded string contains.

Parameters

s	The string to get the length for.

Returns: The length of the string in characters.

Definition at line 359 of file string.cpp.

Referenced by Textbuf::Assign(), CmdAlterGroup(), CmdFoundTown(), CmdPlaceSign(), CmdRenameCompany(), CmdRenameDepot(), CmdRenameEngine(), CmdRenamePresident(), CmdRenameSign(), CmdRenameStation(), CmdRenameTown(), CmdRenameVehicle(), CmdRenameWaypoint(), GenerateCompanyName(), GeneratePresidentName(), Utf8StringLength(), and VerifyTownName().

◆ Utf8StringLength() [2/2]

size_t Utf8StringLength ( const std::string & str )

Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes that the encoded string contains.

Parameters

s	The string to get the length for.

Returns: The length of the string in characters.

Definition at line 373 of file string.cpp.

References Utf8StringLength().

◆ Utf8TrimString()

size_t Utf8TrimString	(	char *	s,
		size_t	maxlen
	)

Properly terminate an UTF8 string to some maximum length.

Parameters

s	string to check if it needs additional trimming
maxlen	the maximum length the buffer can have.

Returns: the new length in bytes of the string (eg. strlen(new_string))

Note: maxlen is the string length INCLUDING the terminating '\0'

Definition at line 508 of file string.cpp.

References Utf8EncodedCharLen().

Data Structures

Typedefs

Functions

Detailed Description

Typedef Documentation

◆ CaseInsensitiveStringView

Function Documentation

◆ ConvertHexNibbleToByte()

◆ ConvertHexToBytes()

◆ FormatArrayAsHex()

◆ ICUStringContains()

◆ IsGarbageCharacter()

◆ IsValidChar()

◆ SkipGarbage()

◆ StrCompareIgnoreCase()

◆ strecpy()

◆ StrEndsWithIgnoreCase()

◆ StrEqualsIgnoreCase()

◆ StrMakeValid() [1/2]

◆ StrMakeValid() [2/2]

◆ StrMakeValidInPlace() [1/2]

◆ StrMakeValidInPlace() [2/2]

◆ StrNaturalCompare()

◆ StrNaturalContains()

◆ StrNaturalContainsIgnoreCase()

◆ StrStartsWithIgnoreCase()

◆ strtolower()

◆ StrTrimInPlace()

◆ StrTrimView()

◆ StrValid()

◆ Utf8Decode()

◆ Utf8Encode() [1/4]

◆ Utf8Encode() [2/4]

◆ Utf8Encode() [3/4]

◆ Utf8Encode() [4/4]

◆ Utf8StringLength() [1/2]

◆ Utf8StringLength() [2/2]

◆ Utf8TrimString()