fileio.cpp

Go to the documentation of this file.

/*
 * This file is part of OpenTTD.
 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
 */

 
#include "stdafx.h"
#include "core/string_consumer.hpp"
#include "fileio_func.h"
#include "spriteloader/spriteloader.hpp"
#include "debug.h"
#include "fios.h"
#include "string_func.h"
#include "tar_type.h"
#ifdef _WIN32
#include <windows.h>
#elif defined(__HAIKU__)
#include <Path.h>
#include <storage/FindDirectory.h>
#else
#include <unistd.h>
#include <pwd.h>
#endif
#include <sys/stat.h>
#include <filesystem>
 
#include "safeguards.h"

static bool _do_scan_working_directory = true;
 
extern std::string _config_file;
extern std::string _highscore_file;
 
static const std::string_view _subdirs[] = {
  "",
  "save" PATHSEP,
  "save" PATHSEP "autosave" PATHSEP,
  "scenario" PATHSEP,
  "scenario" PATHSEP "heightmap" PATHSEP,
  "gm" PATHSEP,
  "data" PATHSEP,
  "baseset" PATHSEP,
  "newgrf" PATHSEP,
  "lang" PATHSEP,
  "ai" PATHSEP,
  "ai" PATHSEP "library" PATHSEP,
  "game" PATHSEP,
  "game" PATHSEP "library" PATHSEP,
  "screenshot" PATHSEP,
  "social_integration" PATHSEP,
  "docs" PATHSEP,
};
static_assert(lengthof(_subdirs) == NUM_SUBDIRS);

std::array<std::string, NUM_SEARCHPATHS> _searchpaths;
std::vector<Searchpath> _valid_searchpaths;
std::array<TarList, NUM_SUBDIRS> _tar_list;
TarFileList _tar_filelist[NUM_SUBDIRS];

static bool IsValidSearchPath(Searchpath sp)
{
  return sp < _searchpaths.size() && !_searchpaths[sp].empty();
}
 
static void FillValidSearchPaths(bool only_local_path)
{
  _valid_searchpaths.clear();
 
  std::set<std::string> seen{};
  for (Searchpath sp = SP_FIRST_DIR; sp < NUM_SEARCHPATHS; sp++) {
    if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
 
    if (only_local_path) {
      switch (sp) {
        case SP_WORKING_DIR:      // Can be influence by "-c" option.
        case SP_BINARY_DIR:       // Most likely contains all the language files.
        case SP_AUTODOWNLOAD_DIR: // Otherwise we cannot download in-game content.
          break;
 
        default:
          continue;
      }
    }
 
    if (IsValidSearchPath(sp)) {
      if (seen.count(_searchpaths[sp]) != 0) continue;
      seen.insert(_searchpaths[sp]);
      _valid_searchpaths.emplace_back(sp);
    }
  }
 
  /* The working-directory is special, as it is controlled by _do_scan_working_directory.
   * Only add the search path if it isn't already in the set. To preserve the same order
   * as the enum, insert it in the front. */
  if (IsValidSearchPath(SP_WORKING_DIR) && seen.count(_searchpaths[SP_WORKING_DIR]) == 0) {
    _valid_searchpaths.insert(_valid_searchpaths.begin(), SP_WORKING_DIR);
  }
}

bool FioCheckFileExists(std::string_view filename, Subdirectory subdir)
{
  auto f = FioFOpenFile(filename, "rb", subdir);
  return f.has_value();
}

bool FileExists(std::string_view filename)
{
  std::error_code ec;
  return std::filesystem::exists(OTTD2FS(filename), ec);
}

std::string FioFindFullPath(Subdirectory subdir, std::string_view filename)
{
  assert(subdir < NUM_SUBDIRS);
 
  for (Searchpath sp : _valid_searchpaths) {
    std::string buf = FioGetDirectory(sp, subdir);
    buf += filename;
    if (FileExists(buf)) return buf;
#if !defined(_WIN32)
    /* Be, as opening files, aware that sometimes the filename
     * might be in uppercase when it is in lowercase on the
     * disk. Of course Windows doesn't care about casing. */
    if (strtolower(buf, _searchpaths[sp].size() - 1) && FileExists(buf)) return buf;
#endif
  }
 
  return {};
}
 
std::string FioGetDirectory(Searchpath sp, Subdirectory subdir)
{
  assert(subdir < NUM_SUBDIRS);
  assert(sp < NUM_SEARCHPATHS);
 
  return fmt::format("{}{}", _searchpaths[sp], _subdirs[subdir]);
}
 
std::string FioFindDirectory(Subdirectory subdir)
{
  /* Find and return the first valid directory */
  for (Searchpath sp : _valid_searchpaths) {
    std::string ret = FioGetDirectory(sp, subdir);
    if (FileExists(ret)) return ret;
  }
 
  /* Could not find the directory, fall back to a base path */
  return _personal_dir;
}
 
static std::optional<FileHandle> FioFOpenFileSp(std::string_view filename, std::string_view mode, Searchpath sp, Subdirectory subdir, size_t *filesize)
{
#if defined(_WIN32)
  /* fopen is implemented as a define with ellipses for
   * Unicode support (prepend an L). As we are not sending
   * a string, but a variable, it 'renames' the variable,
   * so make that variable to makes it compile happily */
  wchar_t Lmode[5];
  MultiByteToWideChar(CP_ACP, 0, mode.data(), static_cast<int>(std::size(mode)), Lmode, static_cast<int>(std::size(Lmode)));
#endif
  std::string buf;
 
  if (subdir == NO_DIRECTORY) {
    buf = filename;
  } else {
    buf = fmt::format("{}{}{}", _searchpaths[sp], _subdirs[subdir], filename);
  }
 
  auto f = FileHandle::Open(buf, mode);
#if !defined(_WIN32)
  if (!f.has_value() && strtolower(buf, subdir == NO_DIRECTORY ? 0 : _searchpaths[sp].size() - 1) ) {
    f = FileHandle::Open(buf, mode);
  }
#endif
  if (f.has_value() && filesize != nullptr) {
    /* Find the size of the file */
    fseek(*f, 0, SEEK_END);
    *filesize = ftell(*f);
    fseek(*f, 0, SEEK_SET);
  }
  return f;
}

static std::optional<FileHandle> FioFOpenFileTar(const TarFileListEntry &entry, size_t *filesize)
{
  auto f = FileHandle::Open(entry.tar_filename, "rb");
  if (!f.has_value()) return std::nullopt;
 
  if (fseek(*f, entry.position, SEEK_SET) < 0) {
    return std::nullopt;
  }
 
  if (filesize != nullptr) *filesize = entry.size;
  return f;
}

std::optional<FileHandle> FioFOpenFile(std::string_view filename, std::string_view mode, Subdirectory subdir, size_t *filesize)
{
  std::optional<FileHandle> f = std::nullopt;
  assert(subdir < NUM_SUBDIRS || subdir == NO_DIRECTORY);
 
  for (Searchpath sp : _valid_searchpaths) {
    f = FioFOpenFileSp(filename, mode, sp, subdir, filesize);
    if (f.has_value() || subdir == NO_DIRECTORY) break;
  }
 
  /* We can only use .tar in case of data-dir, and read-mode */
  if (!f.has_value() && mode[0] == 'r' && subdir != NO_DIRECTORY) {
    /* Filenames in tars are always forced to be lowercase */
    std::string resolved_name{filename};
    strtolower(resolved_name);
 
    /* Resolve ".." */
    std::istringstream ss(resolved_name);
    std::vector<std::string> tokens;
    for (std::string token; std::getline(ss, token, PATHSEPCHAR); /* nothing */) {
      if (token == "..") {
        if (tokens.size() < 2) return std::nullopt;
        tokens.pop_back();
      } else if (token == ".") {
        /* Do nothing. "." means current folder, but you can create tar files with "." in the path.
         * This confuses our file resolver. So, act like this folder doesn't exist. */
      } else {
        tokens.push_back(token);
      }
    }
 
    resolved_name.clear();
    bool first = true;
    for (const std::string &token : tokens) {
      if (!first) {
        resolved_name += PATHSEP;
      }
      resolved_name += token;
      first = false;
    }
 
    TarFileList::iterator it = _tar_filelist[subdir].find(resolved_name);
    if (it != _tar_filelist[subdir].end()) {
      f = FioFOpenFileTar(it->second, filesize);
    }
  }
 
  /* Sometimes a full path is given. To support
   * the 'subdirectory' must be 'removed'. */
  if (!f.has_value() && subdir != NO_DIRECTORY) {
    switch (subdir) {
      case BASESET_DIR:
        f = FioFOpenFile(filename, mode, OLD_GM_DIR, filesize);
        if (f.has_value()) break;
        [[fallthrough]];
      case NEWGRF_DIR:
        f = FioFOpenFile(filename, mode, OLD_DATA_DIR, filesize);
        break;
 
      default:
        f = FioFOpenFile(filename, mode, NO_DIRECTORY, filesize);
        break;
    }
  }
 
  return f;
}

void FioCreateDirectory(const std::string &name)
{
  /* Ignore directory creation errors; they'll surface later on. */
  std::error_code error_code;
  std::filesystem::create_directories(OTTD2FS(name), error_code);
}

bool FioRemove(const std::string &filename)
{
  std::filesystem::path path = OTTD2FS(filename);
  std::error_code error_code;
  if (!std::filesystem::remove(path, error_code)) {
    if (error_code) {
      Debug(misc, 0, "Removing {} failed: {}", filename, error_code.message());
    } else {
      Debug(misc, 0, "Removing {} failed: file does not exist", filename);
    }
    return false;
  }
  return true;
}

void AppendPathSeparator(std::string &buf)
{
  if (buf.empty()) return;
 
  if (buf.back() != PATHSEPCHAR) buf.push_back(PATHSEPCHAR);
}

static void SimplifyFileName(std::string &name)
{
  for (char &c : name) {
    /* Force lowercase */
    c = std::tolower(c);
#if (PATHSEPCHAR != '/')
    /* Tar-files always have '/' path-separator, but we want our PATHSEPCHAR */
    if (c == '/') c = PATHSEPCHAR;
#endif
  }
}

uint TarScanner::DoScan(Subdirectory sd)
{
  _tar_filelist[sd].clear();
  _tar_list[sd].clear();
  uint num = this->Scan(".tar", sd, false);
  if (sd == BASESET_DIR || sd == NEWGRF_DIR) num += this->Scan(".tar", OLD_DATA_DIR, false);
  return num;
}

/* static */ uint TarScanner::DoScan(TarScanner::Modes modes)
{
  Debug(misc, 2, "Scanning for tars");
  TarScanner fs;
  uint num = 0;
  if (modes.Test(TarScanner::Mode::Baseset)) {
    num += fs.DoScan(BASESET_DIR);
  }
  if (modes.Test(TarScanner::Mode::NewGRF)) {
    num += fs.DoScan(NEWGRF_DIR);
  }
  if (modes.Test(TarScanner::Mode::AI)) {
    num += fs.DoScan(AI_DIR);
    num += fs.DoScan(AI_LIBRARY_DIR);
  }
  if (modes.Test(TarScanner::Mode::Game)) {
    num += fs.DoScan(GAME_DIR);
    num += fs.DoScan(GAME_LIBRARY_DIR);
  }
  if (modes.Test(TarScanner::Mode::Scenario)) {
    num += fs.DoScan(SCENARIO_DIR);
    num += fs.DoScan(HEIGHTMAP_DIR);
  }
  Debug(misc, 2, "Scan complete, found {} files", num);
  return num;
}

bool TarScanner::AddFile(Subdirectory sd, const std::string &filename)
{
  this->subdir = sd;
  return this->AddFile(filename, 0);
}

static std::string ExtractString(std::span<char> buffer)
{
  return StrMakeValid(std::string_view(buffer.begin(), buffer.end()));
}
 
bool TarScanner::AddFile(const std::string &filename, size_t, [[maybe_unused]] const std::string &tar_filename)
{
  /* No tar within tar. */
  assert(tar_filename.empty());
 
  /* The TAR-header, repeated for every file */
  struct TarHeader {
    char name[100];      
    char mode[8];
    char uid[8];
    char gid[8];
    char size[12];       
    char mtime[12];
    char chksum[8];
    char typeflag;
    char linkname[100];
    char magic[6];
    char version[2];
    char uname[32];
    char gname[32];
    char devmajor[8];
    char devminor[8];
    char prefix[155];    
 
    char unused[12];
  };
  static_assert(sizeof(TarHeader) == 512);
 
  /* Check if we already seen this file */
  TarList::iterator it = _tar_list[this->subdir].find(filename);
  if (it != _tar_list[this->subdir].end()) return false;
 
  auto of = FileHandle::Open(filename, "rb");
  /* Although the file has been found there can be
   * a number of reasons we cannot open the file.
   * Most common case is when we simply have not
   * been given read access. */
  if (!of.has_value()) return false;
  auto &f = *of;
 
  _tar_list[this->subdir][filename] = std::string{};
 
  std::string filename_base = FS2OTTD(std::filesystem::path(OTTD2FS(filename)).filename().native());
  SimplifyFileName(filename_base);
 
  TarHeader th;
  size_t num = 0, pos = 0;
 
  for (;;) { // Note: feof() always returns 'false' after 'fseek()'. Cool, isn't it?
    size_t num_bytes_read = fread(&th, 1, sizeof(TarHeader), f);
    if (num_bytes_read != sizeof(TarHeader)) break;
    pos += num_bytes_read;
 
    /* Check if we have the new tar-format (ustar) or the old one (a lot of zeros after 'link' field) */
    auto last_of_th = &th.unused[std::size(th.unused)];
    if (std::string_view{th.magic, 5} != "ustar" && std::any_of(th.magic, last_of_th, [](auto c) { return c != 0; })) {
      /* If we have only zeros in the block, it can be an end-of-file indicator */
      if (std::all_of(th.name, last_of_th, [](auto c) { return c == 0; })) continue;
 
      Debug(misc, 0, "The file '{}' isn't a valid tar-file", filename);
      return false;
    }
 
    std::string name;
 
    /* The prefix contains the directory-name */
    if (th.prefix[0] != '\0') {
      name = ExtractString(th.prefix);
      name += PATHSEP;
    }
 
    /* Copy the name of the file in a safe way at the end of 'name' */
    name += ExtractString(th.name);
 
    /* The size of the file, for some strange reason, this is stored as a string in octals. */
    std::string size = ExtractString(th.size);
    size_t skip = 0;
    if (!size.empty()) {
      auto value = ParseInteger<size_t>(size, 8);
      if (!value.has_value()) {
        Debug(misc, 0, "The file '{}' has an invalid size for '{}'", filename, name);
        fclose(f);
        return false;
      }
      skip = *value;
    }
 
    switch (th.typeflag) {
      case '\0':
      case '0': { // regular file
        if (name.empty()) break;
 
        /* Store this entry in the list */
        TarFileListEntry entry;
        entry.tar_filename = filename;
        entry.size         = skip;
        entry.position     = pos;
 
        /* Convert to lowercase and our PATHSEPCHAR */
        SimplifyFileName(name);
 
        Debug(misc, 6, "Found file in tar: {} ({} bytes, {} offset)", name, skip, pos);
        if (_tar_filelist[this->subdir].insert(TarFileList::value_type(filename_base + PATHSEPCHAR + name, entry)).second) num++;
 
        break;
      }
 
      case '1': // hard links
      case '2': { // symbolic links
        std::string link = ExtractString(th.linkname);
 
        Debug(misc, 5, "Ignoring link in tar: {} -> {}", name, link);
        break;
      }
 
      case '5': // directory
        /* Convert to lowercase and our PATHSEPCHAR */
        SimplifyFileName(name);
 
        /* Store the first directory name we detect */
        Debug(misc, 6, "Found dir in tar: {}", name);
        if (_tar_list[this->subdir][filename].empty()) _tar_list[this->subdir][filename] = std::move(name);
        break;
 
      default:
        /* Ignore other types */
        break;
    }
 
    /* Skip to the next block.. */
    skip = Align(skip, 512);
    if (fseek(f, skip, SEEK_CUR) < 0) {
      Debug(misc, 0, "The file '{}' can't be read as a valid tar-file", filename);
      return false;
    }
    pos += skip;
  }
 
  Debug(misc, 4, "Found tar '{}' with {} new files", filename, num);
 
  return true;
}

bool ExtractTar(const std::string &tar_filename, Subdirectory subdir)
{
  TarList::iterator it = _tar_list[subdir].find(tar_filename);
  /* We don't know the file. */
  if (it == _tar_list[subdir].end()) return false;
 
  const auto &dirname = it->second;
 
  /* The file doesn't have a sub directory! */
  if (dirname.empty()) {
    Debug(misc, 3, "Extracting {} failed; archive rejected, the contents must be in a sub directory", tar_filename);
    return false;
  }
 
  std::string filename = tar_filename;
  auto p = filename.find_last_of(PATHSEPCHAR);
  /* The file's path does not have a separator? */
  if (p == std::string::npos) return false;
 
  filename.replace(p + 1, std::string::npos, dirname);
  Debug(misc, 8, "Extracting {} to directory {}", tar_filename, filename);
  FioCreateDirectory(filename);
 
  for (auto &it2 : _tar_filelist[subdir]) {
    if (tar_filename != it2.second.tar_filename) continue;
 
    /* it2.first is tarball + PATHSEPCHAR + name. */
    std::string_view name = it2.first;
    name.remove_prefix(name.find_first_of(PATHSEPCHAR) + 1);
    filename.replace(p + 1, std::string::npos, name);
 
    Debug(misc, 9, "  extracting {}", filename);
 
    /* First open the file in the .tar. */
    size_t to_copy = 0;
    auto in = FioFOpenFileTar(it2.second, &to_copy);
    if (!in.has_value()) {
      Debug(misc, 6, "Extracting {} failed; could not open {}", filename, tar_filename);
      return false;
    }
 
    /* Now open the 'output' file. */
    auto out = FileHandle::Open(filename, "wb");
    if (!out.has_value()) {
      Debug(misc, 6, "Extracting {} failed; could not open {}", filename, filename);
      return false;
    }
 
    /* Now read from the tar and write it into the file. */
    char buffer[4096];
    size_t read;
    for (; to_copy != 0; to_copy -= read) {
      read = fread(buffer, 1, std::min(to_copy, lengthof(buffer)), *in);
      if (read <= 0 || fwrite(buffer, 1, read, *out) != read) break;
    }
 
    if (to_copy != 0) {
      Debug(misc, 6, "Extracting {} failed; still {} bytes to copy", filename, to_copy);
      return false;
    }
  }
 
  Debug(misc, 9, "  extraction successful");
  return true;
}
 
#if defined(_WIN32)
extern void DetermineBasePaths(std::string_view exe);

char *getcwd(char *buf, size_t size);
#else /* defined(_WIN32) */

static bool ChangeWorkingDirectoryToExecutable(std::string_view exe)
{
  std::string path{exe};
 
#ifdef WITH_COCOA
  for (size_t pos = path.find_first_of('.'); pos != std::string::npos; pos = path.find_first_of('.', pos + 1)) {
    if (StrEqualsIgnoreCase(path.substr(pos, 4), ".app")) {
      path.erase(pos);
      break;
    }
  }
#endif /* WITH_COCOA */
 
  size_t pos = path.find_last_of(PATHSEPCHAR);
  if (pos == std::string::npos) return false;
 
  path.erase(pos);
 
  if (chdir(path.c_str()) != 0) {
    Debug(misc, 0, "Directory with the binary does not exist?");
    return false;
  }
 
  return true;
}

bool DoScanWorkingDirectory()
{
  /* No working directory, so nothing to do. */
  if (_searchpaths[SP_WORKING_DIR].empty()) return false;
 
  /* Working directory is root, so do nothing. */
  if (_searchpaths[SP_WORKING_DIR] == PATHSEP) return false;
 
  /* No personal/home directory, so the working directory won't be that. */
  if (_searchpaths[SP_PERSONAL_DIR].empty()) return true;
 
  std::string tmp = _searchpaths[SP_WORKING_DIR] + PERSONAL_DIR;
  AppendPathSeparator(tmp);
 
  return _searchpaths[SP_PERSONAL_DIR] != tmp;
}

static std::string GetHomeDir()
{
#ifdef __HAIKU__
  BPath path;
  find_directory(B_USER_SETTINGS_DIRECTORY, &path);
  return std::string(path.Path());
#else
  auto home_env = GetEnv("HOME"); // Stack var, shouldn't be freed
  if (home_env.has_value()) return std::string(*home_env);
 
  const struct passwd *pw = getpwuid(getuid());
  if (pw != nullptr) return std::string(pw->pw_dir);
#endif
  return {};
}

void DetermineBasePaths(std::string_view exe)
{
  std::string tmp;
  const std::string homedir = GetHomeDir();
#ifdef USE_XDG
  if (auto xdg_data_home = GetEnv("XDG_DATA_HOME"); xdg_data_home.has_value()) {
    tmp = *xdg_data_home;
    tmp += PATHSEP;
    tmp += PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR;
    AppendPathSeparator(tmp);
    _searchpaths[SP_PERSONAL_DIR_XDG] = tmp;
 
    tmp += "content_download";
    AppendPathSeparator(tmp);
    _searchpaths[SP_AUTODOWNLOAD_PERSONAL_DIR_XDG] = tmp;
  } else if (!homedir.empty()) {
    tmp = homedir;
    tmp += PATHSEP ".local" PATHSEP "share" PATHSEP;
    tmp += PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR;
    AppendPathSeparator(tmp);
    _searchpaths[SP_PERSONAL_DIR_XDG] = tmp;
 
    tmp += "content_download";
    AppendPathSeparator(tmp);
    _searchpaths[SP_AUTODOWNLOAD_PERSONAL_DIR_XDG] = tmp;
  } else {
    _searchpaths[SP_PERSONAL_DIR_XDG].clear();
    _searchpaths[SP_AUTODOWNLOAD_PERSONAL_DIR_XDG].clear();
  }
#endif
 
#if !defined(WITH_PERSONAL_DIR)
  _searchpaths[SP_PERSONAL_DIR].clear();
#else
  if (!homedir.empty()) {
    tmp = std::move(homedir);
    tmp += PATHSEP;
    tmp += PERSONAL_DIR;
    AppendPathSeparator(tmp);
    _searchpaths[SP_PERSONAL_DIR] = tmp;
 
    tmp += "content_download";
    AppendPathSeparator(tmp);
    _searchpaths[SP_AUTODOWNLOAD_PERSONAL_DIR] = tmp;
  } else {
    _searchpaths[SP_PERSONAL_DIR].clear();
    _searchpaths[SP_AUTODOWNLOAD_PERSONAL_DIR].clear();
  }
#endif
 
#if defined(WITH_SHARED_DIR)
  tmp = SHARED_DIR;
  AppendPathSeparator(tmp);
  _searchpaths[SP_SHARED_DIR] = tmp;
#else
  _searchpaths[SP_SHARED_DIR].clear();
#endif
 
  char cwd[MAX_PATH];
  if (getcwd(cwd, MAX_PATH) == nullptr) *cwd = '\0';
 
  if (_config_file.empty()) {
    /* Get the path to working directory of OpenTTD. */
    tmp = cwd;
    AppendPathSeparator(tmp);
    _searchpaths[SP_WORKING_DIR] = tmp;
 
    _do_scan_working_directory = DoScanWorkingDirectory();
  } else {
    /* Use the folder of the config file as working directory. */
    size_t end = _config_file.find_last_of(PATHSEPCHAR);
    if (end == std::string::npos) {
      /* _config_file is not in a folder, so use current directory. */
      tmp = cwd;
    } else {
      tmp = FS2OTTD(std::filesystem::weakly_canonical(std::filesystem::path(OTTD2FS(_config_file))).parent_path().native());
    }
    AppendPathSeparator(tmp);
    _searchpaths[SP_WORKING_DIR] = tmp;
  }
 
  /* Change the working directory to that one of the executable */
  if (ChangeWorkingDirectoryToExecutable(exe)) {
    char buf[MAX_PATH];
    if (getcwd(buf, lengthof(buf)) == nullptr) {
      tmp.clear();
    } else {
      tmp = buf;
    }
    AppendPathSeparator(tmp);
    _searchpaths[SP_BINARY_DIR] = tmp;
  } else {
    _searchpaths[SP_BINARY_DIR].clear();
  }
 
  if (cwd[0] != '\0') {
    /* Go back to the current working directory. */
    if (chdir(cwd) != 0) {
      Debug(misc, 0, "Failed to return to working directory!");
    }
  }
 
#if !defined(GLOBAL_DATA_DIR)
  _searchpaths[SP_INSTALLATION_DIR].clear();
#else
  tmp = GLOBAL_DATA_DIR;
  AppendPathSeparator(tmp);
  _searchpaths[SP_INSTALLATION_DIR] = std::move(tmp);
#endif
#ifdef WITH_COCOA
extern void CocoaSetApplicationBundleDir();
  CocoaSetApplicationBundleDir();
#else
  _searchpaths[SP_APPLICATION_BUNDLE_DIR].clear();
#endif
}
#endif /* defined(_WIN32) */
 
std::string _personal_dir;

void DeterminePaths(std::string_view exe, bool only_local_path)
{
  DetermineBasePaths(exe);
  FillValidSearchPaths(only_local_path);
 
#ifdef USE_XDG
  std::string config_home;
  std::string homedir = GetHomeDir();
  if (auto xdg_config_home = GetEnv("XDG_CONFIG_HOME"); xdg_config_home.has_value()) {
    config_home = *xdg_config_home;
    config_home += PATHSEP;
    config_home += PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR;
  } else if (!homedir.empty()) {
    /* Defaults to ~/.config */
    config_home = std::move(homedir);
    config_home += PATHSEP ".config" PATHSEP;
    config_home += PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR;
  }
  AppendPathSeparator(config_home);
#endif
 
  for (Searchpath sp : _valid_searchpaths) {
    if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
    Debug(misc, 3, "{} added as search path", _searchpaths[sp]);
  }
 
  std::string config_dir;
  if (!_config_file.empty()) {
    config_dir = _searchpaths[SP_WORKING_DIR];
  } else {
    std::string personal_dir = FioFindFullPath(BASE_DIR, "openttd.cfg");
    if (!personal_dir.empty()) {
      auto end = personal_dir.find_last_of(PATHSEPCHAR);
      if (end != std::string::npos) personal_dir.erase(end + 1);
      config_dir = std::move(personal_dir);
    } else {
#ifdef USE_XDG
      /* No previous configuration file found. Use the configuration folder from XDG. */
      config_dir = config_home;
#else
      static const Searchpath new_openttd_cfg_order[] = {
          SP_PERSONAL_DIR, SP_BINARY_DIR, SP_WORKING_DIR, SP_SHARED_DIR, SP_INSTALLATION_DIR
        };
 
      config_dir.clear();
      for (const auto &searchpath : new_openttd_cfg_order) {
        if (IsValidSearchPath(searchpath)) {
          config_dir = _searchpaths[searchpath];
          break;
        }
      }
#endif
    }
    _config_file = config_dir + "openttd.cfg";
  }
 
  Debug(misc, 1, "{} found as config directory", config_dir);
 
  _highscore_file = config_dir + "hs.dat";
  extern std::string _hotkeys_file;
  _hotkeys_file = config_dir + "hotkeys.cfg";
  extern std::string _windows_file;
  _windows_file = config_dir + "windows.cfg";
  extern std::string _private_file;
  _private_file = config_dir + "private.cfg";
  extern std::string _secrets_file;
  _secrets_file = config_dir + "secrets.cfg";
  extern std::string _favs_file;
  _favs_file = config_dir + "favs.cfg";
 
#ifdef USE_XDG
  if (config_dir == config_home) {
    /* We are using the XDG configuration home for the config file,
     * then store the rest in the XDG data home folder. */
    _personal_dir = _searchpaths[SP_PERSONAL_DIR_XDG];
    if (only_local_path) {
      /* In case of XDG and we only want local paths and we detected that
       * the user either manually indicated the XDG path or didn't use
       * "-c" option, we change the working-dir to the XDG personal-dir,
       * as this is most likely what the user is expecting. */
      _searchpaths[SP_WORKING_DIR] = _searchpaths[SP_PERSONAL_DIR_XDG];
    }
  } else
#endif
  {
    _personal_dir = config_dir;
  }
 
  /* Make the necessary folders */
  FioCreateDirectory(config_dir);
#if defined(WITH_PERSONAL_DIR)
  FioCreateDirectory(_personal_dir);
#endif
 
  Debug(misc, 1, "{} found as personal directory", _personal_dir);
 
  static const Subdirectory default_subdirs[] = {
    SAVE_DIR, AUTOSAVE_DIR, SCENARIO_DIR, HEIGHTMAP_DIR, BASESET_DIR, NEWGRF_DIR, AI_DIR, AI_LIBRARY_DIR, GAME_DIR, GAME_LIBRARY_DIR, SCREENSHOT_DIR, SOCIAL_INTEGRATION_DIR
  };
 
  for (const auto &default_subdir : default_subdirs) {
    FioCreateDirectory(fmt::format("{}{}", _personal_dir, _subdirs[default_subdir]));
  }
 
  /* If we have network we make a directory for the autodownloading of content */
  _searchpaths[SP_AUTODOWNLOAD_DIR] = _personal_dir + "content_download" PATHSEP;
  Debug(misc, 3, "{} added as search path", _searchpaths[SP_AUTODOWNLOAD_DIR]);
  FioCreateDirectory(_searchpaths[SP_AUTODOWNLOAD_DIR]);
  FillValidSearchPaths(only_local_path);
 
  /* Create the directory for each of the types of content */
  const Subdirectory subdirs[] = { SCENARIO_DIR, HEIGHTMAP_DIR, BASESET_DIR, NEWGRF_DIR, AI_DIR, AI_LIBRARY_DIR, GAME_DIR, GAME_LIBRARY_DIR, SOCIAL_INTEGRATION_DIR };
  for (const auto &subdir : subdirs) {
    FioCreateDirectory(FioGetDirectory(SP_AUTODOWNLOAD_DIR, subdir));
  }
 
  extern std::string _log_file;
  _log_file = _personal_dir + "openttd.log";
}

void SanitizeFilename(std::string &filename)
{
  for (auto &c : filename) {
    switch (c) {
      /* The following characters are not allowed in filenames
       * on at least one of the supported operating systems: */
      case ':': case '\\': case '*': case '?': case '/':
      case '<': case '>': case '|': case '"':
        c = '_';
        break;
    }
  }
}

std::unique_ptr<char[]> ReadFileToMem(const std::string &filename, size_t &lenp, size_t maxsize)
{
  auto in = FileHandle::Open(filename, "rb");
  if (!in.has_value()) return nullptr;
 
  fseek(*in, 0, SEEK_END);
  size_t len = ftell(*in);
  fseek(*in, 0, SEEK_SET);
  if (len > maxsize) return nullptr;
 
  std::unique_ptr<char[]> mem = std::make_unique<char[]>(len + 1);
 
  mem.get()[len] = 0;
  if (fread(mem.get(), len, 1, *in) != 1) return nullptr;
 
  lenp = len;
  return mem;
}

static bool MatchesExtension(std::string_view extension, const std::string &filename)
{
  if (extension.empty()) return true;
  if (filename.length() < extension.length()) return false;
 
  std::string_view filename_sv = filename; // String view to avoid making another copy of the substring.
  return StrCompareIgnoreCase(extension, filename_sv.substr(filename_sv.length() - extension.length())) == 0;
}

static uint ScanPath(FileScanner *fs, std::string_view extension, const std::filesystem::path &path, size_t basepath_length, bool recursive)
{
  uint num = 0;
 
  std::error_code error_code;
  for (const auto &dir_entry : std::filesystem::directory_iterator(path, error_code)) {
    if (dir_entry.is_directory()) {
      if (!recursive) continue;
      num += ScanPath(fs, extension, dir_entry.path(), basepath_length, recursive);
    } else if (dir_entry.is_regular_file()) {
      std::string file = FS2OTTD(dir_entry.path().native());
      if (!MatchesExtension(extension, file)) continue;
      if (fs->AddFile(file, basepath_length, {})) num++;
    }
  }
  if (error_code) {
    Debug(misc, 9, "Unable to read directory {}: {}", path.string(), error_code.message());
  }
 
  return num;
}

static uint ScanTar(FileScanner *fs, std::string_view extension, const TarFileList::value_type &tar)
{
  uint num = 0;
 
  if (MatchesExtension(extension, tar.first) && fs->AddFile(tar.first, 0, tar.second.tar_filename)) num++;
 
  return num;
}

uint FileScanner::Scan(std::string_view extension, Subdirectory sd, bool tars, bool recursive)
{
  this->subdir = sd;
 
  uint num = 0;
 
  for (Searchpath sp : _valid_searchpaths) {
    /* Don't search in the working directory */
    if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
 
    std::string path = FioGetDirectory(sp, sd);
    num += ScanPath(this, extension, OTTD2FS(path), path.size(), recursive);
  }
 
  if (tars && sd != NO_DIRECTORY) {
    for (const auto &tar : _tar_filelist[sd]) {
      num += ScanTar(this, extension, tar);
    }
  }
 
  switch (sd) {
    case BASESET_DIR:
      num += this->Scan(extension, OLD_GM_DIR, tars, recursive);
      [[fallthrough]];
    case NEWGRF_DIR:
      num += this->Scan(extension, OLD_DATA_DIR, tars, recursive);
      break;
 
    default: break;
  }
 
  return num;
}

uint FileScanner::Scan(std::string_view extension, const std::string &directory, bool recursive)
{
  std::string path(directory);
  AppendPathSeparator(path);
  return ScanPath(this, extension, OTTD2FS(path), path.size(), recursive);
}
 
std::optional<FileHandle> FileHandle::Open(const std::string &filename, std::string_view mode)
{
#if defined(_WIN32)
  /* Windows also requires mode to be wchar_t. */
  auto f = _wfopen(OTTD2FS(filename).c_str(), OTTD2FS(mode).c_str());
#else
  auto f = fopen(filename.c_str(), std::string{mode}.c_str());
#endif /* _WIN32 */
 
  if (f == nullptr) return std::nullopt;
  return FileHandle(f);
}