storage.cpp

Go to the documentation of this file.

/*
 *  SatNOGS-COMMS MCU software
 *
 *  Copyright (C) 2025, Libre Space Foundation <http://libre.space>
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *  SPDX-License-Identifier: GNU General Public License v3.0 or later
 */
 
#include "storage.hpp"
#include "scoped_lock.hpp"
#include "settings.hpp"
#include "time.hpp"
 
#include <time.h>
#include <zephyr/drivers/disk.h>
#include <zephyr/fs/fs.h>
#include <zephyr/fs/littlefs.h>
#include <zephyr/kernel.h>
#include <zephyr/storage/disk_access.h>
 
#include <etl/basic_string.h>
#include <etl/format_spec.h>
#include <etl/queue.h>
#include <etl/stack.h>
#include <etl/string.h>
#include <etl/string_stream.h>
#include <etl/to_string.h>
 
namespace satnogs::comms
{
 
FS_LITTLEFS_DECLARE_CUSTOM_CONFIG(lfs_storage, storage::SECTOR_SIZE,
                                  storage::SECTOR_SIZE, storage::SECTOR_SIZE,
                                  storage::SECTOR_SIZE,
                                  4 * storage::SECTOR_SIZE);
 
static struct fs_mount_t storage_mnt = {
    .type        = FS_LITTLEFS,
    .mnt_point   = CONFIG_STORAGE_MNT_POINT,
    .fs_data     = &lfs_storage,
    .storage_dev = (void *)CONFIG_DISK_NAME,
    .flags       = FS_MOUNT_FLAG_USE_DISK_ACCESS | FS_MOUNT_FLAG_NO_FORMAT,
};
 
/* Raw access uses DMA and needs an aligned buffer */
alignas(16) static uint8_t aligned_buffer[CONFIG_MAX_MTU];
 
/* Used for DFS traversal */
static etl::queue<etl::string<CONFIG_STORAGE_MAX_PATH_LEN>,
                  CONFIG_STORAGE_MAX_DEPTH>
    m_dfs_dirs;
 
storage::storage() : m_enabled(false), m_mounted(false)
{
  m_pname.clear();
  k_mutex_init(&m_mtx);
}

float
storage::utilization()
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
  const etl::string<16> emmc_dirpath(CONFIG_STORAGE_MNT_POINT);
  const uint64_t        used_bytes = du(emmc_dirpath);
  const uint64_t        total_bytes =
      (sectors_num() - CONFIG_DISK_START_SECTOR) * 512ULL;
 
  return static_cast<float>(used_bytes) / static_cast<float>(total_bytes);
}
 
void
storage::set_dir_priv(lib::emmc::dir d, bool mnt)
{
  auto &emmc = lib::board::get_instance().emmc();
  /* When the direction switches to FPGA the filesystem should be unmounted */
  if (d == lib::emmc::dir::FPGA) {
    if (m_enabled && m_mounted) {
      fs_unmount(&storage_mnt);
      m_mounted = false;
    }
  }
  emmc.set_dir(d);
  /* If requested, try to mount also in case we switch back to the MCU */
  if (m_enabled && mnt && d == lib::emmc::dir::MCU) {
    int ret = fs_mount(&storage_mnt);
    if (ret == 0 || ret == -EBUSY) {
      m_mounted = true;
    }
  }
}

void
storage::enable(bool en)
{
  /* If already enabled, don't mess anything */
  if (enabled() && en) {
    return;
  }
 
  auto       &emmc = lib::board::get_instance().emmc();
  scoped_lock lock(&m_mtx);
  if (en) {
    emmc.enable(true);
    emmc.reset(true);
    k_msleep(10);
    emmc.reset(false);
    set_dir_priv(lib::emmc::dir::MCU);
    m_enabled = true;
 
    int rc = fs_mount(&storage_mnt);
 
    if (rc == -EBUSY) {
      /* Accept the already mounted case! */
      throw mount_exception(__FILE__, __LINE__, rc);
    } else if (rc < 0) {
      /* Try and format the disk */
      storage_mnt.flags = storage_mnt.flags & ~FS_MOUNT_FLAG_NO_FORMAT;
      rc                = fs_mount(&storage_mnt);
      if (rc < 0) {
        throw mount_exception(__FILE__, __LINE__, rc);
      }
      /* reset the flag */
      storage_mnt.flags |= FS_MOUNT_FLAG_NO_FORMAT;
    }
 
    m_pname.clear();
    m_pname.append(storage_mnt.mnt_point);
    m_pname.append(CONFIG_STORAGE_LOG_MCU_PATH);
    m_mounted = true;
    mkdir(m_pname);
 
  } else {
    int rc = fs_unmount(&storage_mnt);
    disk_access_ioctl(CONFIG_DISK_NAME, DISK_IOCTL_CTRL_DEINIT, NULL);
    m_mounted = false;
    emmc.enable(false);
    m_enabled = false;
    if (rc < 0 && rc != -EINVAL) {
      throw mount_exception(__FILE__, __LINE__, rc);
    }
  }
}

bool
storage::enabled() const
{
  return m_enabled;
}

bool
storage::mounted() const
{
  return m_mounted;
}

void
storage::set_dir(lib::emmc::dir d)
{
  scoped_lock lock(&m_mtx);
  set_dir_priv(d, true);
}

lib::emmc::dir
storage::get_dir() const
{
  auto       &emmc = lib::board::get_instance().emmc();
  scoped_lock lock(&m_mtx);
  return emmc.get_dir();
}

bool
storage::ready() const
{
  return enabled() && get_dir() == lib::emmc::dir::MCU && mounted();
}

void
storage::mkdir(const etl::istring &path)
{
 
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
 
  scoped_lock lock(&m_mtx);
 
  int rc = fs_mkdir(path.c_str());
  if (rc < 0 && rc != -EEXIST) {
    throw fs_exception(__FILE__, __LINE__);
  }
}
 
int
storage::rm_priv(const etl::istring &path)
{
  return fs_unlink(path.c_str());
}

bool
storage::ready_raw() const
{
  return enabled() && get_dir() == lib::emmc::dir::MCU;
}

void
storage::rm(const etl::istring &path)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  if (path.size() > CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
 
  struct fs_dirent entry;
  int              rc = fs_stat(path.c_str(), &entry);
  if (rc < 0 || entry.type != FS_DIR_ENTRY_FILE) {
    throw delete_file_exception(__FILE__, __LINE__);
  }
 
  scoped_lock lock(&m_mtx);
  rc = rm_priv(path);
 
  if (rc < 0) {
    throw delete_file_exception(__FILE__, __LINE__);
  }
}

void
storage::rmdir(const etl::istring &path)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
 
  static struct fs_dirent entry;
 
  int rc = fs_stat(path.c_str(), &entry);
  if (rc < 0 || entry.type != FS_DIR_ENTRY_DIR) {
    throw delete_dir_exception(__FILE__, __LINE__);
  }
 
  etl::string_stream stream(m_pname);
 
  struct fs_dir_t dirp;
  fs_dir_t_init(&dirp);
 
  scoped_lock lock(&m_mtx);
  rc = fs_opendir(&dirp, path.c_str());
  if (rc) {
    throw fs_exception(__FILE__, __LINE__);
  }
 
  for (;;) {
    rc = fs_readdir(&dirp, &entry);
 
    if (rc || entry.name[0] == 0) {
      fs_closedir(&dirp);
      if (rc < 0) {
        throw fs_exception(__FILE__, __LINE__);
      }
      break;
    }
 
    m_pname.clear();
    stream << path << "/" << entry.name;
    /*
     * Try to delete it if it is a file
     * FIXME: Currently we do not support recursive file deletion
     */
    if (entry.type == FS_DIR_ENTRY_FILE) {
      rm_priv(m_pname);
    }
  }
 
  fs_closedir(&dirp);
  rc = fs_unlink(path.c_str());
  if (rc < 0) {
    throw delete_dir_exception(__FILE__, __LINE__);
  }
}

size_t
storage::write(const etl::istring &path, const uint8_t *b, size_t len)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  int rc;
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
  struct fs_file_t file;
 
  scoped_lock lock(&m_mtx);
 
  fs_file_t_init(&file);
  rc = fs_open(&file, path.c_str(), FS_O_CREATE | FS_O_RDWR);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  fs_seek(&file, 0, FS_SEEK_END);
  len = std::min<size_t>(len, CONFIG_MAX_MTU);
  std::copy_n(b, len, aligned_buffer);
  rc = fs_write(&file, aligned_buffer, len);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  fs_close(&file);
  return rc;
}

void
storage::truncate(const etl::istring &path, size_t len)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  int rc;
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
 
  // Compute new size (clamp at 0)
  uint64_t    curr_size = du(path);
  const off_t new_size =
      (curr_size > len) ? static_cast<off_t>(curr_size - len) : 0;
 
  // Init and open the file
  struct fs_file_t file;
  scoped_lock      lock(&m_mtx);
 
  fs_file_t_init(&file);
  rc = fs_open(&file, path.c_str(), FS_O_RDWR);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  // Truncate the file
  rc = fs_truncate(&file, new_size);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  // Ensure metadata is flushed (not necessary, but just to be sure)
  rc = fs_sync(&file);
  fs_close(&file);
  if (rc < 0) {
    throw fs_exception(__FILE__, __LINE__);
  }
}

size_t
storage::write(const etl::istring &path, const uint8_t *b, size_t len,
               size_t offset)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  int rc;
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
  struct fs_file_t file;
 
  scoped_lock lock(&m_mtx);
 
  fs_file_t_init(&file);
  rc = fs_open(&file, path.c_str(), FS_O_CREATE | FS_O_RDWR);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  /* Try to seek, otherwise append at the end */
  rc = fs_seek(&file, offset, FS_SEEK_SET);
  if (rc < 0) {
    fs_seek(&file, 0, FS_SEEK_END);
  }
 
  len = std::min<size_t>(len, CONFIG_MAX_MTU);
  std::copy_n(b, len, aligned_buffer);
  rc = fs_write(&file, aligned_buffer, len);
  if (rc < 0) {
    fs_close(&file);
    throw fs_exception(__FILE__, __LINE__);
  }
 
  fs_close(&file);
  return rc;
}

uint64_t
storage::du(const etl::istring &path)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  if (path.size() >= CONFIG_STORAGE_MAX_PATH_LEN) {
    throw max_path_exception(__FILE__, __LINE__);
  }
 
  scoped_lock lock(&m_mtx);
 
  uint64_t         total = 0;
  struct fs_dirent entry;
 
  /* If the path is a file get the size directly */
  int rc = fs_stat(path.c_str(), &entry);
  if (rc < 0) {
    throw fs_exception(__FILE__, __LINE__);
  }
  if (entry.type == FS_DIR_ENTRY_FILE) {
    return entry.size;
  }
 
  m_dfs_dirs.clear();
  m_dfs_dirs.push(path);
 
  while (!m_dfs_dirs.empty()) {
    /*
     * Get a reference to the path in order to reduce the stack usage as each
     * path consumes quite a lot of memory. Take care not to pop the reference
     * contents, until no more is needed.
     */
    auto     &current = m_dfs_dirs.front();
    directory dir(current);
 
    while (ls(dir, entry)) {
      if (entry.type == FS_DIR_ENTRY_FILE) {
        total += entry.size;
      } else if (entry.type == FS_DIR_ENTRY_DIR && m_dfs_dirs.full() == false) {
        m_dfs_dirs.push(current);
        auto &new_dir = m_dfs_dirs.back();
        new_dir += "/";
        new_dir += entry.name;
      }
    }
    /* Now the path string can be released */
    m_dfs_dirs.pop();
  }
  return total;
}

void
storage::ls(const etl::istring &path, etl::istring &res)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  struct fs_dirent   entry;
  scoped_lock        lock(&m_mtx);
  etl::string_stream stream(m_pname);
  directory          dir(path);
 
  bool more = true;
  while (more) {
    more = ls(dir, entry);
    if (more) {
      m_pname.clear();
      if (entry.type == FS_DIR_ENTRY_DIR) {
        stream << path.c_str() << "/" << entry.name << "/"
               << "\n";
      } else {
        stream << path.c_str() << "/" << entry.name << "\n";
      }
      res.append(m_pname);
    }
  }
}

bool
storage::ls(directory &dir, fs_dirent &entry)
{
  if (!ready()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  int rc = fs_readdir(&dir.m_dir, &entry);
  if (rc || entry.name[0] == 0) {
    return false;
  }
  return true;
}

bool
storage::is_file(const etl::istring &path)
{
  struct fs_dirent entry;
  int              rc = fs_stat(path.c_str(), &entry);
  if (rc < 0) {
    throw fs_exception(__FILE__, __LINE__);
  }
  return entry.type == FS_DIR_ENTRY_FILE;
}

bool
storage::is_dir(const etl::istring &path)
{
  return !is_file(path);
}

size_t
storage::read(const etl::istring &path, uint8_t *b, size_t len, size_t offset)
{
  file f(path);
  return read(f, b, len, offset);
}

size_t
storage::read(file &f, uint8_t *b, size_t len, size_t offset)
{
  scoped_lock lock(&m_mtx);
  fs_seek(&f.m_file, offset, FS_SEEK_SET);
  len         = std::min<size_t>(len, CONFIG_MAX_MTU);
  ssize_t ret = fs_read(&f.m_file, aligned_buffer, len);
  if (ret < 0) {
    throw read_exception(__FILE__, __LINE__);
  }
  std::copy_n(aligned_buffer, ret, b);
  return ret;
}

size_t
storage::read(file &f, uint8_t *b, size_t len)
{
  scoped_lock lock(&m_mtx);
  len         = std::min<size_t>(len, CONFIG_MAX_MTU);
  ssize_t ret = fs_read(&f.m_file, aligned_buffer, len);
  if (ret < 0) {
    throw read_exception(__FILE__, __LINE__);
  }
  std::copy_n(aligned_buffer, ret, b);
  return ret;
}

size_t
storage::write_raw(const uint8_t *b, size_t start_sector, size_t len)
{
  if (!ready_raw()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  /* Get the number of full sectors to write */
  size_t   sectors   = len / SECTOR_SIZE;
  uint32_t remainder = len % SECTOR_SIZE;
 
  /* Do not touch the littleFS
   * check if the sectors that need to be written overlap
   * with the starting sector of the littlefs.
   * if we have remaining bytes we need +1 sector
   */
  if (start_sector + sectors + (remainder ? 1 : 0) >=
      CONFIG_DISK_START_SECTOR) {
    return 0;
  }
 
  for (size_t i = 0; i < sectors; i++) {
    /* Copy the the memory aligned buffer */
    std::copy_n(b + i * SECTOR_SIZE, SECTOR_SIZE, aligned_buffer);
    int ret = disk_access_write(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME,
                                aligned_buffer, start_sector + i, 1);
    if (ret) {
      return 0;
    }
  }
 
  /* Write any semi-full sector */
  if (remainder) {
    int ret = disk_access_read(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME,
                               aligned_buffer, start_sector + sectors, 1);
    if (ret) {
 
      /* We were able to write some stuff so far*/
      return sectors * SECTOR_SIZE;
    }
 
    std::copy_n(b + sectors * SECTOR_SIZE, remainder, aligned_buffer);
    ret = disk_access_write(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME,
                            aligned_buffer, start_sector + sectors, 1);
    if (ret) {
      return sectors * SECTOR_SIZE;
    }
  }
  return len;
}

size_t
storage::read_raw(uint8_t *b, size_t start_sector, size_t sectors)
{
  if (!ready_raw()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  size_t cnt = 0;
  for (size_t i = 0; i < sectors; i++) {
    int ret = disk_access_read(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME,
                               aligned_buffer, start_sector + i, 1);
    if (ret) {
      return cnt;
    }
    cnt++;
    std::copy_n(aligned_buffer, SECTOR_SIZE, b + i * SECTOR_SIZE);
  }
  return sectors;
}

void
storage::erase_sectors(size_t start_sector, size_t sectors)
{
  if (!ready_raw()) {
    throw not_ready_exception(__FILE__, __LINE__);
  }
 
  std::fill_n(aligned_buffer, 512, 0xFF);
  for (size_t i = start_sector; i < start_sector + sectors; i++) {
    disk_access_write(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME, aligned_buffer, i,
                      1);
  }
}

size_t
storage::sectors_num() const
{
  uint32_t cnt = 0;
  int      ret = disk_access_ioctl(CONFIG_STM32_SDMMC_DEFAULT_DISK_NAME,
                                   DISK_IOCTL_GET_SECTOR_COUNT, &cnt);
  if (ret < 0) {
    return 0;
  }
  return cnt;
}
 
} // namespace satnogs::comms