libproto_demo/DataStorageMgmt.cpp

#include "DataStorageMgmt.hpp"
#include <ctime>
#include <memepp/convert/common_def.hpp>
#include <memepp/convert/std/string.hpp>
#include <memepp/convert/std/wstring.hpp>
#include <memepp/convert/self.hpp>
#include <chrono>
#include <mego/util/std/time.h>

using namespace std;
namespace ghcfs = ghc::filesystem;

#define HISDATA_SAVE_MAX_DAYS               24 * 30 * 3      //3个月
#define OFFLINE_DATA_SAVE_MAX_DAYS     24 * 7              //7天

//获取当前日期字符串yyyyMMdd
static memepp::string get_date_string()
{
    struct tm timeinfo;
    mgu_time_t now = time(nullptr);
    mgu_localtime_s(&now, &timeinfo);

    char buffer[20];
    auto size = strftime(buffer, sizeof(buffer), "%Y%m%d", &timeinfo);

    return memepp::string(buffer, size);
}

//获取当前日期字符串yyyyMMddHH
static memepp::string get_datetime_string()
{
    struct tm timeinfo;
    mgu_time_t now = time(nullptr);
    mgu_localtime_s(&now,&timeinfo);

    char buffer[20];
    auto size = strftime(buffer, sizeof(buffer), "%Y%m%d%H", &timeinfo);

    return memepp::string(buffer, size);
}

//获取时间最早的文件
memepp::native_string DataStorageMgmt::get_earliest_file(memepp::native_string folderPath)
{
    ghcfs::path path(folderPath);
    std::string strEarliestFile = "";

    if (ghcfs::exists(folderPath) && ghcfs::is_directory(folderPath))
    {
        ghcfs::file_time_type earliestTime = ghcfs::file_time_type::time_point::max();

        for (const auto& entry : ghcfs::directory_iterator(folderPath))
        {
            if (ghcfs::is_regular_file(entry))
            {
                const auto createTime = ghcfs::last_write_time(entry);
                if (createTime < earliestTime)
                {
                    earliestTime = createTime;
                    strEarliestFile = entry.path().string();
                }
            }
        }
    }
    memepp::string mmpStr = memepp::string(strEarliestFile.c_str(), strEarliestFile.length());
    return mm_to<memepp::native_string>(mmpStr);
}

//获取时间最晚文件的文件名
std::string DataStorageMgmt::get_last_filename(memepp::native_string folderPath)
{
    ghcfs::path path(folderPath);
    std::string strLastFileName = "";

    if (ghcfs::exists(folderPath) && ghcfs::is_directory(folderPath))
    {
        ghcfs::file_time_type lastTime = ghcfs::file_time_type::time_point::min();

        for (const auto& entry : ghcfs::directory_iterator(folderPath))
        {
            if (ghcfs::is_regular_file(entry))
            {
                const auto createTime = ghcfs::last_write_time(entry);
                if (createTime > lastTime)
                {
                    lastTime = createTime;
                    strLastFileName = entry.path().filename().string();
                }
            }
        }
    }
    size_t pos = strLastFileName.rfind('.');
    if (pos != std::string::npos) {
        strLastFileName.erase(pos);
    }
    return strLastFileName;
}

DataStorageMgmt::DataStorageMgmt()
{
    isCanSave = true;
}

DataStorageMgmt::~DataStorageMgmt()
{
    //程序退出前，关闭文件句柄
    if (m_history_data_fs.is_open()) {
        m_history_data_fs.close();
    }

    if (m_offline_file_fs.is_open()) {
        m_offline_file_fs.close();
    }
}

void DataStorageMgmt::init_data_storage(memepp::native_string& folderPath, const uint8_t deviceId)
{
    m_folderPath = folderPath;

    memepp::native_string strDate = mm_to<memepp::native_string>(get_date_string());
#if MG_OS__WIN_AVAIL
    m_history_folder = folderPath + L"/HistoryData";
    m_history_folder_today = m_history_folder + L"/" + strDate;
    m_offline_folder = folderPath + L"/OfflineData";
    m_offline_folder_today = m_offline_folder + L"/" + strDate;
#else
    m_history_folder = folderPath + "/HistoryData";
    m_history_folder_today = m_history_folder + "/" + strDate;
    m_offline_folder = folderPath + "/OffilneData";
    m_offline_folder_today = m_offline_folder + "/" + strDate;
#endif

    // 如果历史数据文件夹不存在则进行创建
    if (!ghcfs::exists(m_history_folder))
    {
        ghcfs::create_directory(m_history_folder);
    }

    //每天一个文件夹，以年月日作为文件夹名
    if (!ghcfs::exists(m_history_folder_today))
    {
        ghcfs::create_directory(m_history_folder_today);
    }

    //如果离线数据文件夹不存在则进行创建
    if (!ghcfs::exists(m_offline_folder))
    {
        ghcfs::create_directory(m_offline_folder);
    }

    //如果离线数据文件夹不存在则进行创建
    if (!ghcfs::exists(m_offline_folder_today))
    {
        ghcfs::create_directory(m_offline_folder_today);
    }
    else
    {
        auto strTmp = get_last_filename(m_offline_folder_today);
        m_strDateTime = mm_to<memepp::native_string>(memepp::string(strTmp.c_str(), strTmp.length()));

        if (!m_strDateTime.empty())
        {
            size_t index = m_strDateTime.find('_'); // 查找下划线的位置
            if (index != std::string::npos)
            {
                std::string afterUnderscore = strTmp.substr(index + 1);
                offline_data_file_index = stoi(afterUnderscore) + 1;
            }
            else 
            {
                offline_data_file_index = 1;
            }
        }
        else
        {
            offline_data_file_index = 0;
        }

    }

    delete_out_date_file(m_history_folder, HISDATA_SAVE_MAX_DAYS);
    delete_out_date_file(m_offline_folder_today, OFFLINE_DATA_SAVE_MAX_DAYS);
}

void DataStorageMgmt::to_new_day()
{
    init_data_storage(m_folderPath,1);

    //如果正在保存数据，则关闭老文件，重新打开新文件
    if (m_offline_file_fs.is_open())
    {
        create_offline_data_file();
    }
}


//遍历文件，并删除存储期限外的文件
void DataStorageMgmt::delete_out_date_file(const memepp::native_string dataFolderPath, uint16_t max_days)
{
    try {
        auto now = std::chrono::system_clock::now();
        auto ninetyDaysAgo = now - std::chrono::hours(max_days);  // 获取90天前的时间点

        for (const auto& dirEntry : ghcfs::directory_iterator(dataFolderPath))
        {
            if (dirEntry.is_directory())
            {
                std::string folderName = dirEntry.path().filename().string();
                if (folderName.length() != 8)  // 校验文件夹名称长度
                    continue;

                int year = std::stoi(folderName.substr(0, 4));
                int month = std::stoi(folderName.substr(4, 2));
                int day = std::stoi(folderName.substr(6, 2));

                std::tm tm = { 0 };
                tm.tm_year = year - 1900;
                tm.tm_mon = month - 1;
                tm.tm_mday = day;

                auto folderTime = std::chrono::system_clock::from_time_t(std::mktime(&tm));
                if (folderTime < ninetyDaysAgo)
                {
                    ghcfs::remove_all(dirEntry);  // 删除文件夹及其内容
                    std::cout << "Deleted folder: " << dirEntry.path() << std::endl;
                }
            }
        }
    }
    catch (const std::exception& e) 
    {
        string errmgs = "Error occurred while deleting files : " + string(e.what());
        std::cerr << errmgs << std::endl;
    }
}

// 存储全量数据到文件系统
void DataStorageMgmt::save_data_to_file(const uint8_t data[], const size_t dataSize, const uint64_t timestamp)
{
    if (isCanSave == false)
        return;


    auto strDateTime = mm_to<memepp::native_string>(get_datetime_string());

    if (strDateTime == m_strDateTime)
    {
        //同一个小时内，如果新创建文件需要增加1
        ++offline_data_file_index;
}
    else
    {
        m_strDateTime = strDateTime;
        offline_data_file_index = 0;
    }

#if MG_OS__WIN_AVAIL
    auto strfile_name = std::to_wstring(timestamp) + L".txt";
    auto m_history_file_path = m_history_folder_today + L"/" + strfile_name;
#else
    auto strfile_name = std::to_string(timestamp) + ".txt";
    auto m_history_file_path = m_history_folder_today + "/" + strfile_name;
#endif

    std::fstream hisdata_fs;
    hisdata_fs.open(m_history_file_path, std::ios::binary | std::ios::app);

    if (hisdata_fs.is_open())
    {
        // 写入 uint8_t 数据
        hisdata_fs.write(reinterpret_cast<const char*>(data), dataSize * sizeof(uint8_t));

        hisdata_fs.close();

        std::cout << "save history succful" << std::endl;
    }
    else 
    {
        std::cout << "open histroy file faile" << std::endl;
    }
}

//通过时间戳读取历史数据
void DataStorageMgmt::get_history_file_list(uint64_t &startTime, uint64_t &endTime, std::vector<ghcfs::path>& hisDataPath)
{
    uint64_t startTimeTmp  = 0;
    uint64_t endTimeTmp = 0;

    const auto folderPath = ghcfs::absolute(m_history_folder);

    if (!is_directory(folderPath)) {
        std::cout << "Invalid folder path." << std::endl;
    }

    for (const auto& dateFolder : ghcfs::directory_iterator(folderPath))
    {
        for (const auto& entry : ghcfs::directory_iterator(dateFolder))
        {
            if (ghcfs::is_directory(entry)) {
                continue;
            }

            const auto filePath = entry.path();
            const auto fileName = filePath.filename().string();
            const auto fileTime = std::stoll(fileName.substr(0, fileName.find('.')));

            if (fileTime < startTime) {
                continue;
            }

            if (fileTime > endTime) {
                break;
            }

            //获取其实时间戳和截止时间戳
            if (startTimeTmp == 0 || fileTime < startTimeTmp) {
                startTimeTmp = fileTime;
            }

            if (endTimeTmp == 0 || fileTime > endTimeTmp) {
                endTimeTmp = fileTime;
            }

            hisDataPath.push_back(filePath);
        }
    }
    startTime = startTimeTmp;
    endTime = endTimeTmp;
}

void DataStorageMgmt::read_history_data(const ghcfs::path filePath, std::vector<uint8_t>& historyData)
{
    std::ifstream fileStream(filePath, std::ios::binary);
    if (fileStream.is_open()) 
    {
        const auto fileSize = ghc::filesystem::file_size(filePath);
        std::vector<uint8_t> fileContent(fileSize);
        fileStream.read((char*)fileContent.data(), fileSize);

        historyData.insert(historyData.end(), fileContent.begin(), fileContent.end());
    }
}

//当网关离线的时候，创建离线文件
void DataStorageMgmt::create_offline_data_file()
{
    if (isCanSave == false)
        return;

    auto strDateTime = mm_to<memepp::native_string>(get_datetime_string());

#if MG_OS__WIN_AVAIL
    auto offline_file_path = m_offline_folder_today + L"/" + strDateTime + L".txt";
#else
    auto offline_file_path = m_offline_folder_today + "/" + strDateTime + ".txt";
#endif

    if (strDateTime == m_strDateTime)
    {
        //保证每次只打开一个文件
        if (!m_offline_file_fs.is_open())
        {
            m_offline_file_fs.open(offline_file_path, std::ios::binary | std::ios::app);
            std::cout << "open offline file:" <<std::endl;
        }
    }
    else
    {
        m_strDateTime = strDateTime;
        m_offline_file_fs.close();
        m_offline_file_fs.open(offline_file_path, std::ios::binary | std::ios::app);
        std::cout << "open new offline file:" << std::endl;
    }
}

//保存离线数据
void DataStorageMgmt::save_offline_data_to_file(const uint8_t data[], const size_t dataSize)
{
    if (isCanSave == false)
        return;

    if (m_offline_file_fs.is_open())
    {
        // 写入数据长度
        m_offline_file_fs.write(reinterpret_cast<const char*>(&dataSize), sizeof(std::size_t));

        // 写入 uint8_t 数据
        m_offline_file_fs.write(reinterpret_cast<const char*>(data), dataSize * sizeof(uint8_t));

        m_offline_file_fs.flush();

        std::cout << "save offline data" << std::endl;
    }
    else
    {
        std::cout << "can't open offline file" << std::endl;
    }

    check_to_next_hour();
}

//获取所有离线数据文件路径，等待发送
void DataStorageMgmt::get_offline_flie_list(std::vector<ghc::filesystem::path>& offlineDataPath)
{
    const auto folderPath = ghcfs::absolute(m_offline_folder);

    if (!is_directory(folderPath)) {
        std::cout << "Invalid offline folder path." << std::endl;
    }

    for (const auto& dateFolder : ghcfs::directory_iterator(folderPath))
    {
        for (const auto& entry : ghcfs::directory_iterator(dateFolder))
        {
            if (ghcfs::is_directory(entry)) {
                continue;
            }

            offlineDataPath.push_back(entry.path());
        }
    }
}

//读取离线数据
void DataStorageMgmt::read_offline_data(const ghcfs::path offline_file_path, std::vector<std::vector<uint8_t>>& cache)
{    
     std::ifstream file;
     file.open(offline_file_path, std::ios::binary);
     if (file.is_open()) 
     {
         // 读取所有数据
         while (file.peek() != EOF)
         {
             // 读取数据长度
             std::size_t dataSize;
             file.read(reinterpret_cast<char*>(&dataSize), sizeof(std::size_t));

             if (dataSize > 160000)
             {
                 continue;
             }

             // 读取数据
             std::vector<uint8_t> data(dataSize);
             file.read(reinterpret_cast<char*>(data.data()), dataSize * sizeof(uint8_t));

             // 添加到结果数组
             cache.push_back(data);
         }

         file.close();
         remove_file(offline_file_path);
         std::cout << "read offline data success:" << offline_file_path.string() << std::endl;
     }
     else 
     {
         std::cout << "can't open offline file" << std::endl;
     }
}

void DataStorageMgmt::check_to_next_hour()
{
    auto strDateTime = mm_to<memepp::native_string>(get_datetime_string());

    if (strDateTime != m_strDateTime)
    {
        std::cout << "check_to_next_hour and create offline file" << std::endl;

        create_offline_data_file();
    }
}

void DataStorageMgmt::remove_file(memepp::native_string filePath)
{
    if (ghcfs::exists(filePath))
    {
        ghcfs::remove(filePath);
        std::cout << "delect file success" << std::endl;

        // 检查文件所在文件夹是否为空
        std::string folderPath = ghcfs::path(filePath).parent_path().string();
        if (ghcfs::is_empty(folderPath)) 
        {
            std::string  folderPathTmp  = folderPath.substr(folderPath.length() - 8, 8);
            auto nameTmp = memepp::string(folderPathTmp.c_str(), folderPathTmp.length());
            auto strTmp = get_date_string();

            //当天的文件名，不用删除
            if (nameTmp != strTmp)
            {
                // 删除文件夹
                ghcfs::remove(folderPath);
                std::cout << "delete folder sucess" << folderPath << std::endl;
            }
        }
    }
}

//停止存储离线数据
void DataStorageMgmt::stop_save_offline()
{
    if (m_offline_file_fs.is_open())
    {
        m_offline_file_fs.close();
    }
}