Merge pull request #262 from jolavillette/ImproveUploadStatistics

implement time-based upload stats (V2) with configurable retention and auto-cleanup

Author: csoler

src/file_sharing/p3filelists.cc

@@ -80,6 +80,8 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
     mLastDataRecvTS = 0 ;
     mTrustFriendNodesForBannedFiles = TRUST_FRIEND_NODES_FOR_BANNED_FILES_DEFAULT;
 	mLastPrimaryBanListChangeTimeStamp = 0;
+    mUploadStatsRetentionDays = 0;
+    mCumulativeUploadedAll = 0;
 
     // This is for the transmission of data
 
@@ -197,6 +199,7 @@ int p3FileDatabase::tick()
     if(mLastCleanupTime + 5 < now)
     {
         cleanup();
+        cleanupUploadStats(mUploadStatsRetentionDays);
         mLastCleanupTime = now ;
     }
 
@@ -379,12 +382,12 @@ cleanup = true;
 
     {
         RS_STACK_MUTEX(mFLSMtx) ;
-        RsFileListsUploadStatsItem *item = nullptr;
+        RsFileListsUploadStatsItemV2 *item = nullptr;
 
         for(auto it(mCumulativeUploaded.begin());it!=mCumulativeUploaded.end();++it)
         {
             if(item == nullptr)
-                item = new RsFileListsUploadStatsItem ;
+                item = new RsFileListsUploadStatsItemV2 ;
 
             item->hash_stats.insert(*it);
 
@@ -497,6 +500,15 @@ cleanup = true;
 
         kv.key = IGNORE_LIST_FLAGS_SS; kv.value = s; rskv->tlvkvs.pairs.push_back(kv);
 	}
+    {
+        std::string s;
+        rs_sprintf(s, "%d", mUploadStatsRetentionDays);
+
+        RsTlvKeyValue kv;
+        kv.key = UPLOAD_STATS_RETENTION_DAYS_SS;
+        kv.value = s;
+        rskv->tlvkvs.pairs.push_back(kv);
+    }
 
     /* Add KeyValue to saveList */
     sList.push_back(rskv);
@@ -529,7 +541,8 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
 	ignored_suffixes.push_back( ".part" );
 #endif
     mPrimaryBanList.clear();
-    mCumulativeUploaded.clear();
+        mCumulativeUploaded.clear();
+        mCumulativeUploadedAll = 0;
 
     for(std::list<RsItem *>::iterator it = load.begin(); it != load.end(); ++it)
     {
@@ -618,6 +631,12 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
                 if(sscanf(kit->value.c_str(),"%d",&t) == 1)
                     max_share_depth = (uint32_t)t ;
 			}
+            else if(kit->key == UPLOAD_STATS_RETENTION_DAYS_SS)
+            {
+                int t=0;
+                if(sscanf(kit->value.c_str(),"%d",&t) == 1)
+                    mUploadStatsRetentionDays = t;
+            }
 
             delete *it ;
             continue ;
@@ -654,7 +673,28 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
 
         if(fu)
         {
-            mCumulativeUploaded.insert(fu->hash_stats.begin(), fu->hash_stats.end()) ;
+            // Migration V1 (deprecated Feb 2026) -> V2: Set timestamp to now
+            rstime_t now = time(NULL);
+            RsDbg() << "UPLOADSTATS Migrating V1 stats (count: " << fu->hash_stats.size() << ") to V2";
+            for(auto const& [hash, bytes] : fu->hash_stats)
+            {
+                TimeBasedUploadStat& stat = mCumulativeUploaded[hash];
+                stat.total_bytes = bytes;
+                stat.last_upload_ts = now;
+                mCumulativeUploadedAll += bytes;
+            }
+        }
+
+        RsFileListsUploadStatsItemV2 *fu2 = dynamic_cast<RsFileListsUploadStatsItemV2*>(*it) ;
+
+        if(fu2)
+        {
+            RsDbg() << "UPLOADSTATS Loading V2 stats (count: " << fu2->hash_stats.size() << ")";
+            for(auto const& it_item : fu2->hash_stats)
+            {
+                mCumulativeUploaded[it_item.first] = it_item.second ;
+                mCumulativeUploadedAll += it_item.second.total_bytes ;
+            }
         }
 
         delete *it ;
@@ -1056,17 +1096,14 @@ uint64_t p3FileDatabase::getCumulativeUpload(const RsFileHash& hash) const
 	RS_STACK_MUTEX(mFLSMtx);
 	auto it = mCumulativeUploaded.find(hash);
 	if (it != mCumulativeUploaded.end())
-		return it->second;
+		return it->second.total_bytes;
 	return 0;
 }
 
 uint64_t p3FileDatabase::getCumulativeUploadAll() const
 {
 	RS_STACK_MUTEX(mFLSMtx);
-	uint64_t total = 0;
-	for (auto it = mCumulativeUploaded.begin(); it != mCumulativeUploaded.end(); ++it)
-		total += it->second;
-	return total;
+	return mCumulativeUploadedAll;
 }
 
 uint64_t p3FileDatabase::getCumulativeUploadNum() const
@@ -1078,15 +1115,93 @@ uint64_t p3FileDatabase::getCumulativeUploadNum() const
 void p3FileDatabase::addUploadStats(const RsFileHash& hash, uint64_t size)
 {
 	RS_STACK_MUTEX(mFLSMtx);
-	mCumulativeUploaded[hash] += size;
+    TimeBasedUploadStat& stat = mCumulativeUploaded[hash];
+    stat.total_bytes += size;
+    stat.last_upload_ts = time(NULL);
+    mCumulativeUploadedAll += size;
+
+    // RsDbg() << "UPLOADSTATS add stats: " << hash << " + " << size << " bytes. Total: " << stat.total_bytes << " ts: " << stat.last_upload_ts;
 	IndicateConfigChanged(RsConfigMgr::CheckPriority::SAVE_OFTEN);
 }
 
 void p3FileDatabase::clearUploadStats()
 {
+	RS_STACK_MUTEX(mFLSMtx);
+    RsDbg() << "UPLOADSTATS clearing all stats";
 	mCumulativeUploaded.clear();
+    mCumulativeUploadedAll = 0;
+}
+
+void p3FileDatabase::cleanupUploadStats(int days)
+{
+    RS_STACK_MUTEX(mFLSMtx);
+    rstime_t now = time(NULL);
+    rstime_t cutoff = now - (rstime_t)days * 24 * 3600;
+    uint32_t removed_count = 0;
+
+    // RsDbg() << "UPLOADSTATS cleanup stats (retention: " << days << " days)";
+    
+    for (auto it = mCumulativeUploaded.begin(); it != mCumulativeUploaded.end(); )
+    {
+        bool expired = (days > 0) && (it->second.last_upload_ts < cutoff);
+        bool still_shared = false;
+        
+        // Optional: check if file is still shared. 
+        // We only do this check if it's not already expired, to save some time.
+        if (!expired)
+        {
+            RsFileHash real_hash;
+            DirectoryStorage::EntryIndex indx;
+            still_shared = mLocalSharedDirs->searchHash(it->first, real_hash, indx);
+        }
+
+        if (expired || !still_shared)
+        {
+            if (expired)
+            {
+                RsDbg() << "UPLOADSTATS removing expired stat: " << it->first << " (ts: " << it->second.last_upload_ts << ")";
+            }
+            else
+            {
+                RsDbg() << "UPLOADSTATS removing stat for unshared file: " << it->first;
+            }
+
+            mCumulativeUploadedAll -= it->second.total_bytes;
+            it = mCumulativeUploaded.erase(it);
+            removed_count++;
+        }
+        else
+        {
+            ++it;
+        }
+    }
+    if (removed_count > 0)
+    {
+        RsDbg() << "UPLOADSTATS cleanup removed " << removed_count << " entries.";
+        IndicateConfigChanged(RsConfigMgr::CheckPriority::SAVE_OFTEN);
+    }
+}
+
+void p3FileDatabase::setUploadStatsRetentionDays(int days)
+{
+    if (mUploadStatsRetentionDays != days)
+    {
+        mUploadStatsRetentionDays = days;
+        RsDbg() << "UPLOADSTATS setting retention days to: " << days;
+        IndicateConfigChanged(RsConfigMgr::CheckPriority::SAVE_OFTEN);
+        
+        // Trigger cleanup immediately if days > 0
+        if (days > 0)
+        {
+            cleanupUploadStats(days);
+        }
+    }
 }
 
+int p3FileDatabase::getUploadStatsRetentionDays() const
+{
+    return mUploadStatsRetentionDays;
+}
 bool p3FileDatabase::removeExtraFile(const RsFileHash& hash)
 {
 	bool ret = false;

src/file_sharing/p3filelists.h

@@ -62,6 +62,7 @@
 #include "ft/ftsearch.h"
 #include "ft/ftextralist.h"
 #include "retroshare/rsfiles.h"
+#include "file_sharing/rsfilelistitems.h"
 #include "services/p3service.h"
 #include "util/rstime.h"
 #include "file_sharing/hash_cache.h"
@@ -177,6 +178,12 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         virtual uint64_t getCumulativeUploadNum() const;
         virtual void addUploadStats(const RsFileHash& hash, uint64_t size);
         void clearUploadStats();
+        void cleanupUploadStats(int days);
+
+        void setUploadStatsRetentionDays(int days);
+        int getUploadStatsRetentionDays() const;
+
+
 
         // interface for hash caching
 
@@ -301,7 +308,12 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
         bool mBannedFileListNeedsUpdate;
         rstime_t mLastPrimaryBanListChangeTimeStamp;
 
-        std::map<RsFileHash, uint64_t> mCumulativeUploaded;
+        /**
+         * @brief Map of uploaded files statistics (bytes + timestamp)
+         */
+        std::map<RsFileHash, TimeBasedUploadStat> mCumulativeUploaded;
+        uint64_t mCumulativeUploadedAll;
+        int mUploadStatsRetentionDays;
 
         void locked_sendBanInfo(const RsPeerId& pid);
         void handleBannedFilesInfo(RsFileListsBannedHashesItem *item);

src/file_sharing/rsfilelistitems.cc

@@ -21,10 +21,16 @@
  ******************************************************************************/
 #include "serialiser/rsbaseserial.h"
 
-#include "serialiser/rstypeserializer.h"
+#include <serialiser/rstypeserializer.h>
 
 #include "file_sharing/rsfilelistitems.h"
 
+template<> void RsTypeSerializer::serial_process<TimeBasedUploadStat>(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, TimeBasedUploadStat& v, const std::string& /*name*/)
+{
+    serial_process(j, ctx, v.last_upload_ts, "last_upload_ts");
+    serial_process(j, ctx, v.total_bytes, "total_bytes");
+}
+
 void RsFileListsSyncRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process          (j,ctx,entry_hash,"entry_hash") ;
@@ -55,6 +61,11 @@ void RsFileListsUploadStatsItem::serial_process(RsGenericSerializer::SerializeJo
     RsTypeSerializer::serial_process(j,ctx,hash_stats,"hash_stats") ;
 }
 
+void RsFileListsUploadStatsItemV2::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process(j,ctx,hash_stats,"hash_stats") ;
+}
+
 RsItem *RsFileListsSerialiser::create_item(uint16_t service,uint8_t type) const
 {
     if(service != RS_SERVICE_TYPE_FILE_DATABASE)
@@ -67,6 +78,7 @@ RsItem *RsFileListsSerialiser::create_item(uint16_t service,uint8_t type) const
     case RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM:        return new RsFileListsBannedHashesItem();
     case RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_CONFIG_ITEM: return new RsFileListsBannedHashesConfigItem();
     case RS_PKT_SUBTYPE_FILELISTS_UPLOAD_STATS_ITEM:         return new RsFileListsUploadStatsItem();
+    case RS_PKT_SUBTYPE_FILELISTS_UPLOAD_STATS_ITEM_V2:      return new RsFileListsUploadStatsItemV2();
     default:
         return NULL ;
     }

src/file_sharing/rsfilelistitems.h

@@ -41,6 +41,7 @@ const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM               = 0x03;
 const uint8_t RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM        = 0x04;
 const uint8_t RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_CONFIG_ITEM = 0x05;
 const uint8_t RS_PKT_SUBTYPE_FILELISTS_UPLOAD_STATS_ITEM         = 0x06;
+const uint8_t RS_PKT_SUBTYPE_FILELISTS_UPLOAD_STATS_ITEM_V2       = 0x07;
 
 /*!
  * Base class for filelist sync items
@@ -128,6 +129,10 @@ class RsFileListsBannedHashesConfigItem: public RsFileListsItem
     std::map<RsFileHash,BannedFileEntry> primary_banned_files_list ;
 };
 
+/**
+ * @deprecated Deprecated on Feb 2026. Use RsFileListsUploadStatsItemV2 instead.
+ * To be removed when no longer needed for migration.
+ */
 class RsFileListsUploadStatsItem: public RsFileListsItem
 {
 public:
@@ -139,6 +144,27 @@ class RsFileListsUploadStatsItem: public RsFileListsItem
     std::map<RsFileHash, uint64_t> hash_stats;
 };
 
+struct TimeBasedUploadStat
+{
+    TimeBasedUploadStat() : last_upload_ts(0), total_bytes(0) {}
+
+    rstime_t last_upload_ts;
+    uint64_t total_bytes;
+
+    bool operator==(const TimeBasedUploadStat& r) const { return last_upload_ts == r.last_upload_ts && total_bytes == r.total_bytes; }
+};
+
+class RsFileListsUploadStatsItemV2: public RsFileListsItem
+{
+public:
+	RsFileListsUploadStatsItemV2() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_UPLOAD_STATS_ITEM_V2){}
+
+    virtual void clear() { hash_stats.clear(); }
+	virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+
+    std::map<RsFileHash, TimeBasedUploadStat> hash_stats;
+};
+
 class RsFileListsSerialiser : public RsServiceSerializer
 {
 public:

src/ft/ftserver.cc

@@ -2363,3 +2363,16 @@ void ftServer::clearUploadStats()
 {
 	return mFileDatabase->clearUploadStats();
 }
+
+void ftServer::setUploadStatsRetentionDays(int days)
+{
+    if (mFileDatabase)
+        mFileDatabase->setUploadStatsRetentionDays(days);
+}
+
+int ftServer::getUploadStatsRetentionDays()
+{
+    if (mFileDatabase)
+        return mFileDatabase->getUploadStatsRetentionDays();
+    return 0;
+}

src/ft/ftserver.h

@@ -207,6 +207,9 @@ class ftServer :
     virtual void setFilePermDirectDL(uint32_t perm)  override;
     virtual uint32_t filePermDirectDL()  override;
 
+    virtual void setUploadStatsRetentionDays(int days) override;
+    virtual int getUploadStatsRetentionDays() override;
+
 	/// @see RsFiles
 	std::error_condition requestFiles(
 	        const RsFileTree& collection,

src/retroshare/rsfiles.h

@@ -792,12 +792,24 @@ class RsFiles
 		virtual void setFilePermDirectDL(uint32_t perm)=0;
 
     /**
-     * @brief Get Direct Download File Permission
+     * @brief Set Direct Download File Permission
      * @jsonapi{development}
      * @return mFilePermDirectDLPolicy direct download permission
      */
 		virtual uint32_t filePermDirectDL()=0;
 
+    /**
+     * @brief Set Upload Statistics Retention in Days
+     * @param days Number of days to keep upload statistics (0 = keep forever)
+     */
+    virtual void setUploadStatsRetentionDays(int days) = 0;
+
+    /**
+     * @brief Get Upload Statistics Retention in Days
+     * @return Number of days to keep upload statistics
+     */
+    virtual int getUploadStatsRetentionDays() = 0;
+
 	/**
 	 * @brief Request remote files search
 	 * @jsonapi{development}

src/serialiser/rstlvkeys.h

@@ -43,6 +43,8 @@ const uint32_t RSTLV_KEY_DISTRIB_ADMIN          = 0x0040;
 const uint32_t RSTLV_KEY_DISTRIB_IDENTITY       = 0x0080;
 const uint32_t RSTLV_KEY_DISTRIB_MASK           = 0x00f0;
 
+static const std::string UPLOAD_STATS_RETENTION_DAYS_SS = "UPLOAD_STATS_RETENTION_DAYS";
+
 // Old class for RsTlvSecurityKey. Is kept for backward compatibility, but should not be serialised anymore
 
 class RsTlvRSAKey: public RsTlvItem