mirror of
https://github.com/Nezumi-2711/astrbot_plugin_qq_group_daily_analysis.git
synced 2026-09-22 20:01:04 +00:00
refactor(增量分析): 从按天存储改为滑动窗口批次架构
- IncrementalBatch: 独立批次实体,每次增量分析产生一个,按批次独立存储到KV - IncrementalState: 聚合视图,不再持久化,报告时由merge_batches合并产生 - IncrementalStore: 批次索引+数据KV持久化,支持按时间窗口查询和过期清理 - IncrementalMergeService: 新增merge_batches方法,负责批次合并和话题/金句去重 - AnalysisApplicationService: 增量分析存独立批次,最终报告按窗口查询合并 - AutoScheduler: 报告发送后清理2×窗口外的过期批次 - main.py: /增量状态命令改为滑动窗口查询展示 - 消除天然日期隔离问题,24h图表展示完整数据
This commit is contained in:
@@ -4,8 +4,8 @@
|
||||
该模块导出所有领域实体类,包括:
|
||||
- AnalysisTask: 分析任务聚合根
|
||||
- GroupAnalysisResult: 群聊分析结果实体
|
||||
- IncrementalState: 增量分析状态实体
|
||||
- BatchRecord: 增量分析批次记录
|
||||
- IncrementalBatch: 增量分析独立批次实体
|
||||
- IncrementalState: 增量分析聚合视图(报告时使用)
|
||||
"""
|
||||
|
||||
from .analysis_result import (
|
||||
@@ -19,7 +19,7 @@ from .analysis_result import (
|
||||
UserTitle,
|
||||
)
|
||||
from .analysis_task import AnalysisTask, TaskStatus
|
||||
from .incremental_state import BatchRecord, IncrementalState
|
||||
from .incremental_state import IncrementalBatch, IncrementalState
|
||||
|
||||
# 别名,保持向后兼容
|
||||
AnalysisResult = GroupAnalysisResult
|
||||
@@ -36,6 +36,6 @@ __all__ = [
|
||||
"EmojiStatistics",
|
||||
"ActivityVisualization",
|
||||
"GroupStatistics",
|
||||
"IncrementalBatch",
|
||||
"IncrementalState",
|
||||
"BatchRecord",
|
||||
]
|
||||
|
||||
@@ -1,88 +1,171 @@
|
||||
"""
|
||||
增量分析状态实体
|
||||
增量分析实体 — 滑动窗口批次存储架构
|
||||
|
||||
存储单个群聊在一天内累积的增量分析数据。
|
||||
每次增量分析产生一个批次(batch),批次结果合并到此状态中。
|
||||
最终报告时从此状态中提取完整的统计数据和分析内容。
|
||||
核心概念:
|
||||
- IncrementalBatch: 单次增量分析产生的独立批次数据,按批次独立存储
|
||||
- IncrementalState: 报告生成时由多个批次合并而成的聚合视图(不再持久化)
|
||||
|
||||
滑动窗口设计:
|
||||
- 每次增量分析产生一个 IncrementalBatch,独立存储到 KV
|
||||
- 最终报告时按 analysis_days × 24h 的时间窗口查询批次并合并
|
||||
- 支持同一天多次发送报告,每次都基于当前时间窗口内的所有批次
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchRecord:
|
||||
"""单次增量分析批次记录"""
|
||||
class IncrementalBatch:
|
||||
"""
|
||||
单次增量分析批次数据
|
||||
|
||||
batch_id: int = 0
|
||||
timestamp: float = 0.0
|
||||
message_count: int = 0
|
||||
new_topics_count: int = 0
|
||||
new_quotes_count: int = 0
|
||||
token_usage: dict = field(default_factory=dict)
|
||||
每次增量分析执行完毕后产生一个 IncrementalBatch,
|
||||
包含该批次的所有统计数据和 LLM 分析结果,独立存储到 KV。
|
||||
|
||||
Attributes:
|
||||
group_id: 群组 ID
|
||||
batch_id: 批次唯一标识(UUID)
|
||||
timestamp: 批次创建时间戳(epoch)
|
||||
messages_count: 本批次分析的消息数量
|
||||
characters_count: 本批次的总字符数
|
||||
hourly_msg_counts: 按小时的消息计数 {hour_str: count}
|
||||
hourly_char_counts: 按小时的字符计数 {hour_str: count}
|
||||
user_stats: 用户统计 {user_id: {name, message_count, char_count, ...}}
|
||||
emoji_stats: 表情统计 {emoji_type: count}
|
||||
topics: 本批次提取的话题列表
|
||||
golden_quotes: 本批次提取的金句列表
|
||||
token_usage: 本批次 token 消耗 {prompt_tokens, completion_tokens, total_tokens}
|
||||
last_message_timestamp: 本批次最后一条消息的时间戳
|
||||
participant_ids: 本批次参与者 ID 列表
|
||||
"""
|
||||
|
||||
group_id: str = ""
|
||||
batch_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
# 统计数据
|
||||
messages_count: int = 0
|
||||
characters_count: int = 0
|
||||
hourly_msg_counts: dict[str, int] = field(default_factory=dict)
|
||||
hourly_char_counts: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
# 用户活跃数据
|
||||
user_stats: dict[str, dict] = field(default_factory=dict)
|
||||
|
||||
# 表情统计
|
||||
emoji_stats: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
# LLM 分析结果
|
||||
topics: list[dict] = field(default_factory=list)
|
||||
golden_quotes: list[dict] = field(default_factory=list)
|
||||
|
||||
# Token 消耗
|
||||
token_usage: dict = field(default_factory=lambda: {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
})
|
||||
|
||||
# 增量追踪
|
||||
last_message_timestamp: int = 0
|
||||
participant_ids: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""序列化为字典"""
|
||||
"""序列化为字典,用于 KV 存储"""
|
||||
return {
|
||||
"group_id": self.group_id,
|
||||
"batch_id": self.batch_id,
|
||||
"timestamp": self.timestamp,
|
||||
"message_count": self.message_count,
|
||||
"new_topics_count": self.new_topics_count,
|
||||
"new_quotes_count": self.new_quotes_count,
|
||||
"messages_count": self.messages_count,
|
||||
"characters_count": self.characters_count,
|
||||
"hourly_msg_counts": self.hourly_msg_counts,
|
||||
"hourly_char_counts": self.hourly_char_counts,
|
||||
"user_stats": self.user_stats,
|
||||
"emoji_stats": self.emoji_stats,
|
||||
"topics": self.topics,
|
||||
"golden_quotes": self.golden_quotes,
|
||||
"token_usage": self.token_usage,
|
||||
"last_message_timestamp": self.last_message_timestamp,
|
||||
"participant_ids": self.participant_ids,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "BatchRecord":
|
||||
def from_dict(cls, data: dict) -> "IncrementalBatch":
|
||||
"""从字典反序列化"""
|
||||
return cls(
|
||||
batch_id=data.get("batch_id", 0),
|
||||
group_id=data.get("group_id", ""),
|
||||
batch_id=data.get("batch_id", ""),
|
||||
timestamp=data.get("timestamp", 0.0),
|
||||
message_count=data.get("message_count", 0),
|
||||
new_topics_count=data.get("new_topics_count", 0),
|
||||
new_quotes_count=data.get("new_quotes_count", 0),
|
||||
token_usage=data.get("token_usage", {}),
|
||||
messages_count=data.get("messages_count", 0),
|
||||
characters_count=data.get("characters_count", 0),
|
||||
hourly_msg_counts=data.get("hourly_msg_counts", {}),
|
||||
hourly_char_counts=data.get("hourly_char_counts", {}),
|
||||
user_stats=data.get("user_stats", {}),
|
||||
emoji_stats=data.get("emoji_stats", {}),
|
||||
topics=data.get("topics", []),
|
||||
golden_quotes=data.get("golden_quotes", []),
|
||||
token_usage=data.get("token_usage", {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
}),
|
||||
last_message_timestamp=data.get("last_message_timestamp", 0),
|
||||
participant_ids=data.get("participant_ids", []),
|
||||
)
|
||||
|
||||
def get_summary(self) -> dict:
|
||||
"""获取批次摘要信息"""
|
||||
return {
|
||||
"batch_id": self.batch_id[:8],
|
||||
"timestamp": datetime.fromtimestamp(self.timestamp).strftime(
|
||||
"%Y-%m-%d %H:%M:%S"
|
||||
),
|
||||
"messages_count": self.messages_count,
|
||||
"topics_count": len(self.topics),
|
||||
"quotes_count": len(self.golden_quotes),
|
||||
"participants": len(self.participant_ids),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class IncrementalState:
|
||||
"""
|
||||
增量分析状态聚合实体
|
||||
增量分析聚合视图(报告时使用)
|
||||
|
||||
该实体代表一个群聊在一天内的增量分析累积状态。
|
||||
随着当天多次增量分析的执行,话题、金句、统计数据会不断合并更新。
|
||||
由多个 IncrementalBatch 合并而成,不直接持久化。
|
||||
IncrementalMergeService.merge_batches() 负责从批次列表构建此对象。
|
||||
|
||||
Attributes:
|
||||
group_id: 群组 ID
|
||||
date_str: 日期字符串 (YYYY-MM-DD)
|
||||
topics: 累积的话题列表(每个元素为 dict,包含 topic/contributors/detail)
|
||||
golden_quotes: 累积的金句列表(每个元素为 dict,包含 content/sender/reason)
|
||||
hourly_message_counts: 每小时消息计数 {hour_int: count}
|
||||
hourly_character_counts: 每小时字符计数 {hour_int: count}
|
||||
user_activities: 用户活跃数据 {user_id: {name, message_count, char_count, ...}}
|
||||
emoji_counts: 表情统计 {emoji_type: count}
|
||||
batch_records: 已完成的增量分析批次记录
|
||||
total_message_count: 当天总消息数
|
||||
total_character_count: 当天总字符数
|
||||
total_analysis_count: 当天已执行的增量分析次数
|
||||
window_start: 滑动窗口起始时间戳
|
||||
window_end: 滑动窗口结束时间戳
|
||||
topics: 合并去重后的话题列表
|
||||
golden_quotes: 合并去重后的金句列表
|
||||
hourly_message_counts: 合并后的每小时消息计数 {hour_str: count}
|
||||
hourly_character_counts: 合并后的每小时字符计数 {hour_str: count}
|
||||
user_activities: 合并后的用户活跃数据
|
||||
emoji_counts: 合并后的表情统计
|
||||
total_message_count: 窗口内总消息数
|
||||
total_character_count: 窗口内总字符数
|
||||
total_analysis_count: 窗口内批次数量
|
||||
total_token_usage: 累计 token 消耗
|
||||
last_analyzed_message_timestamp: 上次分析的最后一条消息时间戳(用于去重)
|
||||
last_analyzed_message_timestamp: 最后分析消息时间戳
|
||||
all_participant_ids: 所有参与者 ID 集合
|
||||
created_at: 状态创建时间
|
||||
updated_at: 状态最后更新时间
|
||||
"""
|
||||
|
||||
# 标识信息
|
||||
group_id: str = ""
|
||||
date_str: str = ""
|
||||
window_start: float = 0.0
|
||||
window_end: float = 0.0
|
||||
|
||||
# 累积的 LLM 分析结果
|
||||
# 合并后的 LLM 分析结果
|
||||
topics: list[dict] = field(default_factory=list)
|
||||
golden_quotes: list[dict] = field(default_factory=list)
|
||||
|
||||
# 累积的统计数据(按小时)
|
||||
# 合并后的统计数据(按小时)
|
||||
hourly_message_counts: dict[str, int] = field(default_factory=dict)
|
||||
hourly_character_counts: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
@@ -92,9 +175,6 @@ class IncrementalState:
|
||||
# 表情统计
|
||||
emoji_counts: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
# 批次记录
|
||||
batch_records: list[BatchRecord] = field(default_factory=list)
|
||||
|
||||
# 汇总统计
|
||||
total_message_count: int = 0
|
||||
total_character_count: int = 0
|
||||
@@ -113,205 +193,6 @@ class IncrementalState:
|
||||
created_at: float = field(default_factory=time.time)
|
||||
updated_at: float = field(default_factory=time.time)
|
||||
|
||||
def merge_batch(
|
||||
self,
|
||||
messages_count: int,
|
||||
characters_count: int,
|
||||
hourly_msg_counts: dict[int, int],
|
||||
hourly_char_counts: dict[int, int],
|
||||
user_stats: dict[str, dict],
|
||||
emoji_stats: dict[str, int],
|
||||
new_topics: list[dict],
|
||||
new_quotes: list[dict],
|
||||
token_usage: dict,
|
||||
last_message_timestamp: int,
|
||||
participant_ids: set[str],
|
||||
) -> "BatchRecord":
|
||||
"""
|
||||
合并一次增量分析的结果到当前状态中。
|
||||
|
||||
Args:
|
||||
messages_count: 本批次分析的消息数量
|
||||
characters_count: 本批次的总字符数
|
||||
hourly_msg_counts: 本批次按小时的消息计数 {hour: count}
|
||||
hourly_char_counts: 本批次按小时的字符计数 {hour: count}
|
||||
user_stats: 本批次用户统计 {user_id: {name, message_count, char_count, ...}}
|
||||
emoji_stats: 本批次表情统计 {emoji_type: count}
|
||||
new_topics: 本批次提取的新话题
|
||||
new_quotes: 本批次提取的新金句
|
||||
token_usage: 本批次 token 消耗 {prompt_tokens, completion_tokens, total_tokens}
|
||||
last_message_timestamp: 本批次最后一条消息的时间戳
|
||||
participant_ids: 本批次参与者 ID 集合
|
||||
|
||||
Returns:
|
||||
BatchRecord: 本次批次的记录
|
||||
"""
|
||||
# 更新统计汇总
|
||||
self.total_message_count += messages_count
|
||||
self.total_character_count += characters_count
|
||||
self.total_analysis_count += 1
|
||||
|
||||
# 合并小时统计
|
||||
for hour, count in hourly_msg_counts.items():
|
||||
hour_key = str(hour)
|
||||
self.hourly_message_counts[hour_key] = (
|
||||
self.hourly_message_counts.get(hour_key, 0) + count
|
||||
)
|
||||
for hour, count in hourly_char_counts.items():
|
||||
hour_key = str(hour)
|
||||
self.hourly_character_counts[hour_key] = (
|
||||
self.hourly_character_counts.get(hour_key, 0) + count
|
||||
)
|
||||
|
||||
# 合并用户活跃数据
|
||||
for user_id, stats in user_stats.items():
|
||||
if user_id in self.user_activities:
|
||||
existing = self.user_activities[user_id]
|
||||
existing["message_count"] = (
|
||||
existing.get("message_count", 0) + stats.get("message_count", 0)
|
||||
)
|
||||
existing["char_count"] = (
|
||||
existing.get("char_count", 0) + stats.get("char_count", 0)
|
||||
)
|
||||
existing["emoji_count"] = (
|
||||
existing.get("emoji_count", 0) + stats.get("emoji_count", 0)
|
||||
)
|
||||
# 合并活跃小时集合
|
||||
existing_hours = set(existing.get("active_hours", []))
|
||||
new_hours = set(stats.get("active_hours", []))
|
||||
existing["active_hours"] = list(existing_hours | new_hours)
|
||||
# 更新最后发言时间
|
||||
if stats.get("last_message_time", 0) > existing.get("last_message_time", 0):
|
||||
existing["last_message_time"] = stats["last_message_time"]
|
||||
else:
|
||||
self.user_activities[user_id] = dict(stats)
|
||||
|
||||
# 合并表情统计
|
||||
for emoji_type, count in emoji_stats.items():
|
||||
self.emoji_counts[emoji_type] = (
|
||||
self.emoji_counts.get(emoji_type, 0) + count
|
||||
)
|
||||
|
||||
# 合并话题(带去重)
|
||||
for new_topic in new_topics:
|
||||
if not self._is_duplicate_topic(new_topic):
|
||||
self.topics.append(new_topic)
|
||||
|
||||
# 合并金句(带去重)
|
||||
for new_quote in new_quotes:
|
||||
if not self._is_duplicate_quote(new_quote):
|
||||
self.golden_quotes.append(new_quote)
|
||||
|
||||
# 更新 token 消耗
|
||||
self.total_token_usage["prompt_tokens"] = (
|
||||
self.total_token_usage.get("prompt_tokens", 0)
|
||||
+ token_usage.get("prompt_tokens", 0)
|
||||
)
|
||||
self.total_token_usage["completion_tokens"] = (
|
||||
self.total_token_usage.get("completion_tokens", 0)
|
||||
+ token_usage.get("completion_tokens", 0)
|
||||
)
|
||||
self.total_token_usage["total_tokens"] = (
|
||||
self.total_token_usage.get("total_tokens", 0)
|
||||
+ token_usage.get("total_tokens", 0)
|
||||
)
|
||||
|
||||
# 更新增量追踪
|
||||
if last_message_timestamp > self.last_analyzed_message_timestamp:
|
||||
self.last_analyzed_message_timestamp = last_message_timestamp
|
||||
self.all_participant_ids.update(participant_ids)
|
||||
|
||||
# 更新时间戳
|
||||
self.updated_at = time.time()
|
||||
|
||||
# 创建批次记录
|
||||
batch = BatchRecord(
|
||||
batch_id=self.total_analysis_count,
|
||||
timestamp=time.time(),
|
||||
message_count=messages_count,
|
||||
new_topics_count=len(new_topics),
|
||||
new_quotes_count=len(new_quotes),
|
||||
token_usage=dict(token_usage),
|
||||
)
|
||||
self.batch_records.append(batch)
|
||||
|
||||
return batch
|
||||
|
||||
def _is_duplicate_topic(self, new_topic: dict, threshold: float = 0.6) -> bool:
|
||||
"""
|
||||
检测话题是否与已有话题重复。
|
||||
|
||||
使用简单的字符重叠相似度判断。
|
||||
当新话题的名称与已有话题名称相似度超过阈值时,认为是重复话题。
|
||||
|
||||
Args:
|
||||
new_topic: 待检测的新话题
|
||||
threshold: 相似度阈值(0-1),默认 0.6
|
||||
|
||||
Returns:
|
||||
bool: 是否重复
|
||||
"""
|
||||
new_name = new_topic.get("topic", "")
|
||||
if not new_name:
|
||||
return False
|
||||
|
||||
for existing in self.topics:
|
||||
existing_name = existing.get("topic", "")
|
||||
if not existing_name:
|
||||
continue
|
||||
similarity = self._char_overlap_similarity(new_name, existing_name)
|
||||
if similarity >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_duplicate_quote(self, new_quote: dict, threshold: float = 0.7) -> bool:
|
||||
"""
|
||||
检测金句是否与已有金句重复。
|
||||
|
||||
Args:
|
||||
new_quote: 待检测的新金句
|
||||
threshold: 相似度阈值(0-1),默认 0.7
|
||||
|
||||
Returns:
|
||||
bool: 是否重复
|
||||
"""
|
||||
new_content = new_quote.get("content", "")
|
||||
if not new_content:
|
||||
return False
|
||||
|
||||
for existing in self.golden_quotes:
|
||||
existing_content = existing.get("content", "")
|
||||
if not existing_content:
|
||||
continue
|
||||
similarity = self._char_overlap_similarity(new_content, existing_content)
|
||||
if similarity >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _char_overlap_similarity(s1: str, s2: str) -> float:
|
||||
"""
|
||||
计算两个字符串的字符重叠相似度。
|
||||
|
||||
使用 Jaccard 相似系数:交集大小 / 并集大小。
|
||||
|
||||
Args:
|
||||
s1: 第一个字符串
|
||||
s2: 第二个字符串
|
||||
|
||||
Returns:
|
||||
float: 相似度值(0-1)
|
||||
"""
|
||||
if not s1 or not s2:
|
||||
return 0.0
|
||||
set1 = set(s1)
|
||||
set2 = set(s2)
|
||||
intersection = set1 & set2
|
||||
union = set1 | set2
|
||||
if not union:
|
||||
return 0.0
|
||||
return len(intersection) / len(union)
|
||||
|
||||
def get_peak_hours(self, top_n: int = 3) -> list[int]:
|
||||
"""
|
||||
获取消息最活跃的时段。
|
||||
@@ -365,71 +246,22 @@ class IncrementalState:
|
||||
users.sort(key=lambda x: x["message_count"], reverse=True)
|
||||
return users[:top_n]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
def get_window_date_str(self) -> str:
|
||||
"""
|
||||
序列化为字典,用于 KV 存储持久化。
|
||||
获取窗口的日期范围字符串,用于报告显示。
|
||||
|
||||
Returns:
|
||||
dict: 可 JSON 序列化的字典
|
||||
str: 如 "2024-01-15" 或 "2024-01-14 ~ 2024-01-15"
|
||||
"""
|
||||
return {
|
||||
"group_id": self.group_id,
|
||||
"date_str": self.date_str,
|
||||
"topics": self.topics,
|
||||
"golden_quotes": self.golden_quotes,
|
||||
"hourly_message_counts": self.hourly_message_counts,
|
||||
"hourly_character_counts": self.hourly_character_counts,
|
||||
"user_activities": self.user_activities,
|
||||
"emoji_counts": self.emoji_counts,
|
||||
"batch_records": [b.to_dict() for b in self.batch_records],
|
||||
"total_message_count": self.total_message_count,
|
||||
"total_character_count": self.total_character_count,
|
||||
"total_analysis_count": self.total_analysis_count,
|
||||
"total_token_usage": self.total_token_usage,
|
||||
"last_analyzed_message_timestamp": self.last_analyzed_message_timestamp,
|
||||
"all_participant_ids": list(self.all_participant_ids),
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
if self.window_start <= 0 or self.window_end <= 0:
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "IncrementalState":
|
||||
"""
|
||||
从字典反序列化。
|
||||
start_date = datetime.fromtimestamp(self.window_start).strftime("%Y-%m-%d")
|
||||
end_date = datetime.fromtimestamp(self.window_end).strftime("%Y-%m-%d")
|
||||
|
||||
Args:
|
||||
data: 从 KV 存储读取的字典数据
|
||||
|
||||
Returns:
|
||||
IncrementalState: 重建的状态实例
|
||||
"""
|
||||
state = cls(
|
||||
group_id=data.get("group_id", ""),
|
||||
date_str=data.get("date_str", ""),
|
||||
topics=data.get("topics", []),
|
||||
golden_quotes=data.get("golden_quotes", []),
|
||||
hourly_message_counts=data.get("hourly_message_counts", {}),
|
||||
hourly_character_counts=data.get("hourly_character_counts", {}),
|
||||
user_activities=data.get("user_activities", {}),
|
||||
emoji_counts=data.get("emoji_counts", {}),
|
||||
batch_records=[
|
||||
BatchRecord.from_dict(b)
|
||||
for b in data.get("batch_records", [])
|
||||
],
|
||||
total_message_count=data.get("total_message_count", 0),
|
||||
total_character_count=data.get("total_character_count", 0),
|
||||
total_analysis_count=data.get("total_analysis_count", 0),
|
||||
total_token_usage=data.get("total_token_usage", {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
}),
|
||||
last_analyzed_message_timestamp=data.get("last_analyzed_message_timestamp", 0),
|
||||
all_participant_ids=set(data.get("all_participant_ids", [])),
|
||||
created_at=data.get("created_at", time.time()),
|
||||
updated_at=data.get("updated_at", time.time()),
|
||||
)
|
||||
return state
|
||||
if start_date == end_date:
|
||||
return end_date
|
||||
return f"{start_date} ~ {end_date}"
|
||||
|
||||
def get_summary(self) -> dict:
|
||||
"""
|
||||
@@ -440,7 +272,7 @@ class IncrementalState:
|
||||
"""
|
||||
return {
|
||||
"group_id": self.group_id,
|
||||
"date": self.date_str,
|
||||
"window": self.get_window_date_str(),
|
||||
"total_messages": self.total_message_count,
|
||||
"total_characters": self.total_character_count,
|
||||
"total_analyses": self.total_analysis_count,
|
||||
@@ -455,3 +287,88 @@ class IncrementalState:
|
||||
),
|
||||
"peak_hours": self.get_peak_hours(3),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def is_duplicate_topic(
|
||||
new_topic: dict, existing_topics: list[dict], threshold: float = 0.6
|
||||
) -> bool:
|
||||
"""
|
||||
检测话题是否与已有话题重复。
|
||||
|
||||
使用简单的字符重叠相似度判断。
|
||||
当新话题的名称与已有话题名称相似度超过阈值时,认为是重复话题。
|
||||
|
||||
Args:
|
||||
new_topic: 待检测的新话题
|
||||
existing_topics: 已有话题列表
|
||||
threshold: 相似度阈值(0-1),默认 0.6
|
||||
|
||||
Returns:
|
||||
bool: 是否重复
|
||||
"""
|
||||
new_name = new_topic.get("topic", "")
|
||||
if not new_name:
|
||||
return False
|
||||
|
||||
for existing in existing_topics:
|
||||
existing_name = existing.get("topic", "")
|
||||
if not existing_name:
|
||||
continue
|
||||
similarity = IncrementalState.char_overlap_similarity(
|
||||
new_name, existing_name
|
||||
)
|
||||
if similarity >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_duplicate_quote(
|
||||
new_quote: dict, existing_quotes: list[dict], threshold: float = 0.7
|
||||
) -> bool:
|
||||
"""
|
||||
检测金句是否与已有金句重复。
|
||||
|
||||
Args:
|
||||
new_quote: 待检测的新金句
|
||||
existing_quotes: 已有金句列表
|
||||
threshold: 相似度阈值(0-1),默认 0.7
|
||||
|
||||
Returns:
|
||||
bool: 是否重复
|
||||
"""
|
||||
new_content = new_quote.get("content", "")
|
||||
if not new_content:
|
||||
return False
|
||||
|
||||
for existing in existing_quotes:
|
||||
existing_content = existing.get("content", "")
|
||||
if not existing_content:
|
||||
continue
|
||||
similarity = IncrementalState.char_overlap_similarity(
|
||||
new_content, existing_content
|
||||
)
|
||||
if similarity >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def char_overlap_similarity(s1: str, s2: str) -> float:
|
||||
"""
|
||||
计算两个字符串的字符重叠相似度(Jaccard 相似系数)。
|
||||
|
||||
Args:
|
||||
s1: 第一个字符串
|
||||
s2: 第二个字符串
|
||||
|
||||
Returns:
|
||||
float: 相似度值(0-1)
|
||||
"""
|
||||
if not s1 or not s2:
|
||||
return 0.0
|
||||
set1 = set(s1)
|
||||
set2 = set(s2)
|
||||
intersection = set1 & set2
|
||||
union = set1 | set2
|
||||
if not union:
|
||||
return 0.0
|
||||
return len(intersection) / len(union)
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
"""
|
||||
增量合并领域服务
|
||||
|
||||
负责将 IncrementalState 累积数据转换为现有实体类型,
|
||||
负责将 IncrementalBatch 列表合并为 IncrementalState,
|
||||
以及将 IncrementalState 累积数据转换为现有实体类型,
|
||||
以便复用现有的报告生成器和分发器。
|
||||
|
||||
核心职责:
|
||||
- merge_batches: 将多个 IncrementalBatch 合并为一个 IncrementalState(滑动窗口聚合)
|
||||
- IncrementalState → GroupStatistics(含 ActivityVisualization、EmojiStatistics)
|
||||
- IncrementalState → list[SummaryTopic]
|
||||
- IncrementalState → list[GoldenQuote]
|
||||
"""
|
||||
|
||||
from ...domain.entities.incremental_state import IncrementalState
|
||||
import time
|
||||
|
||||
from ...domain.entities.incremental_state import IncrementalBatch, IncrementalState
|
||||
from ...domain.models.data_models import (
|
||||
ActivityVisualization,
|
||||
EmojiStatistics,
|
||||
@@ -26,10 +30,126 @@ class IncrementalMergeService:
|
||||
"""
|
||||
增量合并服务
|
||||
|
||||
将一天内累积的增量分析状态转换为现有报告系统所需的数据结构,
|
||||
将滑动窗口内的多个批次数据合并为报告所需的数据结构,
|
||||
确保增量模式下生成的最终报告与传统单次分析报告格式完全一致。
|
||||
"""
|
||||
|
||||
def merge_batches(
|
||||
self,
|
||||
batches: list[IncrementalBatch],
|
||||
window_start: float,
|
||||
window_end: float,
|
||||
) -> IncrementalState:
|
||||
"""
|
||||
从批次列表合并构建 IncrementalState。
|
||||
|
||||
遍历所有批次,累加统计数据并对话题和金句执行去重,
|
||||
生成可用于报告的聚合视图。
|
||||
|
||||
Args:
|
||||
batches: 时间窗口内的批次列表(按时间升序)
|
||||
window_start: 窗口起始时间戳(epoch)
|
||||
window_end: 窗口结束时间戳(epoch)
|
||||
|
||||
Returns:
|
||||
IncrementalState: 合并后的聚合视图
|
||||
"""
|
||||
state = IncrementalState(
|
||||
group_id=batches[0].group_id if batches else "",
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
total_analysis_count=len(batches),
|
||||
created_at=window_start,
|
||||
updated_at=time.time(),
|
||||
)
|
||||
|
||||
for batch in batches:
|
||||
# 累加消息和字符计数
|
||||
state.total_message_count += batch.messages_count
|
||||
state.total_character_count += batch.characters_count
|
||||
|
||||
# 合并每小时消息分布(按键累加)
|
||||
for hour_key, count in batch.hourly_msg_counts.items():
|
||||
hour_str = str(hour_key)
|
||||
state.hourly_message_counts[hour_str] = (
|
||||
state.hourly_message_counts.get(hour_str, 0) + count
|
||||
)
|
||||
|
||||
# 合并每小时字符分布
|
||||
for hour_key, count in batch.hourly_char_counts.items():
|
||||
hour_str = str(hour_key)
|
||||
state.hourly_character_counts[hour_str] = (
|
||||
state.hourly_character_counts.get(hour_str, 0) + count
|
||||
)
|
||||
|
||||
# 合并用户统计(按用户累加消息数、字符数等)
|
||||
for user_id, stats in batch.user_stats.items():
|
||||
if user_id not in state.user_activities:
|
||||
state.user_activities[user_id] = {
|
||||
"name": stats.get("name", user_id),
|
||||
"message_count": 0,
|
||||
"char_count": 0,
|
||||
"emoji_count": 0,
|
||||
"active_hours": [],
|
||||
"last_message_time": 0,
|
||||
}
|
||||
existing = state.user_activities[user_id]
|
||||
existing["message_count"] += stats.get("message_count", 0)
|
||||
existing["char_count"] += stats.get("char_count", 0)
|
||||
existing["emoji_count"] += stats.get("emoji_count", 0)
|
||||
# 合并活跃小时(去重)
|
||||
existing_hours = set(existing.get("active_hours", []))
|
||||
existing_hours.update(stats.get("active_hours", []))
|
||||
existing["active_hours"] = list(existing_hours)
|
||||
# 取最后消息时间的较大值
|
||||
batch_last = stats.get("last_message_time", 0)
|
||||
if batch_last > existing.get("last_message_time", 0):
|
||||
existing["last_message_time"] = batch_last
|
||||
# 更新昵称(使用最新批次的昵称)
|
||||
name = stats.get("name", "")
|
||||
if name:
|
||||
existing["name"] = name
|
||||
|
||||
# 合并表情统计(按键累加)
|
||||
for emoji_key, count in batch.emoji_stats.items():
|
||||
state.emoji_counts[emoji_key] = (
|
||||
state.emoji_counts.get(emoji_key, 0) + count
|
||||
)
|
||||
|
||||
# 合并话题(去重)
|
||||
for topic in batch.topics:
|
||||
if not IncrementalState.is_duplicate_topic(topic, state.topics):
|
||||
state.topics.append(topic)
|
||||
|
||||
# 合并金句(去重)
|
||||
for quote in batch.golden_quotes:
|
||||
if not IncrementalState.is_duplicate_quote(quote, state.golden_quotes):
|
||||
state.golden_quotes.append(quote)
|
||||
|
||||
# 累加 token 消耗
|
||||
for token_key in ("prompt_tokens", "completion_tokens", "total_tokens"):
|
||||
state.total_token_usage[token_key] = (
|
||||
state.total_token_usage.get(token_key, 0)
|
||||
+ batch.token_usage.get(token_key, 0)
|
||||
)
|
||||
|
||||
# 合并参与者 ID(取并集)
|
||||
state.all_participant_ids.update(batch.participant_ids)
|
||||
|
||||
# 记录最后分析消息时间戳(取最大值)
|
||||
if batch.last_message_timestamp > state.last_analyzed_message_timestamp:
|
||||
state.last_analyzed_message_timestamp = batch.last_message_timestamp
|
||||
|
||||
logger.info(
|
||||
f"合并批次完成: 群={state.group_id}, "
|
||||
f"窗口={state.get_window_date_str()}, "
|
||||
f"批次数={len(batches)}, "
|
||||
f"总消息={state.total_message_count}, "
|
||||
f"话题={len(state.topics)}, 金句={len(state.golden_quotes)}"
|
||||
)
|
||||
|
||||
return state
|
||||
|
||||
def build_final_statistics(self, state: IncrementalState) -> GroupStatistics:
|
||||
"""
|
||||
从增量状态构建最终的群组统计数据。
|
||||
@@ -38,7 +158,7 @@ class IncrementalMergeService:
|
||||
包含完整的 24 小时活跃度分布、表情统计和 token 消耗。
|
||||
|
||||
Args:
|
||||
state: 当天的增量分析状态
|
||||
state: 由 merge_batches 合并生成的增量分析状态
|
||||
|
||||
Returns:
|
||||
GroupStatistics: 与传统分析格式一致的统计数据
|
||||
@@ -58,7 +178,7 @@ class IncrementalMergeService:
|
||||
# 构建活跃度可视化数据
|
||||
activity_visualization = ActivityVisualization(
|
||||
hourly_activity=hourly_activity,
|
||||
daily_activity={state.date_str: state.total_message_count},
|
||||
daily_activity={state.get_window_date_str(): state.total_message_count},
|
||||
user_activity_ranking=user_ranking,
|
||||
peak_hours=peak_hours,
|
||||
activity_heatmap_data={},
|
||||
@@ -106,7 +226,7 @@ class IncrementalMergeService:
|
||||
将 IncrementalState 中累积的话题字典转换为 SummaryTopic 实例列表。
|
||||
|
||||
Args:
|
||||
state: 当天的增量分析状态
|
||||
state: 由 merge_batches 合并生成的增量分析状态
|
||||
|
||||
Returns:
|
||||
list[SummaryTopic]: 话题列表,格式与传统分析结果一致
|
||||
@@ -130,7 +250,7 @@ class IncrementalMergeService:
|
||||
将 IncrementalState 中累积的金句字典转换为 GoldenQuote 实例列表。
|
||||
|
||||
Args:
|
||||
state: 当天的增量分析状态
|
||||
state: 由 merge_batches 合并生成的增量分析状态
|
||||
|
||||
Returns:
|
||||
list[GoldenQuote]: 金句列表,格式与传统分析结果一致
|
||||
@@ -160,7 +280,7 @@ class IncrementalMergeService:
|
||||
返回的 analysis_result 完全一致,可直接传入 ReportDispatcher。
|
||||
|
||||
Args:
|
||||
state: 当天的增量分析状态
|
||||
state: 由 merge_batches 合并生成的增量分析状态
|
||||
user_titles: 用户称号列表(由最终报告时 LLM 分析生成)
|
||||
|
||||
Returns:
|
||||
@@ -182,7 +302,7 @@ class IncrementalMergeService:
|
||||
|
||||
logger.info(
|
||||
f"从增量状态构建完整分析结果: "
|
||||
f"群={state.group_id}, 日期={state.date_str}, "
|
||||
f"群={state.group_id}, 窗口={state.get_window_date_str()}, "
|
||||
f"消息={state.total_message_count}, "
|
||||
f"话题={len(topics)}, "
|
||||
f"金句={len(golden_quotes)}, "
|
||||
|
||||
Reference in New Issue
Block a user