feat: 初始化腾讯IM分发工具骨架,含回调分发链路与历史消息补拉

- 应用骨架: Solon + MyBatis-Plus + PostgreSQL + Redis + Sa-Token
- 核心链路: 回调网关 → 消息落库(幂等) → 分发队列(FOR UPDATE SKIP LOCKED) → worker消费/重试/死信
- 多租户隔离: 前缀法账号映射, 所有业务表带 tenant_id
- 管理后台: FreeMarker 渲染, 租户/授权/队列/用量管理页面
- 历史消息补拉: 水位线驱动, getRoamMsg/getGroupMsg 增量拉取, 兜底回调丢失
- msg_key 统一算法(回调/补拉共享, 跨路径去重)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
yixiong
2026-07-08 20:28:32 +08:00
co-authored by Claude
parent 20bd78baa8
commit d22fdcfaad
80 changed files with 4955 additions and 0 deletions
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.AdminUser;
/**
* 管理后台用户 Mapper
*
* @author imutil
*/
public interface AdminUserMapper extends BaseMapper<AdminUser> {
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.ApiCallLog;
/**
* API 调用审计 Mapper
*
* @author imutil
*/
public interface ApiCallLogMapper extends BaseMapper<ApiCallLog> {
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.CrossTenantAudit;
/**
* 跨租户审计 Mapper
*
* @author imutil
*/
public interface CrossTenantAuditMapper extends BaseMapper<CrossTenantAudit> {
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.CrossTenantGrant;
/**
* 跨租户授权 Mapper
*
* @author imutil
*/
public interface CrossTenantGrantMapper extends BaseMapper<CrossTenantGrant> {
}
@@ -0,0 +1,81 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.DistQueue;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import java.time.OffsetDateTime;
import java.util.List;
/**
* 分发队列 Mapper
* <p>
* 提供 FOR UPDATE SKIP LOCKED 抢占消费的 SQL。
*
* @author imutil
*/
public interface DistQueueMapper extends BaseMapper<DistQueue> {
/**
* 抢占式拉取待分发记录:SELECT ... FOR UPDATE SKIP LOCKED
*
* @param limit 每轮条数
* @param lockedBy 抢占者标识
* @param lockTime 锁定时间
* @return 抢占到的记录
*/
@org.apache.ibatis.annotations.Select(
"SELECT id, msg_key, tenant_id, conv_id, target_url, payload, status, retry_count " +
"FROM dist_queue " +
"WHERE status = 0 AND next_retry_at <= #{now} " +
"ORDER BY next_retry_at, id " +
"LIMIT #{limit} " +
"FOR UPDATE SKIP LOCKED")
List<DistQueue> fetchPending(@Param("now") OffsetDateTime now,
@Param("limit") int limit);
/**
* 标记抢占:将 pending 置为 processing
*/
@Update("UPDATE dist_queue SET status = 1, locked_by = #{lockedBy}, locked_at = #{lockTime}, " +
"updated_at = #{lockTime} " +
"WHERE id = #{id} AND status = 0")
int lock(@Param("id") Long id,
@Param("lockedBy") String lockedBy,
@Param("lockTime") OffsetDateTime lockTime);
/**
* 标记分发成功
*/
@Update("UPDATE dist_queue SET status = 2, updated_at = #{now} WHERE id = #{id} AND status = 1")
int markDone(@Param("id") Long id, @Param("now") OffsetDateTime now);
/**
* 标记失败并安排重试(指数退避)
*/
@Update("UPDATE dist_queue SET status = CASE WHEN retry_count + 1 >= #{maxRetry} THEN 3 ELSE 0 END, " +
"retry_count = retry_count + 1, next_retry_at = #{nextRetryAt}, updated_at = #{now} " +
"WHERE id = #{id} AND status = 1")
int markRetry(@Param("id") Long id,
@Param("maxRetry") int maxRetry,
@Param("nextRetryAt") OffsetDateTime nextRetryAt,
@Param("now") OffsetDateTime now);
/**
* 巡检:重置超时未回执的 processing 回到 pending(工作线程宕机恢复)
*/
@Update("UPDATE dist_queue SET status = 0, locked_by = NULL, locked_at = NULL, updated_at = #{now} " +
"WHERE status = 1 AND locked_at < #{threshold}")
int recoverStuck(@Param("threshold") OffsetDateTime threshold, @Param("now") OffsetDateTime now);
/**
* 死信重发:将 status=3(dead) 的记录重置为 pendingretry_count 归零),供工作线程重新消费
*
* @return 影响行数(0 表示该记录非死信或已处理)
*/
@Update("UPDATE dist_queue SET status = 0, retry_count = 0, next_retry_at = now(), " +
"locked_by = NULL, locked_at = NULL, updated_at = now() " +
"WHERE id = #{id} AND status = 3")
int redeliverDead(@Param("id") Long id);
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.GroupMapping;
/**
* 群映射 Mapper
*
* @author imutil
*/
public interface GroupMappingMapper extends BaseMapper<GroupMapping> {
}
@@ -0,0 +1,32 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.ImMessage;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import java.time.OffsetDateTime;
/**
* 消息主表 Mapper
*
* @author imutil
*/
public interface ImMessageMapper extends BaseMapper<ImMessage> {
/**
* 原子更新分发状态(乐观锁:仅当当前状态为期望值时更新)
*
* @param msgKey 消息键
* @param msgTime 消息时间(复合主键的一部分)
* @param fromStatus 期望的当前状态
* @param toStatus 目标状态
* @return 影响行数
*/
@Update("UPDATE im_message SET dist_status = #{toStatus} " +
"WHERE msg_key = #{msgKey} AND msg_time = #{msgTime} AND dist_status = #{fromStatus}")
int updateDistStatus(@Param("msgKey") String msgKey,
@Param("msgTime") OffsetDateTime msgTime,
@Param("fromStatus") int fromStatus,
@Param("toStatus") int toStatus);
}
@@ -0,0 +1,61 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.PullWatermark;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.time.OffsetDateTime;
import java.util.List;
/**
* 补拉水位线 Mapper
* <p>
* upsert 用 PG ON CONFLICTselectNeedCheck 按 updated_at 升序取最久未补拉的会话,
* 配合补拉成功后推进 updated_at,实现会话轮询公平覆盖。
*
* @author imutil
*/
public interface PullWatermarkMapper extends BaseMapper<PullWatermark> {
/**
* 水位线 upsertPG ON CONFLICT
* <p>
* 回调落库 / 补拉推进时调用。last_seq、last_time 取 GREATEST 不回退(防乱序回调回退游标),
* updated_at 始终刷新为 now(驱动 selectNeedCheck 的轮询顺序)。
*
* @param tenantId 租户ID
* @param convId 会话ID
* @param convType 会话类型 1=C2C 2=GROUP
* @param lastSeq 最新消息 Seq
* @param lastTime 最新消息时间
* @param now 当前时间(updated_at
* @return 影响行数(1=新增或更新)
*/
@Insert("INSERT INTO pull_watermark (tenant_id, conv_id, conv_type, last_seq, last_time, updated_at) " +
"VALUES (#{tenantId}, #{convId}, #{convType}, #{lastSeq}, #{lastTime}, #{now}) " +
"ON CONFLICT (tenant_id, conv_id) DO UPDATE SET " +
"last_seq = GREATEST(pull_watermark.last_seq, EXCLUDED.last_seq), " +
"last_time = GREATEST(pull_watermark.last_time, EXCLUDED.last_time), " +
"updated_at = EXCLUDED.updated_at")
int upsert(@Param("tenantId") String tenantId,
@Param("convId") String convId,
@Param("convType") int convType,
@Param("lastSeq") long lastSeq,
@Param("lastTime") OffsetDateTime lastTime,
@Param("now") OffsetDateTime now);
/**
* 取最久未补拉的 N 个会话(按 updated_at 升序)
* <p>
* 补拉任务每轮调用,updated_at 最老的优先;补拉后 updated_at 推进到 now
* 该会话自然排到队尾,实现轮询式公平覆盖。
*
* @param limit 每轮会话数
* @return 待补拉会话列表
*/
@Select("SELECT tenant_id, conv_id, conv_type, last_seq, last_time, updated_at " +
"FROM pull_watermark ORDER BY updated_at ASC LIMIT #{limit}")
List<PullWatermark> selectNeedCheck(@Param("limit") int limit);
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.Recording;
/**
* 录制文件 Mapper
*
* @author imutil
*/
public interface RecordingMapper extends BaseMapper<Recording> {
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.Tenant;
/**
* 租户表 Mapper
*
* @author imutil
*/
public interface TenantMapper extends BaseMapper<Tenant> {
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.TrtcRoom;
/**
* TRTC 房间 Mapper
*
* @author imutil
*/
public interface TrtcRoomMapper extends BaseMapper<TrtcRoom> {
}
@@ -0,0 +1,60 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.UsageStat;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import java.time.OffsetDateTime;
/**
* 用量统计 Mapper
* <p>
* 提供按小时窗口的聚合 upsertPG ON CONFLICT 幂等,可重复跑)。
* 两个数据源各自 upsert 指定列,避免互相覆盖:
* - im_message → im_msg_count / im_dau
* - api_call_log → api_call_count
*
* @author imutil
*/
public interface UsageStatMapper extends BaseMapper<UsageStat> {
/**
* 从 im_message 聚合某小时窗口的消息量与活跃用户数,upsert 到 usage_statstat_level=1 小时)
* <p>
* 仅更新 im_msg_count / im_dau,不触碰已有 api_call_count。
*
* @param start 窗口起点(含)
* @param end 窗口终点(不含)
* @param statTime 统计时刻(=窗口起点整点,作为 usage_stat.stat_time
* @return 受影响租户数
*/
@Insert("INSERT INTO usage_stat (tenant_id, stat_time, stat_level, im_msg_count, im_dau, " +
"api_call_count, trtc_duration_sec, trtc_max_concurrent_room, created_at) " +
"SELECT tenant_id, #{statTime}, 1, count(*), count(DISTINCT from_account), 0, 0, 0, now() " +
"FROM im_message WHERE created_at >= #{start} AND created_at < #{end} " +
"GROUP BY tenant_id " +
"ON CONFLICT (tenant_id, stat_time, stat_level) DO UPDATE SET " +
"im_msg_count = EXCLUDED.im_msg_count, im_dau = EXCLUDED.im_dau")
int aggregateMsgHour(@Param("start") OffsetDateTime start,
@Param("end") OffsetDateTime end,
@Param("statTime") OffsetDateTime statTime);
/**
* 从 api_call_log 聚合某小时窗口的腾讯 API 调用次数,upsert 到 usage_statstat_level=1 小时)
* <p>
* 仅更新 api_call_count,不触碰已有 im_msg_count / im_dau。
*
* @return 受影响租户数
*/
@Insert("INSERT INTO usage_stat (tenant_id, stat_time, stat_level, im_msg_count, im_dau, " +
"api_call_count, trtc_duration_sec, trtc_max_concurrent_room, created_at) " +
"SELECT tenant_id, #{statTime}, 1, 0, 0, count(*), 0, 0, now() " +
"FROM api_call_log WHERE called_at >= #{start} AND called_at < #{end} " +
"GROUP BY tenant_id " +
"ON CONFLICT (tenant_id, stat_time, stat_level) DO UPDATE SET " +
"api_call_count = EXCLUDED.api_call_count")
int aggregateApiCallHour(@Param("start") OffsetDateTime start,
@Param("end") OffsetDateTime end,
@Param("statTime") OffsetDateTime statTime);
}
@@ -0,0 +1,12 @@
package com.imutil.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.imutil.entity.UserMapping;
/**
* 用户映射 Mapper
*
* @author imutil
*/
public interface UserMappingMapper extends BaseMapper<UserMapping> {
}