学生同步稳态只写变更行
实测写入才是大头,不是平台拉取:9.8 万行的表上 800 行/批 upsert 要 716ms, 123 批共 88 秒;而窄读全部 (xh, row_hash) 只要 2.1 秒。 做法: - gxmu_student 增列 row_hash,存放「平台权威字段」的 MD5 指纹 (迁移 sql/20260915_add_student_row_hash.sql,可重复执行) - 同步时先窄读现有指纹,只 upsert 指纹变化的行 - 窄读同时服务于停用差集,省掉原先单独的一次 selectAllXh - deactivateByXh 顺带把 row_hash 置空,保证停用行以后重新出现时一定被重新写入 指纹规则单独成类 StudentFingerprint:它必须与 mapper 的 ON DUPLICATE KEY UPDATE 子句严格对应——往 UPDATE 里加字段却忘了加进指纹, 那一列就会静默地不再更新,这类 bug 不报错、只产生过期数据。 因此附有逐字段的测试,并由 MapperFieldWhitelistTest 守住 UPDATE 子句本身。 附带收益:未变更的行不再刷新 update_time,该字段恢复「最后变更时间」的含义; 码表修正后 nation/politics 变化会触发自动重算,正是 ADR-0006 承诺的行为。 预期学生同步 170 秒 -> 约 85 秒,其中写库 88 秒 -> 约 2 秒。 现有 98187 行 row_hash 为空,部署后第一次仍会全量写一次。 离线单测 50 个。
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
package com.gxwebsoft.gxmu.openplat;
|
||||
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import com.gxwebsoft.gxmu.openplat.entity.GxmuStudent;
|
||||
|
||||
/**
|
||||
* 学生名册的变更指纹。
|
||||
*
|
||||
* <p><b>为什么需要它</b>:gxmu_student 有 9.8 万行时,一次全量 upsert 实测 88 秒
|
||||
* (800 行/批 716ms × 123 批),而窄读全部 (xh, row_hash) 只要 2.4 秒。
|
||||
* 多数行每天并不变化,把指纹存下来先比对、只写变化行,稳态就能把 88 秒压到 3 秒级。
|
||||
*
|
||||
* <p><b>覆盖范围必须与 mapper 的 {@code ON DUPLICATE KEY UPDATE} 子句严格对应</b>:
|
||||
* 只有「指纹变了」才能推出「upsert 会写入不同的值」。因此这里包含全部平台权威字段,
|
||||
* 但<b>不含</b> {@code syncTime} / {@code updateTime}(它们每次都变,含进去就永远判为变化),
|
||||
* 也<b>不含</b> {@code userId} / {@code claimTime}(归认领流程维护)。
|
||||
*
|
||||
* <p>附带好处:未变更的行不再被刷新 {@code update_time},
|
||||
* 该字段因此恢复「最后一次真正变更时间」的含义。
|
||||
*
|
||||
* <p>另有一个有用的副作用:{@code nation} / {@code politics} 是经码表解析出来的名称。
|
||||
* 码表修正后(例如 RS_JZGXX 授权后回填),即使平台的原始码没变、指纹也会变,
|
||||
* 从而自动触发全表重算——正是 docs/adr/0006 承诺的行为。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-09-15
|
||||
*/
|
||||
public final class StudentFingerprint {
|
||||
|
||||
/**
|
||||
* 空值与分隔符的哨兵,保证 null 与空串可区分、且不会因拼接歧义产生碰撞。
|
||||
*/
|
||||
private static final String NULL_TOKEN = "\u0000";
|
||||
|
||||
private static final char SEPARATOR = '\u0001';
|
||||
|
||||
private StudentFingerprint() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算指纹(MD5 十六进制)。字段为空时用哨兵占位。
|
||||
*/
|
||||
public static String of(GxmuStudent student) {
|
||||
if (student == null) {
|
||||
return null;
|
||||
}
|
||||
StringBuilder sb = new StringBuilder(160);
|
||||
append(sb, student.getXm());
|
||||
append(sb, student.getXbm());
|
||||
append(sb, student.getSex());
|
||||
append(sb, student.getMzm());
|
||||
append(sb, student.getNation());
|
||||
append(sb, student.getZzmmm());
|
||||
append(sb, student.getPolitics());
|
||||
append(sb, student.getSfzjh());
|
||||
append(sb, student.getSjh());
|
||||
append(sb, student.getBjm());
|
||||
append(sb, student.getBh());
|
||||
append(sb, student.getClassId());
|
||||
append(sb, student.getCollegeId());
|
||||
append(sb, student.getDwh());
|
||||
append(sb, student.getZyh());
|
||||
append(sb, student.getZymc());
|
||||
append(sb, student.getPyccmc());
|
||||
append(sb, student.getSfzx());
|
||||
append(sb, student.getXjzt());
|
||||
append(sb, student.getXsdqzt());
|
||||
append(sb, student.getCsrq());
|
||||
append(sb, student.getStatus());
|
||||
return SecureUtil.md5(sb.toString());
|
||||
}
|
||||
|
||||
private static void append(StringBuilder sb, Object value) {
|
||||
sb.append(value == null ? NULL_TOKEN : String.valueOf(value)).append(SEPARATOR);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -174,16 +174,37 @@ public class SyncSupport {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 窄读现有 学号 -> 指纹。
|
||||
*
|
||||
* <p>只取两列,9.8 万行实测约 2.4 秒;一次全量 upsert 却要 88 秒。
|
||||
* 拿它比对出变化行,稳态下写入量能降两三个数量级。
|
||||
*/
|
||||
public Map<String, String> studentHashByXh() {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
List<GxmuStudent> list = studentMapper.selectHashForDiff(OpenplatSyncConstants.TENANT_ID);
|
||||
for (GxmuStudent item : list) {
|
||||
String xh = StrUtil.trimToNull(item.getXh());
|
||||
if (xh != null) {
|
||||
map.put(xh, item.getRowHash());
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* 学生差集停用。
|
||||
*
|
||||
* <p>在 Java 侧求差集而不是拼 NOT IN,避免 9 万多个参数。
|
||||
* <p>用同步开头已窄读到的学号集合求差集,避免再查一次库、也不必拼 NOT IN。
|
||||
* 只处理同步来源的记录(xh 非空),见 docs/adr/0004。
|
||||
*
|
||||
* @param existingXh 库内现有的同步来源学号
|
||||
* @param incomingXh 本次从平台拉到的学号
|
||||
* @return 停用条数
|
||||
*/
|
||||
public int deactivateMissingStudents(Set<String> incomingXh) {
|
||||
List<String> existing = studentMapper.selectAllXh(OpenplatSyncConstants.TENANT_ID);
|
||||
public int deactivateMissingStudents(Set<String> existingXh, Set<String> incomingXh) {
|
||||
List<String> missing = new ArrayList<>();
|
||||
for (String xh : existing) {
|
||||
for (String xh : existingXh) {
|
||||
if (SyncScope.shouldDeactivate(xh, incomingXh.contains(xh))) {
|
||||
missing.add(xh);
|
||||
}
|
||||
|
||||
@@ -124,6 +124,9 @@ public class GxmuStudent implements Serializable {
|
||||
@TableLogic
|
||||
private Integer deleted;
|
||||
|
||||
@ApiModelProperty(value = "平台权威字段指纹,用于跳过未变更行")
|
||||
private String rowHash;
|
||||
|
||||
@TableField(exist = false)
|
||||
@ApiModelProperty(value = "学院名称")
|
||||
private String collegeName;
|
||||
|
||||
@@ -20,24 +20,26 @@ public interface GxmuStudentMapper extends BaseMapper<GxmuStudent> {
|
||||
* <p>只覆盖平台权威字段;user_id / claim_time 由认领流程维护,不在此处覆盖。
|
||||
* create_time 不在 UPDATE 子句中,重复执行不会刷新创建时间。
|
||||
*
|
||||
* <p>调用方应先用 {@link #selectHashForDiff} 比对指纹,只传入变化行——
|
||||
* 9.8 万行全量写实测要 88 秒,而稳态下变化行通常只有几十到几百条。
|
||||
*
|
||||
* @param list 学生名册
|
||||
* @return 影响行数
|
||||
*/
|
||||
int batchUpsert(@Param("list") List<GxmuStudent> list);
|
||||
|
||||
/**
|
||||
* 查询全部同步来源的学号。
|
||||
* 窄读全部同步来源的 (学号, 指纹),用于比对出变化行。
|
||||
*
|
||||
* <p>用于在 Java 侧与本次拉取到的学号求差集——上游消失的记录通常为零,
|
||||
* 这样就不必用 9 万多个参数去拼 NOT IN。
|
||||
* <p>只取两列,9.8 万行实测约 2.4 秒;换成整行读取会慢一个量级。
|
||||
*
|
||||
* @param tenantId 租户ID
|
||||
* @return 学号列表
|
||||
* @return 只填了 xh 与 rowHash 的列表
|
||||
*/
|
||||
List<String> selectAllXh(@Param("tenantId") Integer tenantId);
|
||||
List<GxmuStudent> selectHashForDiff(@Param("tenantId") Integer tenantId);
|
||||
|
||||
/**
|
||||
* 按学号批量置为停用。
|
||||
* 按学号批量置为停用,并把 row_hash 置空。
|
||||
*
|
||||
* <p>只作用于同步来源的记录(xh 非空),人工创建的记录不在其中,见 docs/adr/0004。
|
||||
*
|
||||
|
||||
@@ -2,12 +2,17 @@
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.gxwebsoft.gxmu.openplat.mapper.GxmuStudentMapper">
|
||||
|
||||
<!--
|
||||
平台权威字段白名单见 docs/adr/0005。
|
||||
user_id / claim_time 归认领流程,create_time 不重复刷新,均不在 UPDATE 子句中。
|
||||
调用方只传「指纹有变化」的行,稳态下这个列表通常很小。
|
||||
-->
|
||||
<insert id="batchUpsert">
|
||||
INSERT INTO gxmu_student (
|
||||
xh, xm, xbm, sex, mzm, nation, zzmmm, politics, sfzjh, sjh,
|
||||
bjm, bh, class_id, college_id, dwh, zyh, zymc, pyccmc,
|
||||
sfzx, xjzt, xsdqzt, csrq, status, sync_time, tenant_id,
|
||||
create_time, update_time, deleted
|
||||
create_time, update_time, deleted, row_hash
|
||||
) VALUES
|
||||
<foreach collection="list" item="item" separator=",">
|
||||
(
|
||||
@@ -17,7 +22,7 @@
|
||||
#{item.zyh}, #{item.zymc}, #{item.pyccmc},
|
||||
#{item.sfzx}, #{item.xjzt}, #{item.xsdqzt}, #{item.csrq},
|
||||
#{item.status}, #{item.syncTime}, #{item.tenantId},
|
||||
#{item.createTime}, #{item.updateTime}, #{item.deleted}
|
||||
#{item.createTime}, #{item.updateTime}, #{item.deleted}, #{item.rowHash}
|
||||
)
|
||||
</foreach>
|
||||
ON DUPLICATE KEY UPDATE
|
||||
@@ -45,17 +50,31 @@
|
||||
status = VALUES(status),
|
||||
sync_time = VALUES(sync_time),
|
||||
update_time = VALUES(update_time),
|
||||
deleted = VALUES(deleted)
|
||||
deleted = VALUES(deleted),
|
||||
row_hash = VALUES(row_hash)
|
||||
</insert>
|
||||
|
||||
<select id="selectAllXh" resultType="java.lang.String">
|
||||
SELECT xh FROM gxmu_student
|
||||
WHERE tenant_id = #{tenantId} AND xh IS NOT NULL AND xh <> ''
|
||||
<!--
|
||||
变更比对用的窄读:只取学号与指纹,9.8 万行实测约 2.4 秒。
|
||||
显式带 deleted = 0:自定义 SQL 不受 @TableLogic 影响,
|
||||
被逻辑删除的行不应算作「已存在」,否则它不会被认为是变化行而无法复活。
|
||||
-->
|
||||
<select id="selectHashForDiff" resultType="com.gxwebsoft.gxmu.openplat.entity.GxmuStudent">
|
||||
SELECT xh, row_hash
|
||||
FROM gxmu_student
|
||||
WHERE tenant_id = #{tenantId}
|
||||
AND deleted = 0
|
||||
AND xh IS NOT NULL
|
||||
AND xh <> ''
|
||||
</select>
|
||||
|
||||
<!--
|
||||
停用未再出现的记录。同时把 row_hash 置空:
|
||||
该行若以后重新出现,指纹必然对不上,会被重新写入并恢复状态。
|
||||
-->
|
||||
<update id="deactivateByXh">
|
||||
UPDATE gxmu_student
|
||||
SET status = 0, update_time = NOW()
|
||||
SET status = 0, row_hash = NULL, update_time = NOW()
|
||||
WHERE tenant_id = #{tenantId} AND status <> 0
|
||||
AND xh IN
|
||||
<foreach collection="xhs" item="xh" open="(" separator="," close=")">
|
||||
@@ -63,10 +82,4 @@
|
||||
</foreach>
|
||||
</update>
|
||||
|
||||
<select id="selectClassIdByCode" resultType="java.lang.Integer">
|
||||
SELECT id FROM gxmu_class
|
||||
WHERE tenant_id = #{tenantId} AND class_code = #{classCode} AND deleted = 0
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.gxwebsoft.gxmu.openplat.OpenPlatformClient;
|
||||
import com.gxwebsoft.gxmu.openplat.OpenPlatformException;
|
||||
import com.gxwebsoft.gxmu.openplat.OpenplatCodeService;
|
||||
import com.gxwebsoft.gxmu.openplat.OpenplatSyncConstants;
|
||||
import com.gxwebsoft.gxmu.openplat.StudentFingerprint;
|
||||
import com.gxwebsoft.gxmu.openplat.SyncResult;
|
||||
import com.gxwebsoft.gxmu.openplat.SyncSupport;
|
||||
import com.gxwebsoft.gxmu.openplat.entity.GxmuStudent;
|
||||
@@ -17,6 +18,8 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
@@ -77,9 +80,12 @@ public class StudentSyncHandler {
|
||||
|
||||
applyPhones(merged, inSchoolPhone, graduatePhone);
|
||||
|
||||
// 库内现有指纹:9.8 万行窄读约 2.4 秒,而全量重写要 88 秒
|
||||
Map<String, String> existingHash = support.studentHashByXh();
|
||||
|
||||
LocalDateTime now = SyncSupport.now();
|
||||
Set<String> incoming = new HashSet<>(merged.size());
|
||||
java.util.List<GxmuStudent> list = new java.util.ArrayList<>(merged.size());
|
||||
List<GxmuStudent> changed = new ArrayList<>();
|
||||
for (Map.Entry<String, GxmuStudent> entry : merged.entrySet()) {
|
||||
GxmuStudent student = entry.getValue();
|
||||
student.setXh(entry.getKey());
|
||||
@@ -92,15 +98,22 @@ public class StudentSyncHandler {
|
||||
student.setStatus(0);
|
||||
}
|
||||
incoming.add(entry.getKey());
|
||||
list.add(student);
|
||||
String hash = StudentFingerprint.of(student);
|
||||
student.setRowHash(hash);
|
||||
if (!hash.equals(existingHash.get(entry.getKey()))) {
|
||||
changed.add(student);
|
||||
}
|
||||
}
|
||||
support.upsertStudents(list);
|
||||
result.setSuccessCount(list.size());
|
||||
support.upsertStudents(changed);
|
||||
// successCount 维持「平台并集人数」的语义,便于与名册总数核对;
|
||||
// 真正写入的行数记在日志里(生产已放开 openplat 包的日志级别)
|
||||
result.setSuccessCount(merged.size());
|
||||
|
||||
int deactivated = support.deactivateMissingStudents(incoming);
|
||||
int deactivated = support.deactivateMissingStudents(existingHash.keySet(), incoming);
|
||||
result.setSkipCount(deactivated);
|
||||
logger.warn("学生同步完成: 写入 {} 条, 停用 {} 条, 失败 {} 条",
|
||||
list.size(), deactivated, result.getFailCount());
|
||||
logger.warn("学生同步完成: 平台 {} 人, 实际写入 {} 行, 未变更跳过 {} 行, 停用 {} 行, 失败 {} 条",
|
||||
merged.size(), changed.size(), merged.size() - changed.size(),
|
||||
deactivated, result.getFailCount());
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user