gxwebsoft/mp-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(import): 优化批量导入功能提升性能

Browse Source 
- 在多个控制器中引入 BatchImportSupport 工具类
- 实现分块处理机制，每批次处理 500 条记录
- 使用 persistChunkWithFallback 方法替代逐条保存
- 保持原有的数据校验和去重逻辑不变
- 显著提升大量数据导入时的执行效率
- 减少数据库操作次数降低系统负载
This commit is contained in:
赵忠林
2026-01-16 00:55:49 +08:00
parent e0e15cdd45
commit e6736d41ff
30 changed files with 2835 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/main/java/com/gxwebsoft/credit/controller/CreditCompetitorController.java
View File

@@ -37,6 +37,9 @@ public class CreditCompetitorController extends BaseController {
@Resource
private CreditCompetitorService creditCompetitorService;
@Resource
private BatchImportSupport batchImportSupport;
@Operation(summary = "分页查询竞争对手")
@GetMapping("/page")
public ApiResult<PageResult<CreditCompetitor>> page(CreditCompetitorParam param) {
@@ -156,6 +159,11 @@ public class CreditCompetitorController extends BaseController {
Integer currentUserId = loginUser != null ? loginUser.getUserId() : null;
Integer currentTenantId = loginUser != null ? loginUser.getTenantId() : null;
final int chunkSize = 500;
final int mpBatchSize = 500;
List<CreditCompetitor> chunkItems = new ArrayList<>(chunkSize);
List<Integer> chunkRowNumbers = new ArrayList<>(chunkSize);
for (int i = 0; i < list.size(); i++) {
CreditCompetitorImportParam param = list.get(i);
try {
@@ -186,23 +194,46 @@ public class CreditCompetitorController extends BaseController {
continue;
}
boolean saved = creditCompetitorService.save(item);
if (!saved) {
CreditCompetitor existing = creditCompetitorService.lambdaQuery()
.eq(CreditCompetitor::getCompanyName, item.getCompanyName())
.one();
if (existing != null) {
item.setId(existing.getId());
if (creditCompetitorService.updateById(item)) {
successCount++;
continue;
}
}
} else {
successCount++;
continue;
chunkItems.add(item);
chunkRowNumbers.add(excelRowNumber);
if (chunkItems.size() >= chunkSize) {
successCount += batchImportSupport.persistChunkWithFallback(
chunkItems,
chunkRowNumbers,
() -> batchImportSupport.upsertBySingleKey(
creditCompetitorService,
chunkItems,
CreditCompetitor::getId,
CreditCompetitor::setId,
CreditCompetitor::getCompanyName,
CreditCompetitor::getCompanyName,
null,
mpBatchSize
),
(rowItem, rowNumber) -> {
boolean saved = creditCompetitorService.save(rowItem);
if (!saved) {
CreditCompetitor existing = creditCompetitorService.lambdaQuery()
.eq(CreditCompetitor::getCompanyName, rowItem.getCompanyName())
.one();
if (existing != null) {
rowItem.setId(existing.getId());
if (creditCompetitorService.updateById(rowItem)) {
return true;
}
}
} else {
return true;
}
String prefix = rowNumber > 0 ? ("第" + rowNumber + "行：") : "";
errorMessages.add(prefix + "保存失败");
return false;
},
errorMessages
);
chunkItems.clear();
chunkRowNumbers.clear();
}
errorMessages.add("第" + excelRowNumber + "行：保存失败");
} catch (Exception e) {
int excelRowNumber = i + 1 + usedTitleRows + usedHeadRows;
errorMessages.add("第" + excelRowNumber + "行：" + e.getMessage());
@@ -210,6 +241,43 @@ public class CreditCompetitorController extends BaseController {
}
}
if (!chunkItems.isEmpty()) {
successCount += batchImportSupport.persistChunkWithFallback(
chunkItems,
chunkRowNumbers,
() -> batchImportSupport.upsertBySingleKey(
creditCompetitorService,
chunkItems,
CreditCompetitor::getId,
CreditCompetitor::setId,
CreditCompetitor::getCompanyName,
CreditCompetitor::getCompanyName,
null,
mpBatchSize
),
(rowItem, rowNumber) -> {
boolean saved = creditCompetitorService.save(rowItem);
if (!saved) {
CreditCompetitor existing = creditCompetitorService.lambdaQuery()
.eq(CreditCompetitor::getCompanyName, rowItem.getCompanyName())
.one();
if (existing != null) {
rowItem.setId(existing.getId());
if (creditCompetitorService.updateById(rowItem)) {
return true;
}
}
} else {
return true;
}
String prefix = rowNumber > 0 ? ("第" + rowNumber + "行：") : "";
errorMessages.add(prefix + "保存失败");
return false;
},
errorMessages
);
}
if (errorMessages.isEmpty()) {
return success("成功导入" + successCount + "条数据", null);
} else {