diff --git a/README.md b/README.md index d3cb283..377958e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ - 用户管理 - 资源权限管理 - 通知推送中心(短信、邮件) - - 错误码中心(查询错误码,大概产生原因,解决措施) + - 错误码中心(查询错误码,大概产生原因,解决措施、热门错误码统计) - 知识库(记录常见问题排查方式等) - 在线 api 文档中心 diff --git a/doc/shoulder-platform.ddl b/doc/shoulder-platform.ddl index a3d8346..b6ce9ec 100644 --- a/doc/shoulder-platform.ddl +++ b/doc/shoulder-platform.ddl @@ -26,11 +26,12 @@ CREATE TABLE `crypt_info` ( /*Data for the table `crypt_info` */ -/*Table structure for table `import_record` */ +/*Table structure for table `batch_record` */ -CREATE TABLE `import_record` ( - `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', - `data_type` varchar(64) NOT NULL COMMENT '导入数据类型,建议可翻译。对应 导入数据库表名', +CREATE TABLE `batch_record` ( + `id` varchar(48) NOT NULL COMMENT '主键', + `data_type` varchar(64) NOT NULL COMMENT '导入数据类型,建议可翻译。对应 导入数据库表名 / 领域对象名称,如用户、人员、订单', + `operation` varchar(64) COMMENT '业务操作类型,如校验、同步、导入、更新,可空', `total_num` INT NOT NULL COMMENT '导入总条数', `success_num` INT NOT NULL COMMENT '成功条数', `fail_num` INT NOT NULL COMMENT '失败条数', @@ -39,21 +40,22 @@ CREATE TABLE `import_record` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='导入记录'; -/*Data for the table `import_record` */ +/*Data for the table `batch_record` */ -/*Table structure for table `import_record_detail` */ +/*Table structure for table `batch_record_detail` */ -CREATE TABLE `import_record_detail` ( +CREATE TABLE `batch_record_detail` ( `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', - `record_id` INT NOT NULL COMMENT '导入记录表id', - `result` INT NOT NULL COMMENT '结果 0 导入成功 1 校验失败、2 重复跳过、3 重复更新、4 导入失败', - `line` INT NOT NULL COMMENT '导入行号', + `record_id` varchar(48) NOT NULL COMMENT '导入记录表id', + `row_num` INT NOT NULL COMMENT '导入行号', + `operation` varchar(64) NOT NULL COMMENT '业务操作类型,如校验、同步、导入、更新', + `status` INT NOT NULL COMMENT '结果 0 导入成功 1 校验失败、2 重复跳过、3 重复更新、4 导入失败', `fail_reason` varchar(1024) DEFAULT NULL COMMENT '失败原因,推荐支持多语言', `source` text COMMENT '导入的原数据', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='导入记录详情'; -/*Data for the table `import_record_detail` */ +/*Data for the table `batch_record_detail` */ /*Table structure for table `log_operation` */ diff --git a/shoulder-backstage/pom.xml b/shoulder-backstage/pom.xml index e7dbb68..2134b56 100644 --- a/shoulder-backstage/pom.xml +++ b/shoulder-backstage/pom.xml @@ -10,6 +10,18 @@ 4.0.0 shoulder-backstage + + + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + 11 + + + + \ No newline at end of file diff --git a/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchCount.java b/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchCount.java new file mode 100644 index 0000000..82e5069 --- /dev/null +++ b/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchCount.java @@ -0,0 +1,105 @@ +package cn.itlym.platform.search; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author lym + */ +public class HotSearchCount { + + /** + * 本地缓存时长,一般建议使用秒为单位,默认记录最近 2 分钟的 + */ + private static final Node[] buffer = new Node[128]; + + /** + * 每秒不同搜索一般在多少次 + */ + private static final int DISTINCT_EVERY_SECOND = 500; + + /** + * buffer handle 用于 CAS + */ + private final VarHandle nodeHandle = MethodHandles.arrayElementVarHandle(Node[].class); + + public void addSearchWord(String searchWord) { + // fixme 改为秒钟数 + long currentTime = System.currentTimeMillis(); + int index = ((int) (currentTime)) & (buffer.length - 1); + Node currentNode = getNodeAt(index); + if(currentNode.timeStamp != currentTime) { + // 需要 new + Node newNode = new Node(currentTime, DISTINCT_EVERY_SECOND); + if(!casNodeAt(index, currentNode, newNode)){ + currentNode = getNodeAt(index); + } + } + AtomicInteger count = currentNode.computeIfAbsent(searchWord, k -> new AtomicInteger(0)); + count.incrementAndGet(); + } + + + public List takeBefore(long timeStamp) { + // 若频率固定,可以使用 ArrayList,并设置大小 + List result = new LinkedList<>(); + int index = ((int) (timeStamp)) & (buffer.length - 1); + for (int i = 0; i < buffer.length; i++) { + Node node = getNodeAt(i); + if(node.timeStamp < timeStamp) { + result.add(node); + } + } + return result; + + } + + private Node getNodeAt(int index) { + // 获取 buffer 数组的第 index 个元素 + return (Node) nodeHandle.get(buffer, index); + } + + private boolean casNodeAt(int index, Node old, Node newValue) { + // 如果 buffer 第 index 个元素是 old,则该元素被设为 newValue + return nodeHandle.compareAndSet(buffer, index, old, newValue); + } + + + /** + * 这里使用 Map 结构 + * 为了避免突发大量不同词频带来的扩容以及垃圾回收,可以替换为大顶堆 + * 但大顶堆会因数据分散带来意外换出,导致某些词本应统计为热词而未统计(与分散程度相关,宏观上不会影响过多) + */ + public static class Node extends ConcurrentHashMap { + + /** + * 什么时候产生的热词(秒钟) + */ + final long timeStamp; + + /** + * 创建 + * @param timeStamp 时间戳 + * @param size 一般来源预估每秒搜索不同的次不超过多少次 + */ + public Node(long timeStamp, int size) { + super(size); + this.timeStamp = timeStamp; + } + + /** + * 默认每秒约 500 次不同搜索 + * @param timeStamp 时间戳 + */ + public Node(long timeStamp) { + this(timeStamp, 500); + } + + } + + +} diff --git a/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchPorter.java b/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchPorter.java new file mode 100644 index 0000000..d688f14 --- /dev/null +++ b/shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchPorter.java @@ -0,0 +1,69 @@ +package cn.itlym.platform.search; + +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +/** + * 从 HotSearchCount 中取走 + * @author lym + */ +public class HotSearchPorter { + + private ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + + private final AtomicLong lastProcessTimStamp = new AtomicLong(0); + + private HotSearchCount hotSearchCount; + + + /** + * 定期调度该方法,从本地缓存中取 + */ + public void process() { + // fixme 改为秒钟数 + long currentTime = System.currentTimeMillis(); + long processEnd = currentTime - 5; + long processStart = lastProcessTimStamp.getAndSet(processEnd); + // 处理 [processStart, processEnd) 的数据 + List allBeforeEnd = hotSearchCount.takeBefore(processEnd); + List needProcess = allBeforeEnd.stream() + .filter(node -> node.timeStamp >= processStart) + .collect(Collectors.toList()); + // 加到队列里 + // 开一个新线程处理 + doProcess(needProcess); + } + + /** + * 可以本地合并xx秒钟的,然后再刷一次记录,减少存储空间,减少读写重复频次 + * 如 QQ 音乐每 10分钟更新一次、微博,每分钟更新一次,网易云音乐每周4更新 + * @param needProcess + */ + protected void doProcess(List needProcess) { + // 合并这10分钟的词频,合并结果可采用大顶堆,减少内存占用 + HotSearchCount.Node latest_10_minute = new HotSearchCount.Node(needProcess.get(0).timeStamp, 10240); + needProcess.forEach(latest_10_minute::putAll); + + // 保存到时序数据库 + persistent(latest_10_minute); + + // 查出 24 小时前 10 分钟的数据 + + // 与当前热搜合并(可选) + + // 放入 redis 的 zset + + + } + + private void persistent(HotSearchCount.Node node) { + + } + + public static void main(String[] args) { + System.out.println(Integer.MAX_VALUE - (-5)); + } + +} diff --git a/shoulder-gateway/pom.xml b/shoulder-gateway/pom.xml index c4494db..1464a82 100644 --- a/shoulder-gateway/pom.xml +++ b/shoulder-gateway/pom.xml @@ -21,7 +21,7 @@ - 0.5-SNAPSHOT + 0.5 2.2.8.RELEASE Hoxton.SR6 2.2.1.RELEASE diff --git a/shoulder-gateway/shoulder-api-gateway/pom.xml b/shoulder-gateway/shoulder-api-gateway/pom.xml index 5cd25cb..9a02c6e 100644 --- a/shoulder-gateway/shoulder-api-gateway/pom.xml +++ b/shoulder-gateway/shoulder-api-gateway/pom.xml @@ -15,7 +15,7 @@ ${project.artifactId} - 0.5-SNAPSHOT + 0.5 2.2.8.RELEASE Hoxton.SR6 2.2.1.RELEASE diff --git a/shoulder-generator/pom.xml b/shoulder-generator/pom.xml index b34d356..2f8b5e2 100644 --- a/shoulder-generator/pom.xml +++ b/shoulder-generator/pom.xml @@ -5,7 +5,7 @@ cn.itlym shoulder-parent - 0.5-SNAPSHOT + 0.5 4.0.0 diff --git a/shoulder-platform-common/README.md b/shoulder-platform-common/README.md index da1cca1..6a1a13e 100644 --- a/shoulder-platform-common/README.md +++ b/shoulder-platform-common/README.md @@ -16,9 +16,10 @@ Shoulder 平台各个工程的`基础设施层`统一实现(为了简化使用 ---- ## 统一更换 shoulder-framework 版本 +-SNAPSHOT ```xml -0.5-SNAPSHOT -0.5-SNAPSHOT +0.5 +0.5 ``` ## 统一更换 shoulder-platform-common 版本 diff --git a/shoulder-platform-common/pom.xml b/shoulder-platform-common/pom.xml index 9e7cfff..2222318 100644 --- a/shoulder-platform-common/pom.xml +++ b/shoulder-platform-common/pom.xml @@ -30,7 +30,7 @@ - 0.5-SNAPSHOT + 0.5 0.1-SNAPSHOT diff --git a/shoulder-platform-common/shoulder-platform-parent/pom.xml b/shoulder-platform-common/shoulder-platform-parent/pom.xml index 5e8042f..d4f15dc 100644 --- a/shoulder-platform-common/shoulder-platform-parent/pom.xml +++ b/shoulder-platform-common/shoulder-platform-parent/pom.xml @@ -5,7 +5,7 @@ cn.itlym shoulder-parent - 0.5-SNAPSHOT + 0.5 4.0.0 @@ -18,7 +18,7 @@ - 0.5-SNAPSHOT + 0.5 1.0-SNAPSHOT 1.0-SNAPSHOT diff --git a/shoulder-platform-common/shoulder-sms-aliyun-spring-boot-starter/pom.xml b/shoulder-platform-common/shoulder-sms-aliyun-spring-boot-starter/pom.xml index e55b5cd..1381442 100644 --- a/shoulder-platform-common/shoulder-sms-aliyun-spring-boot-starter/pom.xml +++ b/shoulder-platform-common/shoulder-sms-aliyun-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ cn.itlym shoulder-parent - 0.5-SNAPSHOT + 0.5 4.0.0