ZF/shoulder-platform
1
0
Fork 0
mirror of https://gitee.com/ChinaLym/shoulder-platform.git synced 2025-12-30 11:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(version): shoulder->0.5，ddl update

Browse Source
This commit is contained in:
15858193327
2020-12-07 01:00:37 +08:00
parent 5fe56308eb
commit ce046b0e73
12 changed files with 210 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -32,7 +32,7 @@
- 用户管理
- 资源权限管理
- 通知推送中心（短信、邮件）
- 错误码中心（查询错误码，大概产生原因，解决措施）
- 错误码中心（查询错误码，大概产生原因，解决措施、热门错误码统计）
- 知识库(记录常见问题排查方式等）
- 在线 api 文档中心

24
doc/shoulder-platform.ddl
View File

@@ -26,11 +26,12 @@ CREATE TABLE `crypt_info` (
/*Data for the table `crypt_info` */
/*Table structure for table `import_record` */
/*Table structure for table `batch_record` */
CREATE TABLE `import_record` (
`id` INT NOT NULL AUTO_INCREMENT COMMENT '主键',
`data_type` varchar(64) NOT NULL COMMENT '导入数据类型，建议可翻译。对应 导入数据库表名',
CREATE TABLE `batch_record` (
`id` varchar(48) NOT NULL COMMENT '主键',
`data_type` varchar(64) NOT NULL COMMENT '导入数据类型，建议可翻译。对应 导入数据库表名 / 领域对象名称，如用户、人员、订单',
`operation` varchar(64) COMMENT '业务操作类型，如校验、同步、导入、更新，可空',
`total_num` INT NOT NULL COMMENT '导入总条数',
`success_num` INT NOT NULL COMMENT '成功条数',
`fail_num` INT NOT NULL COMMENT '失败条数',
@@ -39,21 +40,22 @@ CREATE TABLE `import_record` (
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='导入记录';
/*Data for the table `import_record` */
/*Data for the table `batch_record` */
/*Table structure for table `import_record_detail` */
/*Table structure for table `batch_record_detail` */
CREATE TABLE `import_record_detail` (
CREATE TABLE `batch_record_detail` (
`id` INT NOT NULL AUTO_INCREMENT COMMENT '主键',
`record_id` INT NOT NULL COMMENT '导入记录表id',
`result` INT NOT NULL COMMENT '结果 0 导入成功 1 校验失败、2 重复跳过、3 重复更新、4 导入失败',
`line` INT NOT NULL COMMENT '导入行号',
`record_id` varchar(48) NOT NULL COMMENT '导入记录表id',
`row_num` INT NOT NULL COMMENT '导入行号',
`operation` varchar(64) NOT NULL COMMENT '业务操作类型，如校验、同步、导入、更新',
`status` INT NOT NULL COMMENT '结果 0 导入成功 1 校验失败、2 重复跳过、3 重复更新、4 导入失败',
`fail_reason` varchar(1024) DEFAULT NULL COMMENT '失败原因，推荐支持多语言',
`source` text COMMENT '导入的原数据',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='导入记录详情';
/*Data for the table `import_record_detail` */
/*Data for the table `batch_record_detail` */
/*Table structure for table `log_operation` */

12
shoulder-backstage/pom.xml
View File

@@ -10,6 +10,18 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>shoulder-backstage</artifactId>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

105
shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchCount.java Normal file
View File

@@ -0,0 +1,105 @@
package cn.itlym.platform.search;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.VarHandle;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author lym
*/
public class HotSearchCount {
/**
* 本地缓存时长，一般建议使用秒为单位，默认记录最近 2 分钟的
*/
private static final Node[] buffer = new Node[128];
/**
* 每秒不同搜索一般在多少次
*/
private static final int DISTINCT_EVERY_SECOND = 500;
/**
* buffer handle 用于 CAS
*/
private final VarHandle nodeHandle = MethodHandles.arrayElementVarHandle(Node[].class);
public void addSearchWord(String searchWord) {
// fixme 改为秒钟数
long currentTime = System.currentTimeMillis();
int index = ((int) (currentTime)) & (buffer.length - 1);
Node currentNode = getNodeAt(index);
if(currentNode.timeStamp != currentTime) {
// 需要 new
Node newNode = new Node(currentTime, DISTINCT_EVERY_SECOND);
if(!casNodeAt(index, currentNode, newNode)){
currentNode = getNodeAt(index);
}
}
AtomicInteger count = currentNode.computeIfAbsent(searchWord, k -> new AtomicInteger(0));
count.incrementAndGet();
}
public List<Node> takeBefore(long timeStamp) {
// 若频率固定，可以使用 ArrayList，并设置大小
List<Node> result = new LinkedList<>();
int index = ((int) (timeStamp)) & (buffer.length - 1);
for (int i = 0; i < buffer.length; i++) {
Node node = getNodeAt(i);
if(node.timeStamp < timeStamp) {
result.add(node);
}
}
return result;
}
private Node getNodeAt(int index) {
// 获取 buffer 数组的第 index 个元素
return (Node) nodeHandle.get(buffer, index);
}
private boolean casNodeAt(int index, Node old, Node newValue) {
// 如果 buffer 第 index 个元素是 old，则该元素被设为 newValue
return nodeHandle.compareAndSet(buffer, index, old, newValue);
}
/**
* 这里使用 Map 结构
* 为了避免突发大量不同词频带来的扩容以及垃圾回收，可以替换为大顶堆
* 但大顶堆会因数据分散带来意外换出，导致某些词本应统计为热词而未统计（与分散程度相关，宏观上不会影响过多）
*/
public static class Node extends ConcurrentHashMap<String, AtomicInteger> {
/**
* 什么时候产生的热词（秒钟）
*/
final long timeStamp;
/**
* 创建
* @param timeStamp 时间戳
* @param size 一般来源预估每秒搜索不同的次不超过多少次
*/
public Node(long timeStamp, int size) {
super(size);
this.timeStamp = timeStamp;
}
/**
* 默认每秒约 500 次不同搜索
* @param timeStamp 时间戳
*/
public Node(long timeStamp) {
this(timeStamp, 500);
}
}
}

69
shoulder-backstage/src/main/java/cn/itlym/platform/search/HotSearchPorter.java Normal file
View File

@@ -0,0 +1,69 @@
package cn.itlym.platform.search;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
/**
* 从 HotSearchCount 中取走
* @author lym
*/
public class HotSearchPorter {
private ConcurrentLinkedQueue<HotSearchCount.Node> queue = new ConcurrentLinkedQueue<>();
private final AtomicLong lastProcessTimStamp = new AtomicLong(0);
private HotSearchCount hotSearchCount;
/**
* 定期调度该方法，从本地缓存中取
*/
public void process() {
// fixme 改为秒钟数
long currentTime = System.currentTimeMillis();
long processEnd = currentTime - 5;
long processStart = lastProcessTimStamp.getAndSet(processEnd);
// 处理 [processStart, processEnd) 的数据
List<HotSearchCount.Node> allBeforeEnd = hotSearchCount.takeBefore(processEnd);
List<HotSearchCount.Node> needProcess = allBeforeEnd.stream()
.filter(node -> node.timeStamp >= processStart)
.collect(Collectors.toList());
// 加到队列里
// 开一个新线程处理
doProcess(needProcess);
}
/**
* 可以本地合并xx秒钟的，然后再刷一次记录，减少存储空间，减少读写重复频次
* 如 QQ 音乐每 10分钟更新一次、微博，每分钟更新一次，网易云音乐每周4更新
* @param needProcess
*/
protected void doProcess(List<HotSearchCount.Node> needProcess) {
// 合并这10分钟的词频，合并结果可采用大顶堆，减少内存占用
HotSearchCount.Node latest_10_minute = new HotSearchCount.Node(needProcess.get(0).timeStamp, 10240);
needProcess.forEach(latest_10_minute::putAll);
// 保存到时序数据库
persistent(latest_10_minute);
// 查出 24 小时前 10 分钟的数据
// 与当前热搜合并（可选）
// 放入 redis 的 zset
}
private void persistent(HotSearchCount.Node node) {
}
public static void main(String[] args) {
System.out.println(Integer.MAX_VALUE - (-5));
}
}

2
shoulder-gateway/pom.xml
View File

@@ -21,7 +21,7 @@
<properties>
<shoulder.version>0.5-SNAPSHOT</shoulder.version><!-- shoulder-version -->
<shoulder.version>0.5</shoulder.version><!-- shoulder-version -->
<spring-boot.version>2.2.8.RELEASE</spring-boot.version>
<spring-cloud.version>Hoxton.SR6</spring-cloud.version>
<spring-cloud-alibaba.version>2.2.1.RELEASE</spring-cloud-alibaba.version>

2
shoulder-gateway/shoulder-api-gateway/pom.xml
View File

@@ -15,7 +15,7 @@
<name>${project.artifactId}</name>
<properties>
<shoulder.version>0.5-SNAPSHOT</shoulder.version><!-- shoulder-version -->
<shoulder.version>0.5</shoulder.version><!-- shoulder-version -->
<spring-boot.version>2.2.8.RELEASE</spring-boot.version>
<spring-cloud.version>Hoxton.SR6</spring-cloud.version>
<spring-cloud-alibaba.version>2.2.1.RELEASE</spring-cloud-alibaba.version>

2
shoulder-generator/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>cn.itlym</groupId>
<artifactId>shoulder-parent</artifactId>
<version>0.5-SNAPSHOT</version><!-- shoulder-version -->
<version>0.5</version><!-- shoulder-version -->
</parent>
<modelVersion>4.0.0</modelVersion>

5
shoulder-platform-common/README.md
View File

@@ -16,9 +16,10 @@ Shoulder 平台各个工程的`基础设施层`统一实现（为了简化使用
----
## 统一更换 shoulder-framework 版本
-SNAPSHOT
```xml
<shoulder.version>0.5-SNAPSHOT</shoulder.version><!-- shoulder-version -->
<version>0.5-SNAPSHOT</version><!-- shoulder-version -->
<shoulder.version>0.5</shoulder.version><!-- shoulder-version -->
<version>0.5</version><!-- shoulder-version -->
```
## 统一更换 shoulder-platform-common 版本

2
shoulder-platform-common/pom.xml
View File

@@ -30,7 +30,7 @@
</modules>
<properties>
<shoulder.version>0.5-SNAPSHOT</shoulder.version><!-- shoulder-version -->
<shoulder.version>0.5</shoulder.version><!-- shoulder-version -->
<shoulder-platform.version>0.1-SNAPSHOT</shoulder-platform.version>
</properties>

4
shoulder-platform-common/shoulder-platform-parent/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>cn.itlym</groupId>
<artifactId>shoulder-parent</artifactId>
<version>0.5-SNAPSHOT</version><!-- shoulder-version -->
<version>0.5</version><!-- shoulder-version -->
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -18,7 +18,7 @@
<properties>
<!-- =============== Shoulder 版本 =============== -->
<shoulder.version>0.5-SNAPSHOT</shoulder.version><!-- shoulder-version -->
<shoulder.version>0.5</shoulder.version><!-- shoulder-version -->
<shoulder-platform.version>1.0-SNAPSHOT</shoulder-platform.version><!-- shoulder-platform-version -->
<shoulder-sms-aliyun.version>1.0-SNAPSHOT</shoulder-sms-aliyun.version><!-- shoulder-sms-aliyun.version -->

2
shoulder-platform-common/shoulder-sms-aliyun-spring-boot-starter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>cn.itlym</groupId>
<artifactId>shoulder-parent</artifactId>
<version>0.5-SNAPSHOT</version><!-- shoulder-version -->
<version>0.5</version><!-- shoulder-version -->
</parent>
<modelVersion>4.0.0</modelVersion>