Merge branch 'rad' into dev

pom.xml

@@ -49,8 +49,14 @@
             <version>${lombok.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-redis</artifactId>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-redis</artifactId>
+            <version>2.4.3</version>
+        </dependency>
+        <dependency>
+            <groupId>redis.clients</groupId>
+            <artifactId>jedis</artifactId>
+            <version>3.3.0</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>

src/main/java/com/weface/component/MenuService.java

@@ -88,7 +88,6 @@ public class MenuService {
         String kk_sh_token = "kk_sh_token";
         try {
             String authToken = GeTuiUtils.getAuthToken(kk_sh_token);
-            System.out.println("===============" + authToken);
             //将token以及用户ID封装调用画像查询接口
             String result = GeTuiUtils.queryTagKKSH(gidList, authToken);
             JSONObject jsonObject = JSONObject.parseObject(result);
@@ -104,7 +103,6 @@ public class MenuService {
                     all.put("error_code", 0);
                 }
                 return all;
-                //如果以上步骤无法走通则直接返回。并修改标识符终止循环
             } else {
                 log.error("返回值内不包含需要解析的数据" + jsonObject.toJSONString());
             }
@@ -134,7 +132,6 @@ public class MenuService {
         if (jsonObject.containsKey("userTag")) {
             JSONArray userTag = jsonObject.getJSONArray("userTag");
             return getUserTagByTag(userTag, 2, tags);
-            //如果以上步骤无法走通则直接返回。并修改标识符终止循环
         } else {
             log.error("返回值内不包含需要解析的数据" + jsonObject.toJSONString());
         }

src/main/java/com/weface/config/RedisConfig.java

@@ -3,9 +3,16 @@ package com.weface.config;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisSentinelConfiguration;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
@@ -16,6 +23,36 @@ import org.springframework.data.redis.serializer.StringRedisSerializer;
 @Configuration
 public class RedisConfig {
 
+    private static Logger logger = LoggerFactory.getLogger(RedisConfig.class);
+
+    @Autowired
+    RedisProperties redisProperties;
+
+    @Bean
+    public JedisConnectionFactory factory() {
+
+        RedisStandaloneConfiguration connectionConf = new RedisStandaloneConfiguration();
+        connectionConf.setHostName(redisProperties.getHostname());
+        connectionConf.setDatabase(redisProperties.getDatabase());
+        connectionConf.setPort(redisProperties.getPort());
+
+
+        JedisConnectionFactory factory = new JedisConnectionFactory(connectionConf);
+        factory.getPoolConfig().setMaxTotal(redisProperties.getMaxTotal());
+        factory.getPoolConfig().setMaxIdle(redisProperties.getMaxIdle());
+        factory.getPoolConfig().setMinIdle(redisProperties.getMinIdle());
+
+        return factory;
+    }
+
+    public RedisConnectionFactory lettuceConnectionFactory() {
+        RedisSentinelConfiguration sentinelConfig = new RedisSentinelConfiguration()
+                .master("mymaster")
+                .sentinel("127.0.0.1", 26379)
+                .sentinel("127.0.0.1", 26380);
+        return new LettuceConnectionFactory(sentinelConfig);
+    }
+
     @Bean
     @SuppressWarnings("all")
     public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
@@ -39,4 +76,5 @@ public class RedisConfig {
         return template;
     }
 
+
 }

src/main/java/com/weface/config/RedisProperties.java

+package com.weface.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties(prefix = "gexiang.redis")
+@Component
+public class RedisProperties {
+    /**
+     * redis hostname
+     */
+    private String hostname;
+    /**
+     * redis database
+     */
+    private int database;
+    /**
+     * redis port
+     */
+    private int port;
+    /**
+     * redis max channel
+     */
+    private int maxTotal;
+    /**
+     * redis max wait channel
+     */
+    private int maxIdle;
+    /**
+     * redis min wait channel
+     */
+    private int minIdle;
+
+    public String getHostname() {
+        return hostname;
+    }
+
+    public void setHostname(String hostname) {
+        this.hostname = hostname;
+    }
+
+    public int getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(int database) {
+        this.database = database;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public void setPort(int port) {
+        this.port = port;
+    }
+
+    public int getMaxTotal() {
+        return maxTotal;
+    }
+
+    public void setMaxTotal(int maxTotal) {
+        this.maxTotal = maxTotal;
+    }
+
+    public int getMaxIdle() {
+        return maxIdle;
+    }
+
+    public void setMaxIdle(int maxIdle) {
+        this.maxIdle = maxIdle;
+    }
+
+    public int getMinIdle() {
+        return minIdle;
+    }
+
+    public void setMinIdle(int minIdle) {
+        this.minIdle = minIdle;
+    }
+
+}

src/main/java/com/weface/task/UserTagsTask.java

@@ -65,11 +65,18 @@ public class UserTagsTask {
             }
             int id = Integer.parseInt(userId);
 
-            //获取每次处理最大数
+            //获取缓存内每次处理最大数
             String pushSize = RedisUtil.StringOps.get("push_max_size");
+            //如果缓存没有则取配置文件中的
             if (StringUtils.isBlank(pushSize)) {
                 RedisUtil.StringOps.set("push_max_size", pushMaxSize);
                 pushSize = pushMaxSize;
+            } else {
+                //如果缓存和配置文件不一致则使用配置文件数值替换缓存
+                if (!pushMaxSize.equals(pushSize)) {
+                    RedisUtil.StringOps.set("push_max_size", pushMaxSize);
+                    pushSize = pushMaxSize;
+                }
             }
             int max = Integer.parseInt(pushSize);
 
@@ -77,51 +84,69 @@ public class UserTagsTask {
             List<UserMenusEntity> userMenusList = new ArrayList<>();
             //获取标签列表
             List<MenuTagsEntity> tags = menuTagsService.list();
-
-
-
             //获取小于起始值,且更新时间为当前时间用户信息
             List<UserTagEntity> beforeUser = userTagService.findUserByTodayAndIdBefore(id);
+            //如果每次更新数据为空则全部设置为新增
+            if (CollUtil.isEmpty(beforeUser)) {
+                List<UserMenusEntity> afterTag = getAfterTag(id, max, tags);
+                if (afterTag != null) {
+                    userMenusList.addAll(afterTag);
+                }
+            } else {
                 //过滤用户gid
                 List<String> beforeGid = beforeUser.stream().map(UserTagEntity::getGid).distinct().collect(Collectors.toList());
-
                 //获取更新总数量
                 int size = beforeGid.size();
                 //如果更新数量大于总量,截取每次处理总量的更新用户信息
                 if (size > max) {
                     beforeGid = beforeGid.subList(0, max - 1);
                 } else {
-                //最大处理数减去当天更新处理数,得到剩余处理数,获取起始值后的新增的用户信息
-                List<UserTagEntity> afterUser = userTagService.findUserByIdAfter(Integer.parseInt(userId), max - size);
-                //如果当天新增数不为空
-                if (CollUtil.isNotEmpty(afterUser)) {
-                    //获取用户最后一条信息
-                    UserTagEntity afterUserInfo = afterUser.get(afterUser.size() - 1);
-                    //并覆盖起始值
-                    RedisUtil.StringOps.set("user_tag_id", String.valueOf(afterUserInfo.getId()));
-                    //过滤用户gid
-                    List<String> afterGid = afterUser.stream().map(UserTagEntity::getGid).distinct().collect(Collectors.toList());
-                    //调用个像接口获取新增用户标签
-                    List<UserMenusEntity> afterTag = getUserTag(afterGid, afterUser, tags);
-                    if (CollUtil.isNotEmpty(afterTag)) {
+                    List<UserMenusEntity> afterTag = getAfterTag(id, max - size, tags);
+                    if (afterTag != null) {
                         userMenusList.addAll(afterTag);
                     }
                 }
-            }
                 //调用个像接口获取更新用户标签
                 List<UserMenusEntity> beforeTag = getUserTag(beforeGid, beforeUser, tags);
                 if (CollUtil.isNotEmpty(beforeTag)) {
                     userMenusList.addAll(beforeTag);
                 }
+            }
             //批量插入用户标签信息
             userMenusService.batchInsert(userMenusList);
-            log.error("执行结束{}", asyncServiceExecutor.getPoolSize());
+            log.info("执行结束,当前最新id{}", RedisUtil.StringOps.get("user_tag_id"));
         } catch (Exception e) {
             e.printStackTrace();
             log.error("执行定时更新标签任务失败!");
         }
     }
 
+    /**
+     * 处理新增用户标签
+     *
+     * @param userId 用户id
+     * @param limit  处理条数
+     * @param tags   标签信息
+     * @return 标签信息
+     * @throws InterruptedException 网络异常
+     */
+    private List<UserMenusEntity> getAfterTag(Integer userId, Integer limit, List<MenuTagsEntity> tags) throws InterruptedException {
+        //最大处理数减去当天更新处理数,得到剩余处理数,获取起始值后的新增的用户信息
+        List<UserTagEntity> afterUser = userTagService.findUserByIdAfter(userId, limit);
+        //如果当天新增数不为空
+        if (CollUtil.isNotEmpty(afterUser)) {
+            //获取用户最后一条信息
+            UserTagEntity afterUserInfo = afterUser.get(afterUser.size() - 1);
+            //并覆盖起始值
+            RedisUtil.StringOps.set("user_tag_id", String.valueOf(afterUserInfo.getId()));
+            //过滤用户gid
+            List<String> afterGid = afterUser.stream().map(UserTagEntity::getGid).distinct().collect(Collectors.toList());
+            //调用个像接口获取新增用户标签
+            return getUserTag(afterGid, afterUser, tags);
+        }
+        return null;
+    }
+
     /**
      * 获取个像用户标签信息
      *

src/main/resources/application-prod.yml

@@ -3,9 +3,9 @@ spring:
     type: com.alibaba.druid.pool.DruidDataSource
     druid:
       driver-class-name: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://localhost:3306/renren_fast?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
-      username: renren
-      password: 123456
+      url: jdbc:mysql://rm-uf6w14h3ark4z3pwxlo.mysql.rds.aliyuncs.com/kk_demain_db?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8
+      username: kk_demain_user
+      password: KkWeface0228
       initial-size: 10
       max-active: 100
       min-idle: 10

src/main/resources/application.yml

@@ -6,15 +6,16 @@ server:
     uri-encoding: UTF-8
     max-threads: 1000
     min-spare-threads: 30
-  port: 8080
     connection-timeout: 5000ms
+  port: 8080
+
 
 spring:
   application:
     name: service-web
     # 环境 dev|test|prod
   profiles:
-    active: dev
+    active: prod
   # jackson时间格式化
   jackson:
     time-zone: GMT+8
@@ -26,14 +27,6 @@ spring:
       enabled: true
   mvc:
     throw-exception-if-no-handler-found: true
-  redis:
-    host: localhost
-    database: 0
-    port: 6379
-    maxTotal: 1000
-    maxIdle: 50
-    minIdle: 10
-    timeout: 1s
 
 getui:
   apps: {
@@ -63,5 +56,12 @@ mybatis-plus:
     jdbc-type-for-null: 'null'
 
 gexiang:
-  user_tag_id: 400000
-  push_max_size: 30000
+  redis:
+    hostname: 172.21.6.6
+    database: 0
+    port: 6379
+    maxTotal: 1000
+    maxIdle: 50
+    minIdle: 10
+  user_tag_id: 251696
+  push_max_size: 10000