安永数据同步优化 cus_get_log service编写

src/main/java/com/api/ebu7dev1/common/getlog/controller/GetLogController.java

@@ -26,13 +26,28 @@ import java.util.Map;
  *
  * @author youHong.ai
  */
-@Path("/ebu7-dev1/common/log")
+@Path("/ebu7-dev1/common/log/")
 public class GetLogController {
 	
 	private final Logger log = Util.getLogger();
 	
 	private final GetLogService service = new GetLogService();
 
+	@GET
+	@Path("ipList")
+	@Produces(MediaType.APPLICATION_JSON)
+	public String getIpList(@Context HttpServletRequest request, @Context HttpServletResponse response) {
+		try {
+			User loginUser = HrmUserVarify.getUser(request, response);
+			String path = request.getRealPath("/");
+			return ApiResult.success(service.getIpList(loginUser, path));
+		} catch (Exception e) {
+			log.error("获取ip节点出错 : " + e.getMessage());
+			log.error(Util.getErrString(e));
+			return ApiResult.error("获取ip节点出错!");
+		}
+	}
+	
 	@GET
 	@Path("get")
 	@Produces(MediaType.APPLICATION_OCTET_STREAM)

src/main/java/com/api/ebu7dev1/common/getlog/service/GetLogService.java

@@ -2,10 +2,14 @@ package com.api.ebu7dev1.common.getlog.service;
 
 import aiyh.utils.Util;
 import aiyh.utils.excention.CustomerException;
+import com.api.ebu7dev1.common.getlog.util.GetLogUtil;
+import org.apache.commons.collections.CollectionUtils;
 import weaver.hrm.User;
 
 import javax.ws.rs.core.Response;
+import java.io.File;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * <h1>获取日志</h1>
@@ -19,12 +23,51 @@ public class GetLogService {
 		return null;
 	}
 
-	public String collectLogInfo(User loginUser, Map<String, Object> params) {
+	/**
-		if (loginUser.getUID() != 1) {
+	 * <h1>获取所有的ip地址</h1>
-			throw new CustomerException("无权限查看！");
+	 * @author xuanran.wang
+	 * @dateTime 2023/7/1 14:19
+	 * @param ecologyPath ec路径
+	 * @return 集群ip 以及 本地ip
+	 **/
+	public Set<String> getIpList(User loginUser, String ecologyPath){
+		GetLogUtil.checkUser(loginUser);
+		String weaverPropertiesPath;
+		String resinPath = System.getProperty("user.dir") + File.separator;
+		if (ecologyPath.endsWith(File.separator)) {
+			weaverPropertiesPath = ecologyPath + "WEB-INF" + File.separator + "prop" + File.separator + "weaver.properties";
+		} else {
+			weaverPropertiesPath = ecologyPath + File.separator + "WEB-INF" + File.separator + "prop" + File.separator + "weaver.properties";
 		}
+		String localIp = GetLogUtil.getInnerIp();
+		Set<String> ipList = GetLogUtil.readEcologyProp(weaverPropertiesPath);
+		if(CollectionUtils.isEmpty(ipList)){
+			String resinBinPath = resinPath + "bin" + File.separator + "startresin.sh";
+			ipList.addAll(GetLogUtil.readResinProp(resinBinPath));
+		}
+		if (ipList.size() != 0) {//没有集群的时候
+			boolean status = false;
+			for (String ip : ipList) {
+				if (ip.contains(localIp) || "127.0.0.1".equals(ip) || "127.0.0.1".equals(localIp)) {
+					status = true;
+					break;
+				}
+			}
+			if (!status) {
+				ipList.add(localIp);
+			}
+		}else {
+			ipList.add(localIp);
+		}
+		return ipList;
+	}
+	
+	public String collectLogInfo(User loginUser, Map<String, Object> params) {
+		GetLogUtil.checkUser(loginUser);
 		String startDate = Util.null2String(params.get("startDate"));
 		String endDate = Util.null2String(params.get("endDate"));
 		return null;
 	}
+
+
 }

src/main/java/weaver/xuanran/wang/eny/data_async/mapper/DataAsyncMapper.java

@@ -1,6 +1,7 @@
 package weaver.xuanran.wang.eny.data_async.mapper;
 
 import aiyh.utils.annotation.recordset.*;
+import io.swagger.models.auth.In;
 import weaver.xuanran.wang.eny.data_async.entity.DataAsyncConfigDetail;
 import weaver.xuanran.wang.eny.data_async.entity.DataAsyncConfigMain;
 
@@ -53,6 +54,13 @@ public interface DataAsyncMapper {
                                             @ParamMapper("min") Integer min,
                                             @ParamMapper("max") Integer max);
 
+    @Select("select id from $t{tableName} where $t{foreignKey} between #{min} and #{max}")
+    @CaseConversion(value = false)
+    List<Integer> selectDataIdsByForeignKey(@ParamMapper("tableName") String tableName,
+                                            @ParamMapper("foreignKey") String foreignKey,
+                                            @ParamMapper("min") Integer min,
+                                            @ParamMapper("max") Integer max);
+
     @Select(custom = true)
     String selectCustomerSql(@SqlString String sql, Map<String, Object> map);
 

src/main/java/weaver/xuanran/wang/eny/data_async/service/DataAsyncServiceImpl.java

@@ -21,6 +21,7 @@ import weaver.xuanran.wang.eny.data_async.entity.DataAsyncConfigMain;
 import weaver.xuanran.wang.eny.data_async.mapper.DataAsyncMapper;
 import weaver.xuanran.wang.eny.data_async.util.EyDataAsyncTokenUtil;
 import weaver.xuanran.wang.eny.data_async.util.ValueRuleMethod;
+
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -79,11 +80,12 @@ public class DataAsyncServiceImpl {
 
     /**
      * <h1>数据同步 有一个点 就是如果接口请求过程中报错了 怎么重新处理</h1>
-     * @author xuanran.wang
+     *
-     * @dateTime 2023/6/9 13:28
      * @param config   配置对象
      * @param tokenUrl token地址
      * @param params   token 参数
+     * @author xuanran.wang
+     * @dateTime 2023/6/9 13:28
      **/
     public void asyncData(DataAsyncConfigMain config, String tokenUrl, Map<String, Object> params) {
         long startTime = System.currentTimeMillis();
@@ -105,6 +107,12 @@ public class DataAsyncServiceImpl {
             count += (pageNo - 1) * pageSize;
             preNum = count;
         }
+        List<DataAsyncConfigDetail> configDetailList = config.getConfigDetailList();
+        List<DataAsyncConfigDetail> primaryKeyList = configDetailList.stream().filter(item -> item.getPrimary_key() == 0).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(primaryKeyList)) {
+            throw new CustomerException("请先配置主键字段!");
+        }
+        DataAsyncConfigDetail primaryKey = primaryKeyList.get(0);
         boolean show = true;
         while (count < total) {
             // 从缓存中获取token
@@ -130,7 +138,7 @@ public class DataAsyncServiceImpl {
                 count += pageSize;
                 List<Map<String, Object>> list = (List<Map<String, Object>>) data.get("list");
                 // 进行数据处理
-                maps = dealData(config, list);
+                maps = dealData(config, list, primaryKey);
                 show = false;
             } catch (Exception e) {
                 for (int i = 0; i < pageSize; i++) {
@@ -149,10 +157,24 @@ public class DataAsyncServiceImpl {
                 // 异步提交数据库处理
                 threadPoolInstance.execute(() -> {
                     log.info("=======================================================================================");
+                    // 查询当天同步的数据
+                    Map<String, Integer> map = parseMaxAndMin(maps, primaryKey);
+                    Integer max = map.get("max");
+                    Integer min = map.get("min");
                     log.info(Thread.currentThread().getName() + " 入库之前信号数量 : " + finalLatch.getCount());
                     boolean success = CusData2OA.baseInsertAndUpdate(config.getModel_id(), maps, finalLatch);
                     log.info("数据库更新标识 : " + success);
+                    if (!success) {
+                        throw new CustomerException("更新数据失败!");
+                    }
                     log.info(Thread.currentThread().getName() + " 入库之后信号数量 : " + finalLatch.getCount());
+                    List<Integer> dataIdList = asyncMapper.selectDataIdsByForeignKey(config.getTable_name(), primaryKey.getModel_field_name(), min, max);
+                    if (CollectionUtils.isNotEmpty(dataIdList)) {
+                        log.info("需要权限重构的数据条数 : " + dataIdList.size());
+                        for (Integer id : dataIdList) {
+                            moderightinfo.rebuildModeDataShareByEdit(1, Util.getIntValue(config.getModel_id(), -1), id);
+                        }
+                    }
                     log.info("=======================================================================================");
                 });
             }
@@ -167,15 +189,6 @@ public class DataAsyncServiceImpl {
                 if (!await) {
                     throw new CustomerException("线程等待时间超过最大时间限制!");
                 }
-                // 查询当天同步的数据
-                // 应该在上面 每一批数据处理完直接权限重构
-                List<Integer> dataIdList = asyncMapper.selectDataIdListByCreateDate(config.getTable_name(), TimeUtil.getCurrentDateString());
-                if(CollectionUtils.isNotEmpty(dataIdList)){
-                    log.info("需要权限重构的数据条数 : " + dataIdList.size());
-                    for (Integer id : dataIdList) {
-                        moderightinfo.rebuildModeDataShareByEdit(1, Util.getIntValue(config.getModel_id(),-1), id);
-                    }
-                }
                 log.info("数据结束同步时间 : " + TimeUtil.getCurrentTimeString());
                 long endTime = System.currentTimeMillis();
                 log.info("同步耗时时间 " + (endTime - startTime) / 1000 + " s");// 等待所有转换操作完成
@@ -187,28 +200,25 @@ public class DataAsyncServiceImpl {
 
     /**
      * <h1>同步处理数据</h1>
-     * @author xuanran.wang
+     *
-     * @dateTime 2023/6/8 16:50
      * @param config 配置对象
      * @param data   接口数据
+     * @author xuanran.wang
+     * @dateTime 2023/6/8 16:50
      **/
-    public List<Map<String, Object>> dealData(DataAsyncConfigMain config, List<Map<String, Object>> data)  {
+    public List<Map<String, Object>> dealData(DataAsyncConfigMain config,
+                                              List<Map<String, Object>> data,
+                                              DataAsyncConfigDetail primaryKey) {
         List<DataAsyncConfigDetail> configDetailList = config.getConfigDetailList();
-        List<DataAsyncConfigDetail> primaryKey = configDetailList.stream().filter(item -> item.getPrimary_key() == 0).collect(Collectors.toList());
+//        List<DataAsyncConfigDetail> primaryKey = configDetailList.stream().filter(item -> item.getPrimary_key() == 0).collect(Collectors.toList());
-        List<Map<String, Object>> maps = new ArrayList<>();
+        List<Map<String, Object>> maps;
-        if(CollectionUtils.isNotEmpty(primaryKey)){
         // 模版-接口外键
-            String primaryKeyModelFieldName = primaryKey.get(0).getModel_field_name();
+        String primaryKeyModelFieldName = primaryKey.getModel_field_name();
-            maps = data.stream().map(item -> convert(item, primaryKey.get(0).getInterface_field(), configDetailList)).filter(MapUtils::isNotEmpty).collect(Collectors.toList());
+        maps = data.stream().map(item -> convert(item, primaryKey.getInterface_field(), configDetailList)).filter(MapUtils::isNotEmpty).collect(Collectors.toList());
         // 进行排序
-            Optional<Integer> minClassId = maps.stream()
+        Map<String, Integer> map = parseMaxAndMin(maps, primaryKey);
-                    .map(map -> Util.getIntValue(Util.null2DefaultStr(map.get(primaryKeyModelFieldName),""),-1))
+        Integer max = map.get("max");
-                    .min(Comparator.naturalOrder());
+        Integer min = map.get("min");
-            Optional<Integer> maxClassId = maps.stream()
-                    .map(map -> Util.getIntValue(Util.null2DefaultStr(map.get(primaryKeyModelFieldName),""),-1))
-                    .max(Comparator.naturalOrder());
-            Integer min = minClassId.orElse(0);
-            Integer max = maxClassId.orElse(0);
         // 按照外键排序并且在oa中范围查询出外键与oa数据的对应关系
         List<Map<String, String>> dataIdList = asyncMapper.selectDataIds(primaryKeyModelFieldName, config.getTable_name(), min, max);
         if ("1".equals(debug)) {
@@ -233,7 +243,6 @@ public class DataAsyncServiceImpl {
                 }
             });
         }
-        }
         return maps;
     }
 
@@ -257,12 +266,13 @@ public class DataAsyncServiceImpl {
 
     /**
      * <h1>将数据进行转换</h1>
-     * @author xuanran.wang
+     *
-     * @dateTime 2023/6/9 13:27
      * @param data             原始数据
      * @param foreignKey       外键
      * @param configDetailList 配置集合
      * @return 转换后的数据
+     * @author xuanran.wang
+     * @dateTime 2023/6/9 13:27
      **/
     public Map<String, Object> convert(Map<String, Object> data,
                                        String foreignKey,
@@ -280,9 +290,10 @@ public class DataAsyncServiceImpl {
 
     /**
      * <h1>将集合数据放到缓存中</h1>
+     *
+     * @param listMap 集合数据
      * @author xuanran.wang
      * @dateTime 2023/4/10 18:33
-     * @param listMap 集合数据
      **/
     public static HashMap<String, String> parseListMap2Map(String key, String value, List<Map<String, String>> listMap) {
         if (CollectionUtils.isEmpty(listMap)) {
@@ -300,6 +311,4 @@ public class DataAsyncServiceImpl {
     }
 
 
-
-
 }

src/test/java/xuanran/wang/eny/DataAsyncTest.java

@@ -37,7 +37,7 @@ public class DataAsyncTest extends BaseTest {
         String clientId = "6d067ac6c14211ed8fd700163e1331c6";
         String appId = "WEAVER";
         String appSecret = "7b09e5f7c14211ed8fd700163e1331c6";
-        int pageSize = 500;
+        int pageSize = 100;
         int pageNo = 25;
         dataAsyncService.setPageSize(25);
         dataAsyncService.setPageSize(pageSize);