Files
nuttyreading-java/src/main/java/com/peanut/common/utils/DataMigrationUtil.java
2024-05-15 16:45:27 +08:00

507 lines
29 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.peanut.common.utils;
import cn.hutool.core.date.DateUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import java.math.BigDecimal;
import java.sql.*;
import java.util.*;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class DataMigrationUtil extends Thread {
Map map;
int i;
Connection fzdsconn;
public DataMigrationUtil(Map map,int i,Connection fzdsconn){
this.map = map;
this.i = i;
this.fzdsconn = fzdsconn;
}
public static void main(String[] args){
// catalogue();
// courseCatalogue();
// courseCatalogueChapter();
// courseCatalogueChapterVideo();
// user();//用的都是copy表里的数据
//TRUNCATE
}
public static void catalogue(){
try {
Connection yljkconn = DriverManager.getConnection(
"jdbc:mysql://goldorchid.mysql.rds.aliyuncs.com:3309/everhealth?",
"yljkmaster", "Wu751019!@");
PreparedStatement statement = yljkconn.prepareStatement("select * from t_curriculum_catalogue where valid = 1" +
" and poid in (select oid from t_curriculum_catalogue where valid = 1 and poid is null) " +
"and (courseFee like '%,%' or courseFee = '0' or courseFee = '' or poid = '00000000000000000000000000000070' )");
ResultSet resultSet = statement.executeQuery();
List<Map> list = new ArrayList();
while(resultSet.next()){
Map map = new HashMap();
map.put("oid",resultSet.getString("oid"));
map.put("title",resultSet.getString("title"));
map.put("sort",resultSet.getString("orderNo"));
map.put("image",resultSet.getString("thumbnail"));
map.put("content",resultSet.getString("introduceTxt"));
map.put("createDate",resultSet.getString("createDate"));
list.add(map);
}
resultSet.close();
yljkconn.close();
statement.close();
Connection connection = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
// 关闭自动提交事务,改为手动提交
connection.setAutoCommit(false);
System.out.println("===== 开始插入数据 =====");
long startTime = System.currentTimeMillis();
PreparedStatement preparedStatement = connection.prepareStatement("INSERT ignore INTO course_copy1 (uid,title,sort,image,content,create_time) VALUES ( ?, ?, ?, ?, ?, ?)");
for (int i = 0; i < list.size(); i++) {
preparedStatement.setString(1,list.get(i).get("oid").toString());
preparedStatement.setString(2,list.get(i).get("title").toString());
preparedStatement.setString(3,list.get(i).get("sort").toString());
preparedStatement.setString(4,list.get(i).get("image").toString());
preparedStatement.setString(5,list.get(i).get("content").toString());
preparedStatement.setString(6,list.get(i).get("createDate").toString());
// 添加到批处理中
preparedStatement.addBatch();
if (i+1 % 5000 == 0) {
long startTime1 = System.currentTimeMillis();
// 每1000条数据提交一次
preparedStatement.executeBatch();
connection.commit();
long spendTime1 = System.currentTimeMillis()-startTime1;
System.out.println("成功插入第 "+ i+1+" 条数据,耗时:"+spendTime1+"毫秒");
}
}
// 处理剩余的数据
preparedStatement.executeBatch();
connection.commit();
connection.setAutoCommit(true);//在把自动提交打开
long spendTime = System.currentTimeMillis()-startTime;
System.out.println("成功插入"+list.size()+"条数据,耗时:"+spendTime+"毫秒");
connection.close();
preparedStatement.close();
} catch (SQLException e) {
System.out.println("Error: " + e.getMessage());
}
}
public static void courseCatalogue(){
try {
Connection fzdsconn = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
PreparedStatement fzdsstatement = fzdsconn.prepareStatement("select id,uid from course ");
ResultSet fzdsresultSet = fzdsstatement.executeQuery();
List<Map> fzdslist = new ArrayList();
while(fzdsresultSet.next()){
Map map = new HashMap();
map.put("id",fzdsresultSet.getString("id"));
map.put("uid",fzdsresultSet.getString("uid"));
fzdslist.add(map);
}
fzdsresultSet.close();
fzdsconn.close();
fzdsstatement.close();
Connection yljkconn = DriverManager.getConnection(
"jdbc:mysql://goldorchid.mysql.rds.aliyuncs.com:3309/everhealth?",
"yljkmaster", "Wu751019!@");
Connection connection = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
// 关闭自动提交事务,改为手动提交
connection.setAutoCommit(false);
long startTime = System.currentTimeMillis();
System.out.println("===== 开始插入数据 =====");
int res = 0;
PreparedStatement preparedStatement = connection.prepareStatement(
"INSERT ignore INTO course_catalogue (type,course_id,title,half_fee,fee,create_time) VALUES (?,?,?,?,?,?)");
for (int q=1;q<=fzdslist.size(); q++){
String selectSql = "select ccp.poid ppoid,cc.* from t_curriculum_catalogue cc left join t_curriculum_catalogue ccp on cc.poid = ccp.oid where ccp.valid = 1 and cc.valid = 1 and cc.poid = '"+fzdslist.get(q-1).get("uid")+"'";
PreparedStatement statement = yljkconn.prepareStatement(selectSql);
ResultSet resultSet = statement.executeQuery();
Map map = new HashMap();
while(resultSet.next()){
String halfFee = "";
String fee = "";
if (resultSet.getString("courseFee")!=null&&resultSet.getString("courseFee").contains(",")){
halfFee = resultSet.getString("courseFee").split(",")[0];
fee = resultSet.getString("courseFee").split(",")[1];
}else if (resultSet.getString("courseFee")!=null&&"0".equals(resultSet.getString("courseFee"))){
halfFee = "0";
fee = "0";
}
if ("0".equals(resultSet.getString("courseFee"))
||("".equals(resultSet.getString("courseFee"))
&&!"00000000000000000000000000000070".equals(resultSet.getString("ppoid")))){
preparedStatement.setInt(1,0);
}else if ("00000000000000000000000000000070".equals(resultSet.getString("ppoid"))){
preparedStatement.setInt(1,2);
}else {
preparedStatement.setInt(1,1);
}
preparedStatement.setString(2,fzdslist.get(q-1).get("id").toString());
preparedStatement.setString(3,resultSet.getString("title"));
preparedStatement.setString(4,halfFee);
preparedStatement.setString(5,fee);
preparedStatement.setString(6,resultSet.getString("createDate"));
// 添加到批处理中
preparedStatement.addBatch();
res++;
if (res % 5000 == 0) {
long startTime1 = System.currentTimeMillis();
preparedStatement.executeBatch();
connection.commit();
long spendTime1 = System.currentTimeMillis()-startTime1;
System.out.println("成功插入第"+ res +" 条数据,耗时:"+spendTime1+"毫秒");
}
}
resultSet.close();
statement.close();
}
//处理剩余的数据
preparedStatement.executeBatch();
connection.commit();
preparedStatement.close();
yljkconn.close();
connection.setAutoCommit(true);//在把自动提交打开
connection.close();
long spendTime = System.currentTimeMillis()-startTime;
System.out.println("成功插入"+res+"条数据,耗时:"+spendTime+"毫秒");
} catch (SQLException e) {
System.out.println("Error: " + e.getMessage());
}
}
public static void courseCatalogueChapter(){
try {
Connection fzdsconn = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
PreparedStatement fzdsstatement = fzdsconn.prepareStatement(
"select c.id,c.uid,cc.id ccid,cc.title cctitle from course c left join course_catalogue cc on c.id = cc.course_id ");
ResultSet fzdsresultSet = fzdsstatement.executeQuery();
List<Map> fzdslist = new ArrayList();
while(fzdsresultSet.next()){
Map map = new HashMap();
map.put("id",fzdsresultSet.getString("id"));
map.put("uid",fzdsresultSet.getString("uid"));
map.put("ccid",fzdsresultSet.getString("ccid"));
map.put("cctitle",fzdsresultSet.getString("cctitle"));
fzdslist.add(map);
}
Connection yljkconn = DriverManager.getConnection(
"jdbc:mysql://goldorchid.mysql.rds.aliyuncs.com:3309/everhealth?",
"yljkmaster", "Wu751019!@");
Connection connection = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
long startTime = System.currentTimeMillis();
System.out.println("===== 开始插入数据 =====");
int res = 0;
PreparedStatement preparedStatement = connection.prepareStatement(
"INSERT ignore INTO course_catalogue_chapter (course_id,catalogue_id,title,img_url,content,create_time,sort) VALUES ( ?, ?, ?, ?, ?, ?, ?)");
for (int q=1;q<=fzdslist.size(); q++){
String catalogueSql = "select oid,title from t_curriculum_catalogue where valid = 1 and poid = '"+fzdslist.get(q-1).get("uid")+"'";
PreparedStatement catalogueStatement = yljkconn.prepareStatement(catalogueSql);
ResultSet catalogueResultSet = catalogueStatement.executeQuery();
while(catalogueResultSet.next()){
if (catalogueResultSet.getString("title").equals(fzdslist.get(q-1).get("cctitle"))){
String detailSql = "select * from t_curriculum_detail where valid = 1 and poid = '"+catalogueResultSet.getString("oid")+"'";
PreparedStatement statement = yljkconn.prepareStatement(detailSql);
ResultSet resultSet = statement.executeQuery();
while(resultSet.next()){
preparedStatement.setString(1,fzdslist.get(q-1).get("id").toString());
preparedStatement.setString(2,fzdslist.get(q-1).get("ccid").toString());
preparedStatement.setString(3,resultSet.getString("title"));
preparedStatement.setString(4,resultSet.getString("imgUrl"));
preparedStatement.setString(5,resultSet.getString("explains"));
preparedStatement.setString(6,resultSet.getString("createDate"));
preparedStatement.setString(7,resultSet.getString("orderNo"));
// 添加到批处理中
preparedStatement.addBatch();
res++;
if (res % 5000 == 0) {
long startTime1 = System.currentTimeMillis();
preparedStatement.executeBatch();
long spendTime1 = System.currentTimeMillis()-startTime1;
System.out.println("成功插入第"+ res +" 条数据,耗时:"+spendTime1+"毫秒");
}
}
}
}
}
//处理剩余的数据
preparedStatement.executeBatch();
long spendTime = System.currentTimeMillis()-startTime;
System.out.println("成功插入"+res+"条数据,耗时:"+spendTime+"毫秒");
fzdsconn.close();
yljkconn.close();
connection.close();
} catch (SQLException e) {
System.out.println("Error: " + e.getMessage());
}
}
public static void courseCatalogueChapterVideo(){
try {
Connection fzdsconn = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
PreparedStatement fzdsstatement = fzdsconn.prepareStatement(
"select c.uid,ccc.id as cccid,ccc.title from course c left join course_catalogue_chapter ccc on c.id = ccc.course_id");
ResultSet fzdsresultSet = fzdsstatement.executeQuery();
List<Map> fzdslist = new ArrayList();
while(fzdsresultSet.next()){
Map map = new HashMap();
map.put("uid",fzdsresultSet.getString("uid"));
map.put("cccid",fzdsresultSet.getString("cccid"));
map.put("title",fzdsresultSet.getString("title"));
fzdslist.add(map);
}
Connection yljkconn = DriverManager.getConnection(
"jdbc:mysql://goldorchid.mysql.rds.aliyuncs.com:3309/everhealth?",
"yljkmaster", "Wu751019!@");
Connection connection = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
long startTime = System.currentTimeMillis();
System.out.println("===== 开始插入数据 =====");
int res = 0;
PreparedStatement preparedStatement = connection.prepareStatement(
"INSERT ignore INTO course_catalogue_chapter_video (chapter_id,type,video,create_time,sort) VALUES (?, ?, ?, ?, ?)");
for (int q=1;q<=fzdslist.size(); q++){
String selectSql = "select * from t_curriculum_detail where valid = 1 and poid in (select oid from t_curriculum_catalogue where valid = 1 and poid = '"+fzdslist.get(q-1).get("uid")+"') and title = '"+fzdslist.get(q-1).get("title")+"' ";
PreparedStatement statement = yljkconn.prepareStatement(selectSql);
ResultSet resultSet = statement.executeQuery();
Map map = new HashMap();
while(resultSet.next()){
if (StringUtils.isNotEmpty(resultSet.getString("videoId"))){
map.put("type","1");
map.put("video",resultSet.getString("videoId"));
}else if (StringUtils.isNotEmpty(resultSet.getString("videos"))){
map.put("type","0");
map.put("video",resultSet.getString("videos"));
}else if (StringUtils.isNotEmpty(resultSet.getString("voices"))){
map.put("type","2");
map.put("video",resultSet.getString("voices"));
}
if (map.size() > 0){
String[] tt = map.get("video").toString().split(",");
if (tt.length>0){
for (int r=0;r<tt.length;r++){
if (!tt[r].equals("")) {
preparedStatement.setString(1,fzdslist.get(q-1).get("cccid").toString());
preparedStatement.setString(2,map.get("type").toString());
preparedStatement.setString(3,tt[r]);
preparedStatement.setString(4,resultSet.getString("createDate"));
preparedStatement.setString(5,r+1+"");
// 添加到批处理中
preparedStatement.addBatch();
res++;
if (res % 5000 == 0) {
long startTime1 = System.currentTimeMillis();
preparedStatement.executeBatch();
long spendTime1 = System.currentTimeMillis()-startTime1;
System.out.println("成功插入第"+ res +" 条数据,耗时:"+spendTime1+"毫秒");
}
}
}
}
}
}
}
//处理剩余的数据
preparedStatement.executeBatch();
long spendTime = System.currentTimeMillis()-startTime;
System.out.println("成功插入"+res+"条数据,耗时:"+spendTime+"毫秒");
fzdsconn.close();
yljkconn.close();
connection.close();
} catch (SQLException e) {
System.out.println("Error: " + e.getMessage());
}
}
public static void user(){
try {
long startTime = System.currentTimeMillis();
Connection yljkconn = DriverManager.getConnection(
"jdbc:mysql://goldorchid.mysql.rds.aliyuncs.com:3309/everhealth?",
"yljkmaster", "Wu751019!@");
PreparedStatement statement = yljkconn.prepareStatement("select oid,cellPhone,nameCN,nickName,icons,superVIP,point,pointByJF,payStatus,paydate,payValidDate from t_customer where valid = 1 and cellphone != '' and cellphone is not null ");
ResultSet resultSet = statement.executeQuery();
List<Map> list = new ArrayList();
while(resultSet.next()){
Map map = new HashMap();
map.put("oid",resultSet.getString("oid"));
map.put("cellPhone",resultSet.getString("cellPhone"));
map.put("nameCN",resultSet.getString("nameCN"));
map.put("nickName",resultSet.getString("nickName"));
map.put("icons",resultSet.getString("icons"));
map.put("superVIP",resultSet.getString("superVIP"));
map.put("point",resultSet.getString("point"));
map.put("pointByJF",resultSet.getString("pointByJF"));
map.put("payStatus",resultSet.getString("payStatus"));
map.put("paydate",resultSet.getString("paydate"));
map.put("payValidDate",resultSet.getString("payValidDate"));
list.add(map);
}
resultSet.close();
statement.close();
yljkconn.close();
Connection fzdsconn = DriverManager.getConnection(
"jdbc:mysql://rm-2zev4157t67trxuu3yo.mysql.rds.aliyuncs.com:3306/e_book_test?rewriteBatchedStatements=true",
"nuttyreading", "Wu751019!");
// 1、创建服务创建线程池
ExecutorService service = Executors.newFixedThreadPool(50);
for (int i=0;i<list.size(); i++){
DataMigrationUtil d1 = new DataMigrationUtil(list.get(i),i+1,fzdsconn);
service.execute(d1);
}
System.out.println("---开始");
} catch (SQLException e) {
System.out.println("Error: " + e.getMessage());
}
}
@Override
public void run(){
try {
if (i % 1000 == 0) {
System.out.println("---已插入数据"+i+"--"+ DateUtil.format(new Date(),"YYYY-MM-dd HH:mm:ss"));
}
String userId = "";
BigDecimal userBalanceCoin = new BigDecimal(0);
BigDecimal userBalanceJf = new BigDecimal(0);
PreparedStatement oidStatements = fzdsconn.prepareStatement("select * from user_copy1 where del_flag = 0 and yljk_oid = '"+map.get("oid")+"' ");
ResultSet oidResultSet = oidStatements.executeQuery();
//已绑定过id
if (oidResultSet.next()){
userBalanceCoin = oidResultSet.getBigDecimal("peanut_coin").add(new BigDecimal(map.get("point").toString()));
userBalanceJf = oidResultSet.getBigDecimal("jf").add(new BigDecimal(map.get("pointByJF").toString()));
String sql = "update user_copy1 set " +
"name = if(isnull(name),'"+map.get("nameCN")+"',name) , nickname = if(isnull(nickname),'"+map.get("nickName")+"',nickname) ," +
"tel = if(isnull(tel),'"+map.get("cellPhone")+"',tel) ," +
"vip= "+map.get("payStatus")+" ,peanut_coin = peanut_coin + "+map.get("point")+" , jf = jf + "+map.get("pointByJF")+" where id = "+oidResultSet.getString("id")+"";
PreparedStatement oidUpdateStatement = fzdsconn.prepareStatement(sql);
userId = oidResultSet.getString("id");
oidUpdateStatement.execute();
oidUpdateStatement.close();
}else {
//未绑定过id用电话查询
PreparedStatement telStatements = fzdsconn.prepareStatement("select * from user_copy1 where del_flag = 0 and tel = '"+map.get("cellPhone")+"' ");
ResultSet telResultSet = telStatements.executeQuery();
if (telResultSet.next()){
if (telResultSet.getRow()>1){//这里需要检查
System.out.println(map.get("oid").toString()+"---"+map.get("nameCN").toString()+"有多账号");
}else {
userBalanceCoin = telResultSet.getBigDecimal("peanut_coin").add(new BigDecimal(map.get("point").toString()));
userBalanceJf = telResultSet.getBigDecimal("jf").add(new BigDecimal(map.get("pointByJF").toString()));
String sql = "update user_copy1 set yljk_oid = '" + map.get("oid") +"',"+
" name = if(isnull(name),'"+map.get("nameCN")+"',name) , nickname = if(isnull(nickname),'"+map.get("nickName")+"',nickname) ," +
"vip= "+map.get("payStatus")+" ,peanut_coin = peanut_coin + "+map.get("point")+" , jf = jf + "+map.get("pointByJF")+" where tel = '"+map.get("cellPhone")+"'";
PreparedStatement telUpdateStatement = fzdsconn.prepareStatement(sql);
userId = telResultSet.getString("id");
telUpdateStatement.execute();
telUpdateStatement.close();
}
}else {
//没有关联,直接导入
PreparedStatement telInsertStatement = fzdsconn.prepareStatement(
"INSERT ignore INTO user_copy1 (name,avatar,nickname,tel,vip,peanut_coin,jf,yljk_oid) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
telInsertStatement.setString(1,map.get("nameCN")==null?"":map.get("nameCN").toString());
telInsertStatement.setString(2,map.get("icons")==null?"":map.get("icons").toString());
telInsertStatement.setString(3,map.get("nickName")==null?"":map.get("nickName").toString());
telInsertStatement.setString(4,map.get("cellPhone")==null?"":map.get("cellPhone").toString());
telInsertStatement.setString(5,map.get("payStatus")==null?"":map.get("payStatus").toString());
telInsertStatement.setString(6,map.get("point")==null?"":map.get("point").toString());
telInsertStatement.setString(7,map.get("pointByJF")==null?"":map.get("pointByJF").toString());
telInsertStatement.setString(8,map.get("oid")==null?"":map.get("oid").toString());
telInsertStatement.execute();//获取添加之后自动增长id值
ResultSet rs = telInsertStatement.getGeneratedKeys();
while(rs.next()) {
Object id = rs.getObject(1);
userId = id.toString();
}
telInsertStatement.close();
userBalanceCoin = new BigDecimal(map.get("point").toString());
userBalanceJf = new BigDecimal(map.get("pointByJF").toString());
}
}
//添加超v
if (map.get("paydate")!=null&&map.get("payValidDate")!=null){
PreparedStatement userVipStatement = fzdsconn.prepareStatement(
"INSERT ignore INTO user_vip_copy1 (user_id,type,start_time,end_time,state) VALUES (?, ?, ?, ?, ?)");
userVipStatement.setString(1,userId);
userVipStatement.setString(2,map.get("payStatus").toString());
userVipStatement.setString(3,map.get("paydate").toString());
userVipStatement.setString(4,map.get("payValidDate").toString());
if (DateUtil.compare(DateUtil.parseDate(map.get("payValidDate").toString()),new Date())>0){
userVipStatement.setString(5,"0");
}else {
userVipStatement.setString(5,"1");
}
userVipStatement.execute();
userVipStatement.close();
}
if (map.get("point")!=null&&!"0".equals(map.get("point").toString())){
//添加虚拟币记录
PreparedStatement transactionDetailStatement = fzdsconn.prepareStatement(
"INSERT ignore INTO transaction_details_copy1 (user_id,order_type,change_amount,remark,user_balance," +
"user_name,tel) VALUES (?, ?, ?, ?, ?, ?, ?)");
transactionDetailStatement.setString(1,userId);
transactionDetailStatement.setString(2,"系统合并结算");
transactionDetailStatement.setString(3,map.get("point").toString());
transactionDetailStatement.setString(4,"合并");
transactionDetailStatement.setBigDecimal(5,userBalanceCoin);
transactionDetailStatement.setString(6,map.get("nameCN")==null?"":map.get("nameCN").toString());
transactionDetailStatement.setString(7,map.get("cellPhone").toString());
transactionDetailStatement.execute();
transactionDetailStatement.close();
}
if (map.get("pointByJF")!=null&&!"0".equals(map.get("pointByJF").toString())){
//添加积分记录
PreparedStatement jfTransactionDetailStatement = fzdsconn.prepareStatement(
"INSERT ignore INTO jf_transaction_details_copy1 (user_id,act_type,change_amount,user_balance,remark) " +
"VALUES (?, ?, ?, ?, ?)");
jfTransactionDetailStatement.setString(1,userId);
jfTransactionDetailStatement.setString(2,"0");
jfTransactionDetailStatement.setString(3,map.get("pointByJF").toString());
jfTransactionDetailStatement.setBigDecimal(4,userBalanceJf);
jfTransactionDetailStatement.setString(5,"系统合并结算");
jfTransactionDetailStatement.execute();
jfTransactionDetailStatement.close();
}
}catch (Exception ee){
}
}
}