去重原生SQL

//先去重00状态的数据

//分组获取不重复的数据进行统计

SELECT * ,count(*) num FROM scp_trade_tradeinfohl where state='00' GROUP BY TAC,txnTime,samId,txnAmt ORDER BY num DESC

//分组获取重复的数据

SELECT * ,count(*) num FROM scp_trade_tradeinfohl where state='00' GROUP BY TAC,txnTime,samId,txnAmt HAVING num>1 ORDER BY num DESC

//分组后获取重复数据中每一组只取uuid最小的一条数据

SELECT * ,MIN(uuid) as uuid ,TAC,count(*) num FROM scp_trade_tradeinfohl where state='00' GROUP BY TAC,txnTime,samId,txnAmt HAVING num>1  ORDER BY num DESC

SELECT * FROM scp_trade_tradeinfohl WHERE TAC='5DA6A1F8';

//===========================================================================================================================================================================================================================================

00    状态代表上送上来的数据状态

04    状态代表需要对进行操作的数据    防止实时数据对去重的干扰

03    状态代表04状态下不重复的数据

02    状态代表04这状态下的重复数据状态

01    状态代表需要05状态下生成交文件后的状态     已经生成交易文件数据的状态

05    状态代表03状态与01状态数据进行比对后的不重复数据

06    状态代表03状态与01状态数据进行比对后的重复数据

//先将00状态的数据状态修改为04代表要向这批数据进行去重 也防止实时数据干扰去重

update scp_trade_tradeinfohl set state = '04' WHERE state='00';

update scp_trade_tradeinfo set state = '04' WHERE state='00';

1.00状态的数据去重

//将分组统计后每组个数小于等于1的数据状态修改为03代表04状态下不重复数据

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='04' GROUP BY TAC,txnTime,samId,txnAmt HAVING num<=1

 ORDER BY num DESC) t SET a.state='03' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid AND a.state='04'

//将分组统计每组个数大于1的数据只取uuid最小的那条数据状态修改为03代表04状态下不重复数据中的一条

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='04' GROUP BY TAC,txnTime,samId,txnAmt HAVING num>1

 ORDER BY num DESC) t SET a.state='03' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid AND a.state='04'

//将00状态剩余的数据状态修改为02代表00状态下的重复数据

UPDATE scp_trade_tradeinfohl SET state='02' WHERE state='04'

更新scp_trade_tradeinfo表中的状态

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='03' where sct.uuid=scth.uuid and scth.state='03';

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='02' where sct.uuid=scth.uuid and scth.state='02';

2.03状态的数据与01状态的数据进行去重

//将分组统计后每组个数小于等于1的数据状态修改为05代表不重复数据

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='03'  OR state='01' GROUP BY TAC,txnTime,samId,txnAmt HAVING num<=1

 ORDER BY num DESC) t SET a.state='05' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid AND a.state='03'

//将分组统计每组个数大于1的数据只取uuid最小的那条数据状态修改为03代表不重复数据中的一条

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='03' OR state='01' GROUP BY TAC,txnTime,samId,txnAmt HAVING num>1

 ORDER BY num DESC) t SET a.state='05' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid AND a.state='03'

//将03状态剩余的数据状态修改为06代表03状态与已经传送过交易状态为01下的重复数据

UPDATE scp_trade_tradeinfohl SET state='06' WHERE state='03'

更新scp_trade_tradeinfo表中的状态

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='05' where sct.uuid=scth.uuid and scth.state='05';

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='06' where sct.uuid=scth.uuid and scth.state='06';

//===========================================================================================================================================================================================================================================

00    状态代表上送上来的数据状态

04    状态代表需要对进行操作的数据    防止实时数据对去重的干扰

03    状态代表04状态下不重复的数据

02    状态代表04这状态下的重复数据状态

01    状态代表需要05状态下生成交文件后的状态     已经生成交易文件数据的状态

05    状态代表03状态与01状态数据进行比对后的不重复数据

06    状态代表03状态与01状态数据进行比对后的重复数据

//先将00状态的数据状态修改为04代表要向这批数据进行去重 也防止实时数据干扰去重

update scp_trade_tradeinfohl set state = '04' WHERE state='00'

update scp_trade_tradeinfo set state = '04' WHERE state='00'

1.00状态的数据去重

//将分组统计后每组个数小于等于1的数据状态修改为03代表04状态下不重复数据

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='04' OR state='01' GROUP BY TAC,txnTime,samId,txnAmt HAVING num<=1

 ORDER BY num DESC) t SET a.state='03' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid and t.state='04'

//将分组统计每组个数大于1的数据只取uuid最小的那条数据状态修改为03代表04状态下不重复数据中的一条

UPDATE scp_trade_tradeinfohl a ,(SELECT TAC,txnTime,samId,txnAmt,state,MIN(uuid) as uuid,count(*) num FROM scp_trade_tradeinfohl where state='04' OR state='01' GROUP BY TAC,txnTime,samId,txnAmt HAVING num>1

 ORDER BY num DESC) t SET a.state='03' WHERE a.TAC=t.TAC AND a.txnTime=t.txnTime AND a.samId=t.samId AND a.txnAmt = t.txnAmt AND a.uuid=t.uuid    and t.state='04'

//将00状态剩余的数据状态修改为02代表00状态下的重复数据

UPDATE scp_trade_tradeinfohl SET state='02' WHERE state='04'

更新scp_trade_tradeinfo表中的状态

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='03' where sct.uuid=scth.uuid and scth.state='03';

UPDATE scp_trade_tradeinfo as sct,scp_trade_tradeinfohl as scth SET sct.state='02' where sct.uuid=scth.uuid and scth.state='02';

//==========================================================================================================================================================================================================================================