进行数据管理时，无效数据可能会对生产力和决策质量造成严重的影响。如何发现和处理无效数据变得愈发重要。一起来唠唠你会如何处理无效数据吧~
方向一：介绍无效数据的概念
最近遇到了pg数据库表中的大量数据重复了，需要删除其中的一条。一条条删除显然不切合实际，还是需要通过计算来删除。
方向二：无效数据的处理方法
实施步骤：
1.对原表进行备份
2.使用一个表结构完全一样的临时表对原始表进行接收。
create table tmp_0524_1 as select * from public.m_user_bak20230524;
3.找出重复的数据：
drop table tmp_0524_2;
create table tmp_0524_2 as
select * from (
select phone ,count(1) cn from tmp_0524_1 where phone is not null group by phone ) t where t.cn > 1 ;4.用第二个临时表接收数据：
drop table tmp_0524_3_1;
create table tmp_0524_3_1 as
select t.* from tmp_0524_1 t where phone in (select phone from tmp_0524_2 ) and opt_user is null;5.建一个表结构一样的空表，多增加一个iid字段
create table tmp_0524_3 as select '' iid , t.* from tmp_0524_3_1 where 1 = 2;
6.给空表增加一个iid字段，建立一个字段自增
CREATE SEQUENCE tmp_0524_3_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
alter table tmp_0524_3 alter column iid set default nextval('tmp_0524_3_id_seq');7.插入数据：
INSERT INTO public.tmp_0524_3
(id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company)
select id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company
from tmp_0524_3_1;8.删除相同数据中的一条：
DELETE FROM tmp_0524_3
WHERE iid
NOT IN (
SELECT max(iid)
FROM tmp_0524_3
GROUP BY phone
);9.删除原始表中的有相同数据的数据
delete from m_user where phone in (select phone from tmp_0524_3);
10.将处理好的数据插回原始表：
INSERT INTO public.m_user
(id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company)
select id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company
from tmp_0524_3;
方向三：如何减少无效数据
在并发系统中应该增加锁对数据进行插入，避免重复的插入数据。
方向四：实际案例举例
案例：
最近遇到了pg数据库表中的大量数据重复了，需要删除其中的一条。一条条删除显然不切合实际，还是需要通过计算来删除。
实施步骤：
1.对原表进行备份
2.使用一个表结构完全一样的临时表对原始表进行接收。
create table tmp_0524_1 as select * from public.m_user_bak20230524;
3.找出重复的数据：
drop table tmp_0524_2;
create table tmp_0524_2 as
select * from (
select phone ,count(1) cn from tmp_0524_1 where phone is not null group by phone ) t where t.cn > 1 ;4.用第二个临时表接收数据：
drop table tmp_0524_3_1;
create table tmp_0524_3_1 as
select t.* from tmp_0524_1 t where phone in (select phone from tmp_0524_2 ) and opt_user is null;5.建一个表结构一样的空表，多增加一个iid字段
create table tmp_0524_3 as select '' iid , t.* from tmp_0524_3_1 where 1 = 2;
6.给空表增加一个iid字段，建立一个字段自增
CREATE SEQUENCE tmp_0524_3_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
alter table tmp_0524_3 alter column iid set default nextval('tmp_0524_3_id_seq');7.插入数据：
INSERT INTO public.tmp_0524_3
(id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company)
select id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company
from tmp_0524_3_1;8.删除相同数据中的一条：
DELETE FROM tmp_0524_3
WHERE iid
NOT IN (
SELECT max(iid)
FROM tmp_0524_3
GROUP BY phone
);9.删除原始表中的有相同数据的数据
delete from m_user where phone in (select phone from tmp_0524_3);
10.将处理好的数据插回原始表：
INSERT INTO public.m_user
(id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company)
select id, user_id, phone, user_name, id_card, wx_account, state, app_id, app_name, head_icon, sex, sex_name, reg_time, user_state, user_state_name, open_id, opt_user, birthday_type, birthday, org_name, last_trading_time, org_code, union_id, orig_phone, update_time, belong_code, belong_name, data_type, client_num, client_name, country_code, inviter, channel_id, channel_name, company
from tmp_0524_3;
