Implementation code for timing incremental synchronization of database tables with java code

import java.io.Serializable;
import java.util.List;

/**
* Created by Administrator on 2017/4/19.
* This is based on a single thread to perform synchronization Do not allow multi-threaded execution Multi-threaded execution is too difficult Write to give up! ! ! ! !
* Here we consider the time factor and the non-locking read factor of mysql mvcc.
* We solve the time factor as follows: 1. The time of the main server shall prevail. instead of local time. Fast or slow local time will not affect the synchronization function
* 2. The main server can adjust its time, forward (change the time to a smaller time) or backward (change the time to a larger time), and the program supports it.
* Assume that our synchronization time is 1 hour synchronization, and the maximum timeout time of things is 5 minutes (that is, no thing can exceed 5 minutes). Assuming that the current master server time
* is: 08:00, then the last synchronization time is: 06:55
* 2.1, forward adjustment: suppose to adjust forward 5 hours, then it is 03:00, when we synchronize, get the master server The server's time is X ( 03:00 <= X <= 04:00 ) because our sync interval is 1 hour
* Assuming X=03:56, then min(06:55+01:00,03:56-01:00-00:05)=02:51 Then our synchronization time will be: 02:51, Then it is the correct synchronization time.
* The next time the last synchronization time is considered will be: 02:51
* 2.2. Backward adjustment: Assuming that the backward adjustment is 5 hours, then it is 13:00. When we synchronize, the time to obtain the master server is: Y (13:00 <= Y <= 14:00) Same thing.
* Assuming Y=13:56, then min(06:55+01:00,13:56-01:00-00:05)=07:55 Then our synchronization time will be: 07:55, Then it is the correct synchronization time.
* The next time I think the last synchronization time will be: 13:56-01:00-00:05 = 12:51 At this time, the time has improved a lot (to keep up with the pace of time).
*
* <T> is the id type
*/
public abstract class BaseSyncUserInfoTaskImpl<T extends Serializable> implements ISyncUserInfoTask{

    //The synchronization interval can be slightly larger (larger than the actual interval in the execution of the scheduled task, but do not be smaller than him , equal to the best timed task execution interval)
    private int sync_time_interval_in_milsecond = 10*60*1000;

    //The longest time for transaction processing It is recommended that the synchronization time interval be greater than this time
    private int tx_time_out_in_milsecond = 5*60*1000;

    //The last synchronization time
    private Long up_sync_time = null;
    //this time Synchronization time
    private Long this_sync_time = null;

    /**
     * Execute the overall architecture
     */
    @Override
    public final void sync() {

        //Get the synchronization time and negotiate the synchronization time with the master server
        long nowSyncTime = getSyncTime();

        //Start data synchronization
        syncDatas (nowSyncTime);

        //Synchronizing data only solves the problem of updating and inserting. Here to solve the problem of deleting.
        //Some tables will not have delete operations. Here,
        if(isNeedDel)
            syncDel();

        //This is put at the end for fear that things will be rolled back and the time has not been rolled back, resulting in incorrect up_sync_time in the next synchronization
        updateUpSyncTime();
    }

    //Adding synchronized is only for fear that jvm optimization will lead to statement rearrangement, and this must be updated at the end Time
    protected synchronized void updateUpSyncTime(){
        up_sync_time = this_sync_time;
    }


    private void syncDel() {

        //1. Get the summary MD5 of all id sets locally, and the number of records for remote comparison, if they are equal, do nothing
        CommonKVQuery<String, Integer> abstractAndCount = getLocalIdsAbstractAndCount();
        boolean isMach = isMachMasterServer(abstractAndCount);
        if(isMach)
            return ;
        //2. Paginate the local data according to id and take it to the remote for comparison, if not, take it back and delete it.
        DefaultPageRequest pageRequest = new DefaultPageRequest();
        pageRequest.setLimit(100);
        pageRequest.setPage(1);
        List<T> ids = null;
        List<T> delIds = null;
        do {
            ids = listLocalIds(pageRequest);
            delIds = getNeedDelIdsFromMasterServer(ids);
            deleteLocalByIds(delIds) ;
            pageRequest.setPage(pageRequest.getPage()+1);
        } while (ids.size() == pageRequest.getLimit());
    }

    /**
     * Delete local data by id collection
     * @param delIds
     */
    protected abstract void deleteLocalByIds(List<T> delIds);

    /**
     * Go far to match to find the id collection to be deleted
     * @param ids
     * @return
     */
    protected abstract List<T> getNeedDelIdsFromMasterServer(List ids);

    //Get the local id collection by paging
    protected abstract List<T> listLocalIds(DefaultPageRequest pageRequest);

    //Go to the master server to match the summary and number of records
    protected abstract boolean isMachMasterServer (CommonKVQuery<String,Integer> abstractAndCount);

    //Get the summary of the local id collection and the number of records
    protected abstract CommonKVQuery<String,Integer> getLocalIdsAbstractAndCount() ;

    //Synchronize the data Update the data after the specified time
    protected abstract void syncDatas(long nowSyncTime) ;

    private long getSyncTime(){

        final long masterServerTime = getMasterServiceTime();
        if(up_sync_time == null){

            up_sync_time = getLocalMaxUpdateTime() - sync_time_interval_in_milsecond;

            //If the last update time is null, it means that the system restarts and the first synchronization is performed, then a full synchronization is performed.
// this_sync_time = masterServerTime - sync_time_interval_in_milsecond-tx_time_out_in_milsecond;
// return 0l;

        }

// min(master server time - synchronization interval (1 hour) - maximum transaction timeout (5 minutes), last negotiated time + synchronization interval)
// 5 minutes here I consider the maximum transaction time. It is assumed that the length of time for all things can not exceed 5 minutes.
// Because we often set the update time in the program first, then insert into the database, and then do something else (wasting some time),
// and finally commit the thing. Then according to mvcc mode, non-locking read is a read snapshot. As a result, the update time should have been synchronized in this synchronization, but was not synchronized to
// (invisible), and the next synchronization time is greater than this update time. As a result, updates will be lost. So every time you synchronize, 5 minutes of data will be synchronized.
// Afraid of dropping data in this gap.
        this_sync_time = Math.min(up_sync_time + sync_time_interval_in_milsecond,
                masterServerTime-sync_time_interval_in_milsecond-tx_time_out_in_milsecond) ;
        final long result = Math.max(0,this_sync_time);
        //The value of this synchronization time here is the master server time-synchronization interval-the maximum timeout time of things
        //and discard up_sync_time + The value of sync_time_interval_in_milsecond is to keep the next update up to the master server's time, not too far
        away this_sync_time = masterServerTime - sync_time_interval_in_milsecond-tx_time_out_in_milsecond;
        return result;
    }

    /**
     * Get the maximum update time in the local record
     * @return
     */
    protected abstract long getLocalMaxUpdateTime() ;

    //Get the current time of the master server
    protected abstract long getMasterServiceTime();

    /**
     * Whether the table data needs to be deleted, if it will not be deleted, the deleted data can be reduced to synchronize
     */
    private boolean isNeedDel = false;

    public void setSync_time_interval_in_milsecond(int sync_time_interval_in_milsecond) {
        this.sync_time_interval_in_milsecond = sync_time_interval_in_milsecond;
    }

    public void setTx_time_out_in_milsecond(int tx_time_out_in_milsecond ) {
        this.tx_time_out_in_milsecond = tx_time_out_in_milsecond;
    }

    public void setNeedDel(boolean needDel) {
        isNeedDel = needDel;
    }
}