Eureka 实例注册状态保持 STARTING 的问题排查

import com.sun.btrace.annotations.BTrace;
import com.sun.btrace.annotations.Kind;
import com.sun.btrace.annotations.Location;
import com.sun.btrace.annotations.OnMethod;

import java.util.concurrent.atomic.AtomicBoolean;

import static com.sun.btrace.BTraceUtils.currentThread;
import static com.sun.btrace.BTraceUtils.println;

/**
 * @author Addo.Zhang
 * @date 2019-07-31
 */
@BTrace(unsafe = true)
public class EurekaRequest {
    public static final AtomicBoolean heartbeatThreadRegistrationStarted = new AtomicBoolean(false);
    public static final AtomicBoolean replicatorThreadRegistrationCompleted = new AtomicBoolean(false);
    public static final AtomicBoolean statusUP = new AtomicBoolean(false);

    @OnMethod(clazz = "org.springframework.cloud.netflix.eureka.serviceregistry.EurekaServiceRegistry", location = @Location(value = Kind.LINE, line = 45))
    public static void waitHeartbeatExecution() {
        println(currentThread() + " is waiting heartbeatThreadRegistrationStarted thread executing first");
        while (!heartbeatThreadRegistrationStarted.get()) ;
    }

    @OnMethod(clazz = "org.springframework.cloud.netflix.eureka.serviceregistry.EurekaServiceRegistry", location = @Location(value = Kind.LINE, line = 46))
    public static void markStatusUp() {
        statusUP.set(true);
        println("Heartbeat thread executed and " + currentThread() + " continues procedure to change status to [UP]");
    }

    @OnMethod(clazz = "com.netflix.discovery.converters.EurekaJacksonCodec$InstanceInfoSerializer", location = @Location(value = Kind.LINE, line = 369))
    public static void continueRegistrationExecution() {
        doExecution();
    }

    @OnMethod(clazz = "com.logancloud.forge.discovery.converters.LoganEurekaJacksonCodec$LoganInstanceInfoSerializer", location = @Location(value = Kind.LINE, line = 117))
    public static void continueRegistrationExecution2() {
        doExecution();
    }

    private static void doExecution() {
        println(currentThread() + " started to proceed registration");
        if (Thread.currentThread().getName().contains("HeartbeatExecutor")) {
            heartbeatThreadRegistrationStarted.set(true);
            while (!statusUP.get() || !replicatorThreadRegistrationCompleted.get()) {
            }
            try {
                Thread.sleep(500); //interval for replicator registration request completed.
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        } else if (Thread.currentThread().getName().contains("InstanceInfoReplicator")) {
            replicatorThreadRegistrationCompleted.set(true);
        }
        println(currentThread() + " thread registration completed");
    }
}

public void start() {
    ...
    if (!this.running.get() && this.registration.getNonSecurePort() > 0) {
      //调用EurekaServiceRegistry进行注册
    	this.serviceRegistry.register(this.registration);
    	//发布实例注册的事件
    	this.context.publishEvent(
    			new InstanceRegisteredEvent<>(this, this.registration.getInstanceConfig()));
    	this.running.set(true);
    }
}

public void register(EurekaRegistration reg) {
	maybeInitializeClient(reg);
	if (log.isInfoEnabled()) {
		log.info("Registering application " + reg.getInstanceConfig().getAppname()
				+ " with eureka with status "
				+ reg.getInstanceConfig().getInitialStatus());
	}
	//1
	reg.getApplicationInfoManager()
			.setInstanceStatus(reg.getInstanceConfig().getInitialStatus());
	if (reg.getHealthCheckHandler() != null) {
	   //2
		reg.getEurekaClient().registerHealthCheck(reg.getHealthCheckHandler());
	}
}

statusChangeListener = new ApplicationInfoManager.StatusChangeListener() {
    @Override
    public String getId() {
        return "statusChangeListener";
    }
    @Override
    public void notify(StatusChangeEvent statusChangeEvent) {
        if (InstanceStatus.DOWN == statusChangeEvent.getStatus() ||
                InstanceStatus.DOWN == statusChangeEvent.getPreviousStatus()) {
            // log at warn level if DOWN was involved
            logger.warn("Saw local status change event {}", statusChangeEvent);
        } else {
            logger.info("Saw local status change event {}", statusChangeEvent);
        }
        instanceInfoReplicator.onDemandUpdate();
    }
};

public InstanceInfo.InstanceStatus getStatus(InstanceInfo.InstanceStatus currentStatus) {
    if (null == callback || InstanceInfo.InstanceStatus.STARTING == currentStatus
            || InstanceInfo.InstanceStatus.OUT_OF_SERVICE == currentStatus) { // Do not go to healthcheck handler if the status is starting or OOS.
        return currentStatus;
    }
    return callback.isHealthy() ? InstanceInfo.InstanceStatus.UP : InstanceInfo.InstanceStatus.DOWN;
}

 <instance>
      <instanceId>xp-xtower-webapp-boot-6-txcxb:xp-xtower-webapp-boot:10100</instanceId>
      <hostName>10.128.41.74</hostName>
      <app>XP-XTOWER-WEBAPP-BOOT</app>
      <ipAddr>10.128.41.74</ipAddr>
      <status>STARTING</status>
      <overriddenstatus>UNKNOWN</overriddenstatus>
      <port enabled="true">10100</port>
      <securePort enabled="false">443</securePort>
      <countryId>1</countryId>
      <dataCenterInfo class="com.netflix.appinfo.InstanceInfo$DefaultDataCenterInfo">
        <name>MyOwn</name>
      </dataCenterInfo>
      <leaseInfo>
        <renewalIntervalInSecs>5</renewalIntervalInSecs>
        <durationInSecs>20</durationInSecs>
        <registrationTimestamp>1545039481898</registrationTimestamp>
        <lastRenewalTimestamp>1545950719063</lastRenewalTimestamp>
        <evictionTimestamp>0</evictionTimestamp>
        <serviceUpTimestamp>0</serviceUpTimestamp>
      </leaseInfo>
      <metadata>
        <forge>1.0.0</forge>
        <management.port>10100</management.port>
        <jmx.port>1099</jmx.port>
        <group>innovation</group>
      </metadata>
      <homePageUrl>http://10.128.41.74:10100/</homePageUrl>
      <statusPageUrl></statusPageUrl>
      <healthCheckUrl>http://10.128.41.74:10100/health</healthCheckUrl>
      <vipAddress>xp-xtower-webapp-boot</vipAddress>
      <secureVipAddress>xp-xtower-webapp-boot</secureVipAddress>
      <isCoordinatingDiscoveryServer>false</isCoordinatingDiscoveryServer>
      <lastUpdatedTimestamp>1545039481899</lastUpdatedTimestamp>
      <lastDirtyTimestamp>1545039481813</lastDirtyTimestamp>
      <actionType>ADDED</actionType>
    </instance>

private Response validateDirtyTimestamp(Long lastDirtyTimestamp,
                                        boolean isReplication) {
    InstanceInfo appInfo = registry.getInstanceByAppAndId(app.getName(), id, false);
    if (appInfo != null) {
        if ((lastDirtyTimestamp != null) && (!lastDirtyTimestamp.equals(appInfo.getLastDirtyTimestamp()))) {
            Object[] args = {id, appInfo.getLastDirtyTimestamp(), lastDirtyTimestamp, isReplication};
            if (lastDirtyTimestamp > appInfo.getLastDirtyTimestamp()) {
                logger.debug(
                        "Time to sync, since the last dirty timestamp differs -"
                                + " ReplicationInstance id : {},Registry : {} Incoming: {} Replication: {}",
                        args);
                return Response.status(Status.NOT_FOUND).build();
            } else if (appInfo.getLastDirtyTimestamp() > lastDirtyTimestamp) {
                // In the case of replication, send the current instance info in the registry for the
                // replicating node to sync itself with this one.
                if (isReplication) {
                    logger.debug(
                            "Time to sync, since the last dirty timestamp differs -"
                                    + " ReplicationInstance id : {},Registry : {} Incoming: {} Replication: {}",
                            args);
                    return Response.status(Status.CONFLICT).entity(appInfo).build();
                } else {
                    return Response.ok().build();
                }
            }
        }
    }
    return Response.ok().build();
}