Bugfix: after the center restarts, the device status may be overwritten (#690)

<!-- Please provide brief information about the PR, what it contains &
its purpose, new behaviors after the change. And let us know here if you
need any help: https://github.com/microsoft/HydraLab/issues/new -->

## Description

<!-- A few words to explain your changes -->

### Linked GitHub issue ID: #  

## Pull Request Checklist
<!-- Put an x in the boxes that apply. This is simply a reminder of what
we are going to look for before merging your code. -->

- [ ] Tests for the changes have been added (for bug fixes / features)
- [x] Code compiles correctly with all tests are passed.
- [x] I've read the [contributing
guide](https://github.com/microsoft/HydraLab/blob/main/CONTRIBUTING.md#making-changes-to-the-code)
and followed the recommended practices.
- [ ] [Wikis](https://github.com/microsoft/HydraLab/wiki) or
[README](https://github.com/microsoft/HydraLab/blob/main/README.md) have
been reviewed and added / updated if needed (for bug fixes / features)

### Does this introduce a breaking change?
*If this introduces a breaking change for Hydra Lab users, please
describe the impact and migration path.*

- [ ] Yes
- [x] No

## How you tested it
*Please make sure the change is tested, you can test it by adding UTs,
do local test and share the screenshots, etc.*


Please check the type of change your PR introduces:
- [x] Bugfix
- [ ] Feature
- [ ] Technical design
- [ ] Build related changes
- [ ] Refactoring (no functional changes, no api changes)
- [ ] Code style update (formatting, renaming) or Documentation content
changes
- [ ] Other (please describe): 

### Feature UI screenshots or Technical design diagrams
*If this is a relatively large or complex change, kick it off by drawing
the tech design with PlantUML and explaining why you chose the solution
you did and what alternatives you considered, etc...*

Author: zhou9584

agent/src/main/java/com/microsoft/hydralab/agent/socket/AgentWebSocketClient.java

@@ -22,6 +22,7 @@ public class AgentWebSocketClient extends WebSocketClient {
 
     private boolean connectionActive = false;
     private int reconnectTime = 0;
+    private int violatedReconnectTime = 0;
 
     public AgentWebSocketClient(URI serverUri, AgentWebSocketClientService agentWebSocketClientService) {
         super(serverUri);
@@ -54,6 +55,7 @@ public void onMessage(ByteBuffer bytes) {
         agentWebSocketClientService.onMessage(message);
         if (Const.Path.DEVICE_LIST.equals(message.getPath())) {
             reconnectTime = 0;
+            violatedReconnectTime = 0;
         }
     }
 
@@ -65,13 +67,30 @@ public void onMessage(String message) {
 
     @Override
     public void onClose(int code, String reason, boolean remote) {
-        log.error("onClose {}, {}, {}", code, reason, remote);
+        log.error("onClose {}, {}, {}, {}", code, reason, remote, reconnectTime);
         reconnectTime++;
         connectionActive = false;
-        // if the connection is closed by server with 1008,1003, exit the agent
-        if (code == CloseReason.CloseCodes.CANNOT_ACCEPT.getCode() || code == CloseReason.CloseCodes.VIOLATED_POLICY.getCode()) {
+        // if the connection is closed by server with 1003, exit the agent
+        if (code == CloseReason.CloseCodes.CANNOT_ACCEPT.getCode()) {
             System.exit(code);
         }
+
+        // if the connection is closed by server with 1008, wait and retry
+        if (code == CloseReason.CloseCodes.VIOLATED_POLICY.getCode()) {
+            violatedReconnectTime++;
+            if(violatedReconnectTime > 5) {
+                log.error("onClose, code: {}, reason: {}, remote: {}, reconnectTime: {}, {}", code, reason, remote, reconnectTime, violatedReconnectTime);
+                System.exit(code);
+            } else {
+                // wait for 10 seconds and then retry
+                try {
+                    log.info("onClose, code: {}, reason: {}, remote: {}, reconnectTime: {}, {} sleep 10 seconds", code, reason, remote, reconnectTime, violatedReconnectTime);
+                    Thread.sleep(10000);
+                } catch (InterruptedException e) {
+                    log.error("onClose, sleep error", e);
+                }
+            }
+        }
     }
 
     @Override

center/src/main/java/com/microsoft/hydralab/center/service/DeviceAgentManagementService.java

@@ -21,16 +21,16 @@
 import com.microsoft.hydralab.common.entity.common.AgentUpdateTask;
 import com.microsoft.hydralab.common.entity.common.AgentUser;
 import com.microsoft.hydralab.common.entity.common.AnalysisTask;
+import com.microsoft.hydralab.common.entity.common.BlockedDeviceInfo;
 import com.microsoft.hydralab.common.entity.common.DeviceInfo;
+import com.microsoft.hydralab.common.entity.common.DeviceOperation;
 import com.microsoft.hydralab.common.entity.common.Message;
 import com.microsoft.hydralab.common.entity.common.StatisticData;
 import com.microsoft.hydralab.common.entity.common.StorageFileInfo;
 import com.microsoft.hydralab.common.entity.common.Task;
 import com.microsoft.hydralab.common.entity.common.TestRun;
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.entity.common.TestTaskSpec;
-import com.microsoft.hydralab.common.entity.common.BlockedDeviceInfo;
-import com.microsoft.hydralab.common.entity.common.DeviceOperation;
 import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.management.device.DeviceType;
 import com.microsoft.hydralab.common.repository.BlockedDeviceInfoRepository;
@@ -68,11 +68,11 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.Iterator;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -431,10 +431,10 @@ public void updateDeviceGroup(List<DeviceInfo> agentDeviceInfos, String agentId)
             DeviceInfo centerDevice = deviceListMap.get(agentDeviceInfo.getSerialNum());
             // if the status saved in Center is testing, the value will not be covered
             if (centerDevice != null && centerDevice.isTesting()) {
-                log.warn("Center status: {}, Agent status: {}, status should be synced to CENTER's value when TESTING.", centerDevice.getStatus(), agentDeviceInfo.getStatus());
+                log.info("Center status: {}, Agent status: {}, status should be synced to CENTER's value when TESTING.", centerDevice.getStatus(), agentDeviceInfo.getStatus());
                 agentDeviceInfo.setStatus(DeviceInfo.TESTING);
-            } else if (agentDeviceInfo.isTesting()) {
-                log.warn("Test on the device is canceled, status of device in AGENT should be reset to ONLINE, otherwise TESTING would never be covered by agent");
+            } else if (agentDeviceInfo.isTesting() && !checkTaskIsRunning(agentDeviceInfo)) {
+                log.info("Test on the device is canceled, status of device in AGENT should be reset to ONLINE, otherwise TESTING would never be covered by agent");
                 agentDeviceInfo.setStatus(DeviceInfo.ONLINE);
             }
 
@@ -448,6 +448,27 @@ public void updateDeviceGroup(List<DeviceInfo> agentDeviceInfos, String agentId)
         }
     }
 
+    private boolean checkTaskIsRunning(DeviceInfo deviceInfo) {
+        String taskId = deviceInfo.getRunningTaskId();
+        if (taskId == null) {
+            return false;
+        }
+        Task task = testDataService.getTaskDetail(taskId);
+        // check if the task is running
+        if (task == null || !Task.TaskStatus.RUNNING.equals(task.getStatus())) {
+            return false;
+        }
+
+        // check the timeout of the task
+        if (task.getTimeOutSecond() > 0) {
+            long timeout = task.getStartDate().getTime() + TimeUnit.SECONDS.toMillis(task.getTimeOutSecond());
+            if (System.currentTimeMillis() > timeout) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     //update Device Status : start task,complete task,device offline,device online
     public void updateDeviceStatus(String serialNum, String status, String testTaskId) {
         DeviceInfo device = deviceListMap.get(serialNum);