Skip to content

Commit 61b3f20

Browse files
zhou9584zhou9584
authored
Feature: support remote control android device from portal (#679)
<!-- Please provide brief information about the PR, what it contains & its purpose, new behaviors after the change. And let us know here if you need any help: https://github.com/microsoft/HydraLab/issues/new --> ## Description supported action: click, long-click, swipe ![image](https://github.com/user-attachments/assets/4f1d8f54-65f5-4e6c-9868-63f76c0a3858) <!-- A few words to explain your changes --> ### Linked GitHub issue ID: # ## Pull Request Checklist <!-- Put an x in the boxes that apply. This is simply a reminder of what we are going to look for before merging your code. --> - [ ] Tests for the changes have been added (for bug fixes / features) - [x] Code compiles correctly with all tests are passed. - [x] I've read the [contributing guide](https://github.com/microsoft/HydraLab/blob/main/CONTRIBUTING.md#making-changes-to-the-code) and followed the recommended practices. - [ ] [Wikis](https://github.com/microsoft/HydraLab/wiki) or [README](https://github.com/microsoft/HydraLab/blob/main/README.md) have been reviewed and added / updated if needed (for bug fixes / features) ### Does this introduce a breaking change? *If this introduces a breaking change for Hydra Lab users, please describe the impact and migration path.* - [ ] Yes - [x] No ## How you tested it *Please make sure the change is tested, you can test it by adding UTs, do local test and share the screenshots, etc.* Please check the type of change your PR introduces: - [ ] Bugfix - [x] Feature - [ ] Technical design - [ ] Build related changes - [ ] Refactoring (no functional changes, no api changes) - [ ] Code style update (formatting, renaming) or Documentation content changes - [ ] Other (please describe): ### Feature UI screenshots or Technical design diagrams *If this is a relatively large or complex change, kick it off by drawing the tech design with PlantUML and explaining why you chose the solution you did and what alternatives you considered, etc...*
1 parent efcf865 commit 61b3f20

File tree

17 files changed

+565
-194
lines changed

17 files changed

+565
-194
lines changed

‎agent/src/main/java/com/microsoft/hydralab/agent/service/AgentWebSocketClientService.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.microsoft.hydralab.common.entity.common.AgentUpdateTask;
1313
import com.microsoft.hydralab.common.entity.common.AgentUser;
1414
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
15+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
1516
import com.microsoft.hydralab.common.entity.common.Message;
1617
import com.microsoft.hydralab.common.entity.common.Task;
1718
import com.microsoft.hydralab.common.entity.common.TestRun;
@@ -164,6 +165,13 @@ public void onMessage(Message message) {
164165
}
165166
response = handleTestTaskRun(message);
166167
break;
168+
case Const.Path.DEVICE_OPERATION:
169+
if (!(message.getBody() instanceof DeviceOperation)) {
170+
break;
171+
}
172+
DeviceOperation deviceOperation = (DeviceOperation) message.getBody();
173+
deviceControlService.operateDevice(deviceOperation);
174+
break;
167175
default:
168176
break;
169177
}

‎agent/src/main/java/com/microsoft/hydralab/agent/service/DeviceControlService.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.microsoft.hydralab.common.entity.agent.MobileDevice;
99
import com.microsoft.hydralab.common.entity.common.AgentUser;
1010
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
11+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
1112
import com.microsoft.hydralab.common.entity.common.Message;
1213
import com.microsoft.hydralab.common.management.AgentManagementService;
1314
import com.microsoft.hydralab.common.management.device.DeviceType;
@@ -165,4 +166,16 @@ public void rebootDevices(DeviceType deviceType) {
165166
}
166167
});
167168
}
169+
170+
public void operateDevice(DeviceOperation deviceOperation) {
171+
Set<DeviceInfo> allActiveConnectedDevice = agentManagementService.getActiveDeviceList(log);
172+
List<DeviceInfo> devices = allActiveConnectedDevice.stream()
173+
.filter(adbDeviceInfo -> deviceOperation.getDeviceSerial().equals(adbDeviceInfo.getSerialNum()))
174+
.collect(Collectors.toList());
175+
if (devices.size() != 1) {
176+
throw new RuntimeException("Device " + deviceOperation.getDeviceSerial() + " not connected!");
177+
}
178+
DeviceInfo device = devices.get(0);
179+
deviceDriverManager.execDeviceOperation(device, deviceOperation, log);
180+
}
168181
}

‎center/src/main/java/com/microsoft/hydralab/center/controller/DeviceManageController.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,17 @@
1313
import com.microsoft.hydralab.common.entity.center.DeviceGroup;
1414
import com.microsoft.hydralab.common.entity.center.SysUser;
1515
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
16+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
1617
import com.microsoft.hydralab.common.util.Const;
1718
import lombok.extern.slf4j.Slf4j;
19+
import org.apache.commons.lang3.StringUtils;
1820
import org.springframework.http.HttpStatus;
1921
import org.springframework.http.MediaType;
2022
import org.springframework.security.access.prepost.PreAuthorize;
2123
import org.springframework.security.core.annotation.CurrentSecurityContext;
2224
import org.springframework.web.bind.annotation.GetMapping;
2325
import org.springframework.web.bind.annotation.PostMapping;
26+
import org.springframework.web.bind.annotation.RequestBody;
2427
import org.springframework.web.bind.annotation.RequestParam;
2528
import org.springframework.web.bind.annotation.RestController;
2629

@@ -161,4 +164,48 @@ public Result updateDeviceScope(@CurrentSecurityContext SysUser requestor,
161164

162165
return Result.ok("Saved success!");
163166
}
167+
168+
// API used to operate the device
169+
@PostMapping(value = "/api/device/operate", produces = MediaType.APPLICATION_JSON_VALUE)
170+
public Result operateDevice(@CurrentSecurityContext SysUser requestor,
171+
@RequestBody DeviceOperation operation) {
172+
try {
173+
if (!deviceAgentManagementService.checkDeviceAuthorization(requestor, operation.getDeviceSerial())) {
174+
return Result.error(HttpStatus.UNAUTHORIZED.value(), "Authentication failed");
175+
}
176+
if (operation.getOperationType() == null) {
177+
return Result.error(HttpStatus.BAD_REQUEST.value(), "Invalid operation type");
178+
}
179+
switch (operation.getOperationType()) {
180+
case TAP:
181+
if(StringUtils.isEmpty(operation.getFromPositionX())|| StringUtils.isEmpty(operation.getFromPositionY())){
182+
return Result.error(HttpStatus.BAD_REQUEST.value(), "Invalid tap position");
183+
}
184+
break;
185+
case LONG_TAP:
186+
if (StringUtils.isEmpty(operation.getFromPositionX()) || StringUtils.isEmpty(operation.getFromPositionY())) {
187+
return Result.error(HttpStatus.BAD_REQUEST.value(), "Invalid long tap position");
188+
}
189+
break;
190+
case SWIPE:
191+
if (StringUtils.isEmpty(operation.getFromPositionX()) || StringUtils.isEmpty(operation.getFromPositionY())
192+
|| StringUtils.isEmpty(operation.getToPositionX()) || StringUtils.isEmpty(operation.getToPositionY())) {
193+
return Result.error(HttpStatus.BAD_REQUEST.value(), "Invalid swipe position");
194+
}
195+
break;
196+
case REBOOT:
197+
break;
198+
case WAKEUP:
199+
break;
200+
default:
201+
return Result.error(HttpStatus.BAD_REQUEST.value(), "Invalid operation type");
202+
}
203+
deviceAgentManagementService.operateDevice(operation);
204+
} catch (IllegalArgumentException e) {
205+
return Result.error(HttpStatus.BAD_REQUEST.value(), e);
206+
} catch (Exception e) {
207+
return Result.error(HttpStatus.INTERNAL_SERVER_ERROR.value(), e);
208+
}
209+
return Result.ok("Operate success!");
210+
}
164211
}

‎center/src/main/java/com/microsoft/hydralab/center/service/DeviceAgentManagementService.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.microsoft.hydralab.common.entity.common.AgentUser;
2323
import com.microsoft.hydralab.common.entity.common.AnalysisTask;
2424
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
25+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
2526
import com.microsoft.hydralab.common.entity.common.Message;
2627
import com.microsoft.hydralab.common.entity.common.StatisticData;
2728
import com.microsoft.hydralab.common.entity.common.StorageFileInfo;
@@ -1240,6 +1241,21 @@ public void recordStatisticData() {
12401241
log.info("Storing current online device number {}.", currentDeviceNum);
12411242
}
12421243

1244+
public void operateDevice(DeviceOperation operation) {
1245+
DeviceInfo deviceInfo = deviceListMap.get(operation.getDeviceSerial());
1246+
if (deviceInfo == null) {
1247+
return;
1248+
}
1249+
AgentSessionInfo agentSessionInfo = getAgentSessionInfoByAgentId(deviceInfo.getAgentId());
1250+
if (agentSessionInfo == null) {
1251+
return;
1252+
}
1253+
Message message = new Message();
1254+
message.setPath(Const.Path.DEVICE_OPERATION);
1255+
message.setBody(operation);
1256+
sendMessageToSession(agentSessionInfo.session, message);
1257+
}
1258+
12431259
static class AgentSessionInfo {
12441260
Session session;
12451261
AgentUser agentUser;

‎common/src/main/java/com/microsoft/hydralab/common/entity/common/DeviceOperation.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
package com.microsoft.hydralab.common.entity.common;
5+
6+
import lombok.Data;
7+
8+
/**
9+
* @author zhoule
10+
* @date 03/06/2025
11+
*/
12+
13+
@Data
14+
public class DeviceOperation {
15+
String deviceSerial;
16+
OperationType operationType;
17+
String fromPositionX;
18+
String fromPositionY;
19+
String toPositionX;
20+
String toPositionY;
21+
22+
public enum OperationType {
23+
SWIPE,
24+
TAP,
25+
REBOOT,
26+
WAKEUP,
27+
LONG_TAP,
28+
}
29+
}
30+

‎common/src/main/java/com/microsoft/hydralab/common/management/device/DeviceDriver.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.microsoft.hydralab.common.entity.common.AgentUser;
44
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
5+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
56
import com.microsoft.hydralab.common.entity.common.TestRun;
67
import com.microsoft.hydralab.common.entity.common.TestTask;
78
import com.microsoft.hydralab.common.logger.LogCollector;
@@ -104,4 +105,6 @@ boolean grantProjectionAndBatteryPermission(@NotNull DeviceInfo deviceInfo,
104105
void rebootDeviceAsync(DeviceInfo deviceInfo, Logger logger);
105106

106107
void rebootDeviceIfNeeded(DeviceInfo deviceInfo, Logger logger);
108+
109+
void execDeviceOperation(DeviceInfo deviceInfo, DeviceOperation operation, Logger logger);
107110
}

‎common/src/main/java/com/microsoft/hydralab/common/management/device/impl/AbstractDeviceDriver.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,31 @@ public File getScreenShot(@NotNull DeviceInfo deviceInfo, @Nullable Logger logge
8585
return screenShotImageFile;
8686
}
8787

88-
;
88+
public File getScreenShotNoWakeUp(@NotNull DeviceInfo deviceInfo, @Nullable Logger logger) {
89+
File screenShotImageFile = deviceInfo.getScreenshotImageFile();
90+
if (screenShotImageFile == null) {
91+
screenShotImageFile = new File(agentManagementService.getScreenshotDir(), deviceInfo.getName() + "-" + deviceInfo.getSerialNum() + ".jpg");
92+
deviceInfo.setScreenshotImageFile(screenShotImageFile);
93+
String imageRelPath = screenShotImageFile.getAbsolutePath().replace(new File(agentManagementService.getDeviceStoragePath()).getAbsolutePath(), "");
94+
imageRelPath = agentManagementService.getDeviceFolderUrlPrefix() + imageRelPath.replace(File.separator, "/");
95+
deviceInfo.setImageRelPath(imageRelPath);
96+
}
97+
deviceInfo.setScreenshotUpdateTimeMilli(System.currentTimeMillis());
98+
try {
99+
screenCapture(deviceInfo, screenShotImageFile.getAbsolutePath(), logger);
100+
} catch (Exception e) {
101+
classLogger.error("Screen capture failed for device: {}", deviceInfo, e);
102+
}
103+
StorageFileInfo fileInfo = new StorageFileInfo(screenShotImageFile,
104+
"device/screenshots/" + screenShotImageFile.getName(), StorageFileInfo.FileType.SCREENSHOT, EntityType.SCREENSHOT);
105+
String fileDownloadUrl = agentManagementService.getStorageServiceClientProxy().upload(screenShotImageFile, fileInfo).getBlobUrl();
106+
if (org.apache.commons.lang3.StringUtils.isBlank(fileDownloadUrl)) {
107+
classLogger.warn("Screenshot download url is empty for device {}", deviceInfo.getName());
108+
} else {
109+
deviceInfo.setScreenshotImageUrl(fileDownloadUrl);
110+
}
111+
return screenShotImageFile;
112+
}
89113

90114
public File getScreenShotWithStrategy(@NotNull DeviceInfo deviceInfo, @Nullable Logger logger,
91115
@NotNull AgentUser.BatteryStrategy batteryStrategy) {

‎common/src/main/java/com/microsoft/hydralab/common/management/device/impl/AndroidDeviceDriver.java‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.microsoft.hydralab.common.entity.agent.EnvCapability;
1414
import com.microsoft.hydralab.common.entity.agent.EnvCapabilityRequirement;
1515
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
16+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
1617
import com.microsoft.hydralab.common.entity.common.TestRun;
1718
import com.microsoft.hydralab.common.entity.common.TestTask;
1819
import com.microsoft.hydralab.common.logger.MultiLineNoCancelLoggingReceiver;
@@ -161,6 +162,42 @@ public void init() {
161162
}
162163
}
163164

165+
@Override
166+
public void execDeviceOperation(DeviceInfo deviceInfo, DeviceOperation operation, Logger logger) {
167+
String command = "";
168+
int sleepTime = 500;
169+
switch (operation.getOperationType()) {
170+
case TAP:
171+
command = String.format("input tap %s %s", operation.getFromPositionX(), operation.getFromPositionY());
172+
break;
173+
case LONG_TAP:
174+
command = String.format("input swipe %s %s %s %s 1000", operation.getFromPositionX(),
175+
operation.getFromPositionY(), operation.getFromPositionX(), operation.getFromPositionY());
176+
sleepTime = 500;
177+
break;
178+
case SWIPE:
179+
command = String.format("input swipe %s %s %s %s", operation.getFromPositionX(),
180+
operation.getFromPositionY(), operation.getToPositionX(), operation.getToPositionY());
181+
break;
182+
case REBOOT:
183+
command = "reboot";
184+
break;
185+
case WAKEUP:
186+
command = "input keyevent 82";
187+
break;
188+
default:
189+
logger.error("Invalid operation type");
190+
return;
191+
}
192+
try {
193+
adbOperateUtil.execOnDevice(deviceInfo, command, new MultiLineNoCancelLoggingReceiver(logger), logger);
194+
ThreadUtils.safeSleep(sleepTime);
195+
getScreenShotNoWakeUp(deviceInfo, logger);
196+
} catch (Exception e) {
197+
logger.error(e.getMessage(), e);
198+
}
199+
}
200+
164201
@Override
165202
public List<EnvCapabilityRequirement> getEnvCapabilityRequirements() {
166203
return List.of(new EnvCapabilityRequirement(EnvCapability.CapabilityKeyword.adb, MAJOR_ADB_VERSION, MINOR_ADB_VERSION));
@@ -378,7 +415,7 @@ public void pullFileFromDevice(@NotNull DeviceInfo deviceInfo, @NotNull String p
378415

379416
@Override
380417
public ScreenRecorder getScreenRecorder(DeviceInfo deviceInfo, File folder, Logger logger) {
381-
if (PhoneAppScreenRecorder.RECORD_PACKAGE_NAME.equals(deviceInfo.getRunningTaskPackageName())) {
418+
if (RECORD_PACKAGE_NAME.equals(deviceInfo.getRunningTaskPackageName())) {
382419
return new ADBScreenRecorder(this, this.adbOperateUtil, deviceInfo, logger, folder);
383420
}
384421
return new PhoneAppScreenRecorder(this, this.adbOperateUtil, deviceInfo, folder, logger);

‎common/src/main/java/com/microsoft/hydralab/common/management/device/impl/DeviceDriverManager.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.microsoft.hydralab.common.entity.common.AgentUser;
77
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
8+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
89
import com.microsoft.hydralab.common.entity.common.TestRun;
910
import com.microsoft.hydralab.common.entity.common.TestTask;
1011
import com.microsoft.hydralab.common.logger.LogCollector;
@@ -237,4 +238,9 @@ public void rebootDeviceAsync(DeviceInfo deviceInfo, Logger logger) {
237238
public void rebootDeviceIfNeeded(DeviceInfo deviceInfo, Logger logger) {
238239
getDeviceDriver(deviceInfo.getType()).rebootDeviceIfNeeded(deviceInfo, logger);
239240
}
241+
242+
@Override
243+
public void execDeviceOperation(DeviceInfo deviceInfo, DeviceOperation operation, Logger logger) {
244+
getDeviceDriver(deviceInfo.getType()).execDeviceOperation(deviceInfo, operation, logger);
245+
}
240246
}

‎common/src/main/java/com/microsoft/hydralab/common/management/device/impl/IOSDeviceDriver.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import com.microsoft.hydralab.common.entity.agent.EnvCapability;
1010
import com.microsoft.hydralab.common.entity.agent.EnvCapabilityRequirement;
1111
import com.microsoft.hydralab.common.entity.common.DeviceInfo;
12+
import com.microsoft.hydralab.common.entity.common.DeviceOperation;
1213
import com.microsoft.hydralab.common.entity.common.TestRun;
1314
import com.microsoft.hydralab.common.logger.LogCollector;
1415
import com.microsoft.hydralab.common.logger.impl.IOSLogCollector;
@@ -70,6 +71,11 @@ public void init() {
7071
}
7172
}
7273

74+
@Override
75+
public void execDeviceOperation(DeviceInfo deviceInfo, DeviceOperation operation, Logger logger) {
76+
classLogger.info("Nothing Implemented for iOS in " + currentMethodName());
77+
}
78+
7379
@Override
7480
public List<EnvCapabilityRequirement> getEnvCapabilityRequirements() {
7581
// todo XCCode / iTunes

0 commit comments

Comments
 (0)