Skip to content

Commit 65cd30f

Browse files
Jiseong-ohJiseong-oh
committed
Using Devicefarm for E2E test
Signed-off-by: jiseong.oh <[email protected]>
1 parent 7892602 commit 65cd30f

File tree

2 files changed

+164
-64
lines changed

2 files changed

+164
-64
lines changed

‎.ci/scripts/setup-samsung-linux-deps.sh‎

Lines changed: 148 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -8,61 +8,172 @@
88

99
set -ex
1010

11+
API_KEY="${SAMSUK_AI_LITECORE_KEY:-}"
12+
if [[ -z "${API_KEY}" ]]; then
13+
echo "ERROR: It didn't set up SAMSUNG_AI_LITECORE_KEY." >&2
14+
exit 1
15+
fi
16+
17+
OS_NAME="Ubuntu 22.04"
18+
LITECORE_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/ai-litecore/download"
19+
DEVICEFARM_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/remotelab/download"
20+
21+
parse_url() {
22+
local json="$1"
23+
if command -v jq >/dev/null 2>&1; then
24+
jq -r '.data // empty' <<<"$json"
25+
else
26+
sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$json"
27+
fi
28+
}
1129

12-
download_ai_lite_core() {
13-
API_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/ai-litecore/download"
14-
API_KEY=$SAMSUNG_AI_LITECORE_KEY
15-
16-
VERSION="0.7"
17-
OS_NAME="Ubuntu 22.04"
18-
OUT_FILE="/tmp/exynos-ai-litecore-v${VERSION}.tar.gz"
19-
TARGET_PATH="/tmp/exynos_ai_lite_core"
20-
21-
mkdir -p ${TARGET_PATH}
22-
# Presigned issue URL
23-
JSON_RESP=$(curl -sS -G \
24-
--location --fail --retry 3 \
30+
download_and_extract() {
31+
local base_url="$1"
32+
local version="$2"
33+
local out_dir="$3"
34+
local out_file="$4"
35+
36+
local resp
37+
resp=$(curl -fsSL -G \
2538
-H "apikey: ${API_KEY}" \
26-
--data-urlencode "version=${VERSION}" \
39+
--data-urlencode "version=${version}" \
2740
--data-urlencode "os=${OS_NAME}" \
28-
"${API_BASE}")
41+
"${base_url}")
42+
43+
local download_url
44+
download_url=$(parse_url "$resp")
45+
if [[ -z "${download_url}" ]]; then
46+
echo "ERROR: It failed to download from ${base_url} ."
47+
echo "Response: $resp" >&2
48+
exit 1
49+
fi
50+
51+
curl -fsSL -L --retry 3 -o "${out_file}" "${download_url}"
52+
echo "Download completed: ${out_file}"
53+
54+
mkdir -p "${out_dir}"
55+
case "${out_file##*.}" in
56+
tar|tgz|gz)
57+
echo "Extracting TAR.GZ..."
58+
tar -C "${out_dir}" --strip-components=1 -xzvf "${out_file}"
59+
;;
2960

30-
DOWNLOAD_URL=$(echo "$JSON_RESP" | sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p')
61+
zip)
62+
echo "Extracting ZIP..."
63+
unzip -q -d "${out_dir}" "${out_file}"
64+
;;
3165

32-
if [[ -z "$DOWNLOAD_URL" ]]; then
33-
echo "Failed to extract download URL"
34-
echo "$JSON_RESP"
66+
*)
3567
exit 1
68+
;;
69+
esac
70+
echo "Extracted to: ${out_dir}"
71+
}
72+
73+
download_ai_lite_core() {
74+
local litecore_version="${1:-0.7}"
75+
local litecore_out="/tmp/exynos-ai-litecore-v${litecore_version}.tar.gz"
76+
local litecore_dir="/tmp/exynos_ai_lite_core"
77+
78+
download_and_extract \
79+
"${LITECORE_BASE}" \
80+
"${litecore_version}" \
81+
"${litecore_dir}" \
82+
"${litecore_out}"
83+
84+
export EXYNOS_AI_LITECORE_ROOT="${litecore_dir}"
85+
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:${EXYNOS_AI_LITECORE_ROOT}/lib/x86_64-linux"
86+
}
87+
88+
install_devicefarm_cli() {
89+
local cli_version="${1:-beta-1.0.6}"
90+
local cli_out="/tmp/devicefarm-cli-v${cli_version}.tar.gz"
91+
local cli_dir="/tmp/devicefarm_cli"
92+
93+
download_and_extract \
94+
"${DEVICEFARM_BASE}" \
95+
"${cli_version}" \
96+
"${cli_dir}" \
97+
"${cli_out}"
98+
99+
export PATH="${PATH}:${cli_dir}"
100+
chmod +x "${cli_dir}/devicefarm-cli"
101+
}
102+
103+
reserve_if_needed() {
104+
local info_output
105+
info_output="$(devicefarm-cli -I)"
106+
107+
local found_count
108+
found_count=$(printf '%s\n' "$info_output" |
109+
awk -F': ' '/Found available reservations/{gsub(/\r/,""); print $NF; exit}')
110+
[[ -z "$found_count" ]] && found_count=0
111+
112+
local THRESHOLD_SECONDS=1800
113+
local any_below_threshold=0
114+
115+
if (( found_count > 0 )); then
116+
while IFS= read -r time_str; do
117+
[[ -z "$time_str" ]] && continue
118+
IFS=: read -r hh mm ss <<< "$time_str"
119+
(( seconds = 10#$hh * 3600 + 10#$mm * 60 + 10#$ss ))
120+
if (( seconds <= THRESHOLD_SECONDS )); then
121+
any_below_threshold=1
122+
break
123+
fi
124+
done < <(printf '%s\n' "$info_output" |
125+
awk '($1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]{1,2}:[0-9]{2}:[0-9]{2}$/){print $2}')
126+
else
127+
any_below_threshold=1
36128
fi
37129

38-
# Download LiteCore
39-
curl -sS -L --fail --retry 3 \
40-
--output "$OUT_FILE" \
41-
"$DOWNLOAD_URL"
130+
if (( any_below_threshold )); then
131+
devicefarm-cli -R
132+
fi
42133

43-
echo "Download done: $OUT_FILE"
134+
local info_after reservation_id max_seconds=0 max_id
44135

136+
info_after="$(devicefarm-cli -I)"
45137

46-
tar -C "${TARGET_PATH}" --strip-components=1 -xzvf "${OUT_FILE}"
138+
reservation_id=$(printf '%s\n' "$info_after" | awk '
139+
($1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]{1,2}:[0-9]{2}:[0-9]{2}$/) {
140+
split($2, t, ":");
141+
seconds = (t[1] * 3600) + (t[2] * 60) + t[3];
142+
if (seconds > max_seconds) { max_seconds = seconds; max_id = $NF }
143+
}
144+
END { if (max_id) print max_id }
145+
')
47146

48-
export EXYNOS_AI_LITECORE_ROOT=${TARGET_PATH}
49-
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${EXYNOS_AI_LITECORE_ROOT}/lib/x86_64-linux
147+
if [[ -n "$reservation_id" ]]; then
148+
devicefarm-cli -C $reservation_id
149+
devicefarm-cli -E "ls /"
150+
else
151+
echo "[WARN] There is no available devices"
152+
fi
50153
}
51154

52155
install_enn_backend() {
53-
NDK_INSTALLATION_DIR=/opt/ndk
54-
rm -rf "${NDK_INSTALLATION_DIR}" && sudo mkdir -p "${NDK_INSTALLATION_DIR}"
55-
ANDROID_NDK_VERSION=r28c
156+
local ndk_dir="/opt/ndk"
157+
local ndk_version="r28c"
158+
159+
if [[ ! -d "${ndk_dir}" ]]; then
160+
sudo mkdir -p "${ndk_dir}"
161+
sudo chown "$(whoami)":"$(whoami)" "${ndk_dir}"
162+
fi
163+
164+
export ANDROID_NDK_ROOT="${ndk_dir}"
165+
echo "NDK will be installed/used at: ${ANDROID_NDK_ROOT}"
56166

57-
# build Exynos backend
58-
export ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT:-/opt/ndk}
59167
bash backends/samsung/build.sh --build all
60-
# set env variable
61-
export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
62-
export PYTHONPATH=${PYTHONPATH:-}:${EXECUTORCH_ROOT}/..
168+
169+
export EXECUTORCH_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
170+
export PYTHONPATH="${PYTHONPATH:-}:${EXECUTORCH_ROOT}/.."
63171
}
64172

65-
AI_LITE_CORE_VERSION=0.7.0
173+
local litecore_ver="0.7"
174+
local devicefarm_ver="beta-1.0.6"
66175

67-
download_ai_lite_core ${AI_LITE_CORE_VERSION}
176+
download_ai_lite_core ${litecore_ver}
177+
install_devicefarm_cli "${devicefarm_ver}"
68178
install_enn_backend
179+
reserve_if_needed

‎backends/samsung/test/utils/runtime_executor.py‎

Lines changed: 16 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def get_runner_path() -> Path:
2121
return Path(git_root) / "build_samsung_android/backends/samsung/enn_executor_runner"
2222

2323

24-
class ADBTestManager:
24+
class EDBTestManager:
2525
def __init__(
2626
self,
2727
pte_file,
@@ -34,11 +34,10 @@ def __init__(
3434
self.artifacts_dir = Path(self.pte_file).parent.absolute()
3535
self.output_folder = f"{self.work_directory}/output"
3636
self.runner = str(get_runner_path())
37+
self.devicefarm = "devicefarm-cli"
3738

38-
def _adb(self, cmd):
39-
cmds = ["adb"]
40-
41-
assert self._is_adb_connected, "Fail to get available device to execute."
39+
def _edb(self, cmd):
40+
cmds = [self.devicefarm]
4241

4342
cmds.extend(cmd)
4443
command = " ".join(cmds)
@@ -49,27 +48,27 @@ def _adb(self, cmd):
4948
if result.returncode != 0:
5049
logging.info(result.stdout.decode("utf-8").strip())
5150
logging.error(result.stderr.decode("utf-8").strip())
52-
raise RuntimeError("adb command execute failed")
51+
raise RuntimeError("edb(Exynos device bridge) command execute failed")
5352

5453
def push(self):
55-
self._adb(["shell", f"rm -rf {self.work_directory}"])
56-
self._adb(["shell", f"mkdir -p {self.work_directory}"])
57-
self._adb(["push", self.pte_file, self.work_directory])
58-
self._adb(["push", self.runner, self.work_directory])
54+
self._edb(["-E", f"rm -rf {self.work_directory}"])
55+
self._edb(["-E", f"mkdir -p {self.work_directory}"])
56+
self._edb(["-U", self.pte_file, self.work_directory])
57+
self._edb(["-U", self.runner, self.work_directory])
5958

6059
for input_file in self.input_files:
6160
input_file_path = os.path.join(self.artifacts_dir, input_file)
6261
if Path(input_file).name == input_file and os.path.isfile(input_file_path):
6362
# default search the same level directory with pte
64-
self._adb(["push", input_file_path, self.work_directory])
63+
self._edb(["-U", input_file_path, self.work_directory])
6564
elif os.path.isfile(input_file):
66-
self._adb(["push", input_file, self.work_directory])
65+
self._edb(["-U", input_file, self.work_directory])
6766
else:
6867
raise FileNotFoundError(f"Invalid input file path: {input_file}")
6968

7069
def execute(self):
71-
self._adb(["shell", f"rm -rf {self.output_folder}"])
72-
self._adb(["shell", f"mkdir -p {self.output_folder}"])
70+
self._edb(["-E", f"rm -rf {self.output_folder}"])
71+
self._edb(["-E", f"mkdir -p {self.output_folder}"])
7372
# run the delegation
7473
input_files_list = " ".join([os.path.basename(x) for x in self.input_files])
7574
enn_executor_runner_args = " ".join(
@@ -86,20 +85,10 @@ def execute(self):
8685
]
8786
)
8887

89-
self._adb(["shell", f"{enn_executor_runner_cmd}"])
88+
self._edb(["-E", f"{enn_executor_runner_cmd}"])
9089

9190
def pull(self, output_path):
92-
self._adb(["pull", "-a", self.output_folder, output_path])
93-
94-
@staticmethod
95-
def _is_adb_connected():
96-
try:
97-
output = subprocess.check_output(["adb", "devices"])
98-
devices = output.decode("utf-8").splitlines()[1:]
99-
return [device.split()[0] for device in devices if device.strip() != ""]
100-
except subprocess.CAlledProcessError:
101-
return False
102-
91+
self._edb(["-D", self.output_folder, output_path])
10392

10493
class RuntimeExecutor:
10594
def __init__(self, executorch_program, inputs):
@@ -109,7 +98,7 @@ def __init__(self, executorch_program, inputs):
10998
def run_on_device(self) -> Tuple[torch.Tensor]:
11099
with tempfile.TemporaryDirectory() as tmp_dir:
111100
pte_filename, input_files = self._save_model_and_inputs(tmp_dir)
112-
test_manager = ADBTestManager(
101+
test_manager = EDBTestManager(
113102
pte_file=os.path.join(tmp_dir, pte_filename),
114103
work_directory="/data/local/tmp/enn-executorch-test",
115104
input_files=input_files,

0 commit comments

Comments
 (0)