1#!/usr/bin/env python3 2# ===----------------------------------------------------------------------===## 3# 4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5# See https://llvm.org/LICENSE.txt for license information. 6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7# 8# ===----------------------------------------------------------------------===## 9 10"""adb_run.py is a utility for running a libc++ test program via adb. 11""" 12 13import argparse 14import hashlib 15import os 16import re 17import shlex 18import socket 19import subprocess 20import sys 21from typing import List, Tuple 22 23 24# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file. 25REMOTE_BASE_DIR = "/data/local/tmp/adb_run" 26 27g_job_limit_socket = None 28g_verbose = False 29 30 31def run_adb_sync_command(command: List[str]) -> None: 32 """Run an adb command and discard the output, unless the command fails. If 33 the command fails, dump the output instead, and exit the script with 34 failure. 35 """ 36 if g_verbose: 37 sys.stderr.write(f"running: {shlex.join(command)}\n") 38 proc = subprocess.run(command, universal_newlines=True, 39 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, 40 stderr=subprocess.STDOUT, encoding="utf-8") 41 if proc.returncode != 0: 42 # adb's stdout (e.g. for adb push) should normally be discarded, but 43 # on failure, it should be shown. Print it to stderr because it's 44 # unrelated to the test program's stdout output. A common error caught 45 # here is "No space left on device". 46 sys.stderr.write(f"{proc.stdout}\n" 47 f"error: adb command exited with {proc.returncode}: " 48 f"{shlex.join(command)}\n") 49 sys.exit(proc.returncode) 50 51 52def sync_test_dir(local_dir: str, remote_dir: str) -> None: 53 """Sync the libc++ test directory on the host to the remote device.""" 54 55 # Optimization: The typical libc++ test directory has only a single 56 # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is 57 # normally necessary because we don't know if the target directory already 58 # exists on the device. 59 local_files = os.listdir(local_dir) 60 if len(local_files) == 1: 61 local_file = os.path.join(local_dir, local_files[0]) 62 remote_file = os.path.join(remote_dir, local_files[0]) 63 if not os.path.islink(local_file) and os.path.isfile(local_file): 64 run_adb_sync_command(["adb", "push", "--sync", local_file, 65 remote_file]) 66 return 67 68 assert os.path.basename(local_dir) == os.path.basename(remote_dir) 69 run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir]) 70 run_adb_sync_command(["adb", "push", "--sync", local_dir, 71 os.path.dirname(remote_dir)]) 72 73 74def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str: 75 components = [] 76 for arg in env_args: 77 k, v = arg.split("=", 1) 78 components.append(f"export {k}={shlex.quote(v)}; ") 79 for k, v in prepend_path_args: 80 components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ") 81 return "".join(components) 82 83 84def run_command(args: argparse.Namespace) -> int: 85 local_dir = args.execdir 86 assert local_dir.startswith("/") 87 assert not local_dir.endswith("/") 88 89 # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using 90 # a hash of local_dir so that concurrent adb_run invocations don't create 91 # the same intermediate parent directory. At least `adb push` has trouble 92 # with concurrent mkdir syscalls on common parent directories. (Somehow 93 # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug, 94 # b/289311228.) 95 local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest() 96 remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}" 97 sync_test_dir(local_dir, remote_dir) 98 99 adb_shell_command = ( 100 # Set the environment early so that PATH can be overridden. Overriding 101 # PATH is useful for: 102 # - Replacing older shell utilities with toybox (e.g. on old devices). 103 # - Adding a `bash` command that delegates to `sh` (mksh). 104 f"{build_env_arg(args.env, args.prepend_path_env)}" 105 106 # Set a high oom_score_adj so that, if the test program uses too much 107 # memory, it is killed before anything else on the device. The default 108 # oom_score_adj is -1000, so a test using too much memory typically 109 # crashes the device. 110 "echo 1000 >/proc/self/oom_score_adj; " 111 112 # If we're running as root, switch to the shell user. The libc++ 113 # filesystem tests require running without root permissions. Some x86 114 # emulator devices (before Android N) do not have a working `adb unroot` 115 # and always run as root. Non-debug builds typically lack `su` and only 116 # run as the shell user. 117 # 118 # Some libc++ tests create temporary files in the working directory, 119 # which might be owned by root. Before switching to shell, make the 120 # cwd writable (and readable+executable) to every user. 121 # 122 # N.B.: 123 # - Avoid "id -u" because it wasn't supported until Android M. 124 # - The `env` and `which` commands were also added in Android M. 125 # - Starting in Android M, su from root->shell resets PATH, so we need 126 # to modify it again in the new environment. 127 # - Avoid chmod's "a+rwx" syntax because it's not supported until 128 # Android N. 129 # - Defining this function allows specifying the arguments to the test 130 # program (i.e. "$@") only once. 131 "run_without_root() {" 132 " chmod 777 .;" 133 " case \"$(id)\" in" 134 " *\"uid=0(root)\"*)" 135 " if command -v env >/dev/null; then" 136 " su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";" 137 " else" 138 " su shell \"$@\";" 139 " fi;;" 140 " *) \"$@\";;" 141 " esac;" 142 "}; " 143 ) 144 145 # Older versions of Bionic limit the length of argv[0] to 127 bytes 146 # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this 147 # limit. Changing the working directory works around this limit. The limit 148 # is increased to 4095 (PATH_MAX-1) in Android M (API 23). 149 command_line = [arg.replace(local_dir + "/", "./") for arg in args.command] 150 151 # Prior to the adb feature "shell_v2" (added in Android N), `adb shell` 152 # always created a pty: 153 # - This merged stdout and stderr together. 154 # - The pty converts LF to CRLF. 155 # - The exit code of the shell command wasn't propagated. 156 # Work around all three limitations, unless "shell_v2" is present. 157 proc = subprocess.run(["adb", "features"], check=True, 158 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, 159 encoding="utf-8") 160 adb_features = set(proc.stdout.strip().split()) 161 has_shell_v2 = "shell_v2" in adb_features 162 if has_shell_v2: 163 adb_shell_command += ( 164 f"cd {remote_dir} && run_without_root {shlex.join(command_line)}" 165 ) 166 else: 167 adb_shell_command += ( 168 f"{{" 169 f" stdout=$(" 170 f" cd {remote_dir} && run_without_root {shlex.join(command_line)};" 171 f" echo -n __libcxx_adb_exit__=$?" 172 f" ); " 173 f"}} 2>&1; " 174 f"echo -n __libcxx_adb_stdout__\"$stdout\"" 175 ) 176 177 adb_command_line = ["adb", "shell", adb_shell_command] 178 if g_verbose: 179 sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n") 180 181 if has_shell_v2: 182 proc = subprocess.run(adb_command_line, shell=False, check=False, 183 encoding="utf-8") 184 return proc.returncode 185 else: 186 proc = subprocess.run(adb_command_line, shell=False, check=False, 187 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 188 encoding="utf-8") 189 # The old `adb shell` mode used a pty, which converted LF to CRLF. 190 # Convert it back. 191 output = proc.stdout.replace("\r\n", "\n") 192 193 if proc.returncode: 194 sys.stderr.write(f"error: adb failed:\n" 195 f" command: {shlex.join(adb_command_line)}\n" 196 f" output: {output}\n") 197 return proc.returncode 198 199 match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$", 200 output, re.DOTALL) 201 if not match: 202 sys.stderr.write(f"error: could not parse adb output:\n" 203 f" command: {shlex.join(adb_command_line)}\n" 204 f" output: {output}\n") 205 return 1 206 207 sys.stderr.write(match.group(1)) 208 sys.stdout.write(match.group(2)) 209 return int(match.group(3)) 210 211 212def connect_to_job_limiter_server(sock_addr: str) -> None: 213 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 214 215 try: 216 sock.connect(sock_addr) 217 except (FileNotFoundError, ConnectionRefusedError) as e: 218 # Copying-and-pasting an adb_run.py command-line from a lit test failure 219 # is likely to fail because the socket no longer exists (or is 220 # inactive), so just give a warning. 221 sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n") 222 return 223 224 # The connect call can succeed before the server has called accept, because 225 # of the listen backlog, so wait for the server to send a byte. 226 sock.recv(1) 227 228 # Keep the socket open until this process ends, then let the OS close the 229 # connection automatically. 230 global g_job_limit_socket 231 g_job_limit_socket = sock 232 233 234def main() -> int: 235 """Main function (pylint wants this docstring).""" 236 parser = argparse.ArgumentParser() 237 parser.add_argument("--execdir", type=str, required=True) 238 parser.add_argument("--env", type=str, required=False, action="append", 239 default=[], metavar="NAME=VALUE") 240 parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False, 241 action="append", default=[], 242 metavar=("NAME", "PATH")) 243 parser.add_argument("--job-limit-socket") 244 parser.add_argument("--verbose", "-v", default=False, action="store_true") 245 parser.add_argument("command", nargs=argparse.ONE_OR_MORE) 246 args = parser.parse_args() 247 248 global g_verbose 249 g_verbose = args.verbose 250 if args.job_limit_socket is not None: 251 connect_to_job_limiter_server(args.job_limit_socket) 252 return run_command(args) 253 254 255if __name__ == '__main__': 256 sys.exit(main()) 257