diff --git a/apps/backend/agent/solution_maker.py b/apps/backend/agent/solution_maker.py index 866e1385b..e437b93da 100644 --- a/apps/backend/agent/solution_maker.py +++ b/apps/backend/agent/solution_maker.py @@ -33,12 +33,16 @@ def __init__( name: str, text: str, description: str, + child_dir: str = None, + always_download: bool = False, show_description: bool = True, ): self.name = name self.text = text self.description = description + self.child_dir = child_dir self.show_description = show_description + self.always_download = always_download class ExecutionSolutionStep: @@ -641,24 +645,36 @@ def _make(self) -> ExecutionSolution: ], ) - # 3. 执行安装命令 - download_cmd: str = ( - f"{self.dest_dir}curl.exe {self.get_agent_tools_url(self.script_file_name)} " - f"-o {self.dest_dir}{self.script_file_name} -sSfg" + dependencies_step.contents.append( + ExecutionSolutionStepContent( + name="setup_agent.bat", + text=f"{self.get_agent_tools_url(self.script_file_name)}", + description="Install Scripts", + child_dir=self.agent_setup_info.agent_tools_relative_dir, + # 在云区域场景下需要实时更新 + always_download=True, + show_description=False, + ) ) - download_cmd = self.adjust_cmd_proxy_config(download_cmd) + + # 3. 执行安装命令 + # download_cmd: str = ( + # f"{self.dest_dir}curl.exe {self.get_agent_tools_url(self.script_file_name)} " + # f"-o {self.dest_dir}{self.script_file_name} -sSfg" + # ) + # download_cmd = self.adjust_cmd_proxy_config(download_cmd) run_cmd: str = f"{self.dest_dir}{self.script_file_name} {' '.join(self.get_run_cmd_base_params())}" run_cmds_step: ExecutionSolutionStep = ExecutionSolutionStep( step_type=constants.CommonExecutionSolutionStepType.COMMANDS.value, description=str(_("执行{setup_type_alias}命令").format(setup_type_alias=self.get_setup_type_alias())), contents=[ - ExecutionSolutionStepContent( - name="download_cmd", - text=download_cmd, - description=str(_("下载{setup_type_alias}脚本").format(setup_type_alias=self.get_setup_type_alias())), - show_description=False, - ), + # ExecutionSolutionStepContent( + # name="download_cmd", + # text=download_cmd, + # description=str(_("下载{setup_type_alias}脚本").format(setup_type_alias=self.get_setup_type_alias())), + # show_description=False, + # ), ExecutionSolutionStepContent( name="run_cmd", text=run_cmd, diff --git a/apps/backend/components/collections/agent_new/install.py b/apps/backend/components/collections/agent_new/install.py index 11d72cf72..9229d7246 100644 --- a/apps/backend/components/collections/agent_new/install.py +++ b/apps/backend/components/collections/agent_new/install.py @@ -109,7 +109,12 @@ def handle_lan_windows_sub_inst(self, install_sub_inst_objs: List[InstallSubInst run_commands: List[str] = [] for solution_step in execution_solution.steps[1:]: if solution_step.type == constants.CommonExecutionSolutionStepType.DEPENDENCIES.value: - dependencies.extend([content.name for content in solution_step.contents]) + dependencies.extend( + [ + f"{content.child_dir}/{content.name}" if content.child_dir else content.name + for content in solution_step.contents + ] + ) else: run_commands.extend([content.text for content in solution_step.contents]) diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index 3da7f9536..9ddd5060f 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -357,13 +357,16 @@ def test_batch_solution(self): run_cmd_param_extract={"token": r"(.*) -c (.*?) -s"}, ) - self.assertEqual(constants.AgentWindowsDependencies.list_member_values(), solution_parse_result["dependencies"]) + self.assertEqual( + constants.AgentWindowsDependencies.list_member_values() + ["setup_agent.bat"], + solution_parse_result["dependencies"], + ) self.assertEqual( solution_parse_result["cmds"], [ f"mkdir {installation_tool.dest_dir}", - f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" - f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg", + # f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" + # f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg", f"{installation_tool.dest_dir}setup_agent.bat" f" -O 48668 -E 58925 -A 58625 -V 58930 -B 10020 -S 60020 -Z 60030 -K 10030" f' -e "127.0.0.1" -a "127.0.0.1" -k "127.0.0.1"' @@ -383,6 +386,57 @@ async def connect(self): super().start_patch() +class InstallAgent2WindowsTestCase(InstallWindowsTestCase): + def adjust_db(self): + sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] + sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.save(update_fields=["config"]) + + def structure_common_inputs(self): + inputs = super().structure_common_inputs() + inputs["meta"] = {"GSE_VERSION": GseVersion.V2.value} + return inputs + + def test_batch_solution(self): + host = models.Host.objects.get(bk_host_id=self.obj_factory.bk_host_ids[0]) + agent_step_adapter: AgentStepAdapter = AgentStepAdapter( + self.obj_factory.sub_step_objs[0], + gse_version=GseVersion.V2.value, + ) + installation_tool = gen_commands( + agent_step_adapter.get_setup_info(), + host, + mock_data_utils.JOB_TASK_PIPELINE_ID, + is_uninstall=False, + sub_inst_id=0, + ) + solution_parse_result: Dict[str, Any] = self.execution_solution_parser( + installation_tool=installation_tool, + solution_type=constants.CommonExecutionSolutionType.BATCH.value, + run_cmd_param_extract={"token": r"(.*) -c (.*?) -s"}, + ) + + self.assertEqual( + constants.AgentWindowsDependencies.list_member_values() + ["setup_agent.bat"], + solution_parse_result["dependencies"], + ) + self.assertEqual( + solution_parse_result["cmds"], + [ + f"mkdir {installation_tool.dest_dir}", + # f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" + # f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg", + f"{installation_tool.dest_dir}setup_agent.bat" + f" -O 48668 -E 58925 -A 58625 -V 58930 -B 10020 -S 60020 -Z 60030 -K 10030" + f' -e "127.0.0.1" -a "127.0.0.1" -k "127.0.0.1"' + f" -l http://127.0.0.1/download -r http://127.0.0.1/backend" + f" -i 0 -I {host.inner_ip} -T C:\\tmp\\ -p c:\\gse" + f" -c {solution_parse_result['params']['token']} -s {mock_data_utils.JOB_TASK_PIPELINE_ID}" + f" -N SERVER -n gse_agent -t 2.0.0", + ], + ) + + class InstallLinuxPagentTestCase(InstallBaseTestCase): NODE_TYPE = constants.NodeType.PAGENT CLOUD_ID = 1 @@ -531,14 +585,17 @@ def test_target_host_batch_solution(self): run_cmd_param_extract={"token": r"(.*) -c (.*?) -s"}, ) - self.assertEqual(constants.AgentWindowsDependencies.list_member_values(), solution_parse_result["dependencies"]) + self.assertEqual( + constants.AgentWindowsDependencies.list_member_values() + ["setup_agent.bat"], + solution_parse_result["dependencies"], + ) self.assertEqual( solution_parse_result["cmds"], [ f"mkdir {installation_tool.dest_dir}", - f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" - f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg" - f" -x http://1.1.1.1:{settings.BK_NODEMAN_NGINX_PROXY_PASS_PORT}", + # f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" + # f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg" + # f" -x http://1.1.1.1:{settings.BK_NODEMAN_NGINX_PROXY_PASS_PORT}", f"{installation_tool.dest_dir}setup_agent.bat" f" -O 48668 -E 58925 -A 58625 -V 58930 -B 10020 -S 60020 -Z 60030 -K 10030" f' -e "1.1.1.1" -a "1.1.1.1" -k "1.1.1.1"' @@ -684,6 +741,12 @@ class LinuxAgent2InstallTestCase(InstallBaseTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.save() + + def structure_common_inputs(self): + inputs = super().structure_common_inputs() + inputs["meta"] = {"GSE_VERSION": GseVersion.V2.value} + return inputs def test_shell_solution(self): host = models.Host.objects.get(bk_host_id=self.obj_factory.bk_host_ids[0]) @@ -756,14 +819,17 @@ def test_batch_solution(self): run_cmd_param_extract={"token": r"(.*) -c (.*?) -s"}, ) - self.assertEqual(constants.AgentWindowsDependencies.list_member_values(), solution_parse_result["dependencies"]) + self.assertEqual( + constants.AgentWindowsDependencies.list_member_values() + ["setup_agent.bat"], + solution_parse_result["dependencies"], + ) self.assertEqual( solution_parse_result["cmds"], [ f"mkdir {installation_tool.dest_dir}", script_hook_objs[0].script_info_obj.oneline, - f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" - f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg", + # f"{installation_tool.dest_dir}curl.exe http://127.0.0.1/download/setup_agent.bat" + # f" -o {installation_tool.dest_dir}setup_agent.bat -sSfg", f"{installation_tool.dest_dir}setup_agent.bat" f" -O 48668 -E 58925 -A 58625 -V 58930 -B 10020 -S 60020 -Z 60030 -K 10030" f' -e "127.0.0.1" -a "127.0.0.1" -k "127.0.0.1"' diff --git a/script_tools/setup_pagent.py b/script_tools/setup_pagent.py index c07fc7a7f..ab3d5faf4 100644 --- a/script_tools/setup_pagent.py +++ b/script_tools/setup_pagent.py @@ -250,13 +250,16 @@ def execute_batch_solution( for step in execution_solution["steps"]: for content in step["contents"]: if step["type"] == "dependencies": - localpath = os.path.join(args.download_path, content["name"]) + download_path: str = args.download_path + if content.get("child_dir"): + download_path = os.path.join(download_path, content["child_dir"]) + localpath = os.path.join(download_path, content["name"]) # 文件不存在，从下载源同步 - if not os.path.exists(localpath): + if not os.path.exists(localpath) or content.get("always_download"): report_log( "execute_batch_solution", f"file -> {content['name']} not exists, sync from {content['text']}" ) - download_file(content["text"], args.download_path) + download_file(content["text"], download_path) # 构造文件推送命令 cmd: str = "put {localpath} {tmp_dir}".format(localpath=localpath, tmp_dir=tmp_dir) @@ -360,10 +363,20 @@ def json_parser(json_file: str) -> List: def download_file(url: str, dest_dir: str): """get files via http""" try: + # 创建下载目录 + os.makedirs(dest_dir, exist_ok=True) local_filename = url.split("/")[-1] # NOTE the stream=True parameter below local_file = os.path.join(dest_dir, local_filename) + # 如果修改时间临近，跳过下载，避免多个 setup 脚本文件互相覆盖 + mtimestamp: float = os.path.getmtime(local_file) + if time.time() - mtimestamp < 10: + report_log( + "download_file", f"File download skipped due to sync time approaching, mtimestamp -> {mtimestamp}" + ) + return + r = requests.get(url, stream=True) r.raise_for_status() diff --git a/script_tools/setup_pagent2.py b/script_tools/setup_pagent2.py index 321e3f1d5..7e5acc53e 100644 --- a/script_tools/setup_pagent2.py +++ b/script_tools/setup_pagent2.py @@ -267,14 +267,16 @@ def execute_batch_solution( for step in execution_solution["steps"]: for content in step["contents"]: if step["type"] == "dependencies": - - localpath = os.path.join(args.download_path, content["name"]) + download_path: str = args.download_path + if content.get("child_dir"): + download_path = os.path.join(download_path, content["child_dir"]) + localpath = os.path.join(download_path, content["name"]) # 文件不存在，从下载源同步 - if not os.path.exists(localpath): + if not os.path.exists(localpath) or content.get("always_download"): logger.logging( "execute_batch_solution", f"file -> {content['name']} not exists, sync from {content['text']}" ) - download_file(content["text"], args.download_path) + download_file(content["text"], download_path) # 构造文件推送命令 cmd: str = "put {localpath} {tmp_dir}".format(localpath=localpath, tmp_dir=tmp_dir) @@ -325,7 +327,7 @@ def execute_shell_solution( cmd: str = content["text"] # 根据用户名判断是否采用sudo - if account not in ["root", "Administrator", "administrator"]: + if account not in ["root", "Administrator", "administrator"] and not cmd.startswith("sudo"): cmd = "sudo %s" % cmd if content["name"] == "run_cmd": @@ -407,10 +409,20 @@ def json_parser(json_file: str) -> List: def download_file(url: str, dest_dir: str): """get files via http""" try: + # 创建下载目录 + os.makedirs(dest_dir, exist_ok=True) local_filename = url.split("/")[-1] # NOTE the stream=True parameter below local_file = os.path.join(dest_dir, local_filename) + # 如果修改时间临近，跳过下载，避免多个 setup 脚本文件互相覆盖 + mtimestamp: float = os.path.getmtime(local_file) + if time.time() - mtimestamp < 10: + logger.logging( + "download_file", f"File download skipped due to sync time approaching, mtimestamp -> {mtimestamp}" + ) + return + r = requests.get(url, stream=True) r.raise_for_status()