diff --git a/OMPython/OMCSession.py b/OMPython/OMCSession.py index 79f8d16b..0d295e86 100644 --- a/OMPython/OMCSession.py +++ b/OMPython/OMCSession.py @@ -20,7 +20,7 @@ import sys import tempfile import time -from typing import Any, Optional, Tuple +from typing import Any, Optional, Tuple, Type import uuid import warnings @@ -291,6 +291,8 @@ class OMPathCompatibilityWindows(pathlib.WindowsPath, OMPathCompatibility): OMCPath = OMPathCompatibility OMPathRunnerABC = OMPathCompatibility OMPathRunnerLocal = OMPathCompatibility + OMPathRunnerBash = OMPathCompatibility + else: class OMPathABC(pathlib.PurePosixPath, metaclass=abc.ABCMeta): """ @@ -544,10 +546,10 @@ def _path(self) -> pathlib.Path: class _OMPathRunnerLocal(OMPathRunnerABC): """ - Implementation of OMPathBase which does not use the session data at all. Thus, this implementation can run + Implementation of OMPathABC which does not use the session data at all. Thus, this implementation can run locally without any usage of OMC. - This class is based on OMPathBase and, therefore, on pathlib.PurePosixPath. This is working well, but it is not + This class is based on OMPathABC and, therefore, on pathlib.PurePosixPath. This is working well, but it is not the correct implementation on Windows systems. To get a valid Windows representation of the path, use the conversion via pathlib.Path(.as_posix()). """ @@ -564,7 +566,7 @@ def is_dir(self) -> bool: """ return self._path().is_dir() - def is_absolute(self): + def is_absolute(self) -> bool: """ Check if the path is an absolute path. """ @@ -580,9 +582,12 @@ def write_text(self, data: str): """ Write text data to the file represented by this path. """ + if not isinstance(data, str): + raise TypeError(f"data must be str, not {data.__class__.__name__}") + return self._path().write_text(data=data, encoding='utf-8') - def mkdir(self, parents: bool = True, exist_ok: bool = False): + def mkdir(self, parents: bool = True, exist_ok: bool = False) -> None: """ Create a directory at the path represented by this class. @@ -590,21 +595,21 @@ def mkdir(self, parents: bool = True, exist_ok: bool = False): Python < 3.12. In this case, pathlib.Path is used directly and this option ensures, that missing parent directories are also created. """ - return self._path().mkdir(parents=parents, exist_ok=exist_ok) + self._path().mkdir(parents=parents, exist_ok=exist_ok) - def cwd(self): + def cwd(self) -> OMPathABC: """ - Returns the current working directory as an OMPathBase object. + Returns the current working directory as an OMPathABC object. """ - return self._path().cwd() + return type(self)(self._path().cwd().as_posix(), session=self._session) def unlink(self, missing_ok: bool = False) -> None: """ Unlink (delete) the file or directory represented by this path. """ - return self._path().unlink(missing_ok=missing_ok) + self._path().unlink(missing_ok=missing_ok) - def resolve(self, strict: bool = False): + def resolve(self, strict: bool = False) -> OMPathABC: """ Resolve the path to an absolute path. This is done based on available OMC functions. """ @@ -621,8 +626,177 @@ def size(self) -> int: path = self._path() return path.stat().st_size + class _OMPathRunnerBash(OMPathRunnerABC): + """ + Implementation of OMPathABC which does not use the session data at all. Thus, this implementation can run + locally without any usage of OMC. The special case of this class is the usage of POSIX bash to run all the + commands. Thus, it can be used in WSL or docker. + + This class is based on OMPathABC and, therefore, on pathlib.PurePosixPath. This is working well, but it is not + the correct implementation on Windows systems. To get a valid Windows representation of the path, use the + conversion via pathlib.Path(.as_posix()). + """ + + def is_file(self) -> bool: + """ + Check if the path is a regular file. + """ + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'test -f "{self.as_posix()}"'] + + try: + subprocess.run(cmdl, check=True) + return True + except subprocess.CalledProcessError: + return False + + def is_dir(self) -> bool: + """ + Check if the path is a directory. + """ + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'test -d "{self.as_posix()}"'] + + try: + subprocess.run(cmdl, check=True) + return True + except subprocess.CalledProcessError: + return False + + def is_absolute(self) -> bool: + """ + Check if the path is an absolute path. + """ + + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'case "{self.as_posix()}" in /*) exit 0;; *) exit 1;; esac'] + + try: + subprocess.check_call(cmdl) + return True + except subprocess.CalledProcessError: + return False + + def read_text(self) -> str: + """ + Read the content of the file represented by this path as text. + """ + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'cat "{self.as_posix()}"'] + + result = subprocess.run(cmdl, capture_output=True, check=True) + if result.returncode == 0: + return result.stdout.decode('utf-8') + raise FileNotFoundError(f"Cannot read file: {self.as_posix()}") + + def write_text(self, data: str) -> int: + """ + Write text data to the file represented by this path. + """ + if not isinstance(data, str): + raise TypeError(f"data must be str, not {data.__class__.__name__}") + + data_escape = self._session.escape_str(data) + + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'printf %s "{data_escape}" > "{self.as_posix()}"'] + + try: + subprocess.run(cmdl, check=True) + return len(data) + except subprocess.CalledProcessError as exc: + raise IOError(f"Error writing data to file {self.as_posix()}!") from exc + + def mkdir(self, parents: bool = True, exist_ok: bool = False) -> None: + """ + Create a directory at the path represented by this class. + + The argument parents with default value True exists to ensure compatibility with the fallback solution for + Python < 3.12. In this case, pathlib.Path is used directly and this option ensures, that missing parent + directories are also created. + """ + + if self.is_file(): + raise OSError(f"The given path {self.as_posix()} exists and is a file!") + if self.is_dir() and not exist_ok: + raise OSError(f"The given path {self.as_posix()} exists and is a directory!") + if not parents and not self.parent.is_dir(): + raise FileNotFoundError(f"Parent directory of {self.as_posix()} does not exists!") + + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'mkdir -p "{self.as_posix()}"'] + + try: + subprocess.run(cmdl, check=True) + except subprocess.CalledProcessError as exc: + raise OMCSessionException(f"Error on directory creation for {self.as_posix()}!") from exc + + def cwd(self) -> OMPathABC: + """ + Returns the current working directory as an OMPathABC object. + """ + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', 'pwd'] + + result = subprocess.run(cmdl, capture_output=True, text=True, check=True) + if result.returncode == 0: + return type(self)(result.stdout.strip(), session=self._session) + raise OSError("Can not get current work directory ...") + + def unlink(self, missing_ok: bool = False) -> None: + """ + Unlink (delete) the file or directory represented by this path. + """ + + if not self.is_file(): + raise OSError(f"Can not unlink a directory: {self.as_posix()}!") + + if not self.is_file(): + return + + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'rm "{self.as_posix()}"'] + + try: + subprocess.run(cmdl, check=True) + except subprocess.CalledProcessError as exc: + raise OSError(f"Cannot unlink file {self.as_posix()}: {exc}") from exc + + def resolve(self, strict: bool = False) -> OMPathABC: + """ + Resolve the path to an absolute path. This is done based on available OMC functions. + """ + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'readlink -f "{self.as_posix()}"'] + + result = subprocess.run(cmdl, capture_output=True, text=True, check=True) + if result.returncode == 0: + return type(self)(result.stdout.strip(), session=self._session) + raise FileNotFoundError(f"Cannot resolve path: {self.as_posix()}") + + def size(self) -> int: + """ + Get the size of the file in bytes - implementation baseon on pathlib.Path. + """ + if not self.is_file(): + raise OMCSessionException(f"Path {self.as_posix()} is not a file!") + + cmdl = self.get_session().get_cmd_prefix() + cmdl += ['bash', '-c', f'stat -c %s "{self.as_posix()}"'] + + result = subprocess.run(cmdl, capture_output=True, text=True, check=True) + stdout = result.stdout.strip() + if result.returncode == 0: + try: + return int(stdout) + except ValueError as exc: + raise OSError(f"Invalid return value for filesize ({self.as_posix()}): {stdout}") from exc + else: + raise OSError(f"Cannot get size for file {self.as_posix()}") + OMCPath = _OMCPath OMPathRunnerLocal = _OMPathRunnerLocal + OMPathRunnerBash = _OMPathRunnerBash class ModelExecutionException(Exception): @@ -1258,8 +1432,9 @@ class OMCSessionPort(OMCSessionABC): def __init__( self, omc_port: str, + timeout: float = 10.0, ) -> None: - super().__init__() + super().__init__(timeout=timeout) self._omc_port = omc_port @@ -1422,7 +1597,9 @@ class OMCSessionDockerABC(OMCSessionABC, metaclass=abc.ABCMeta): def __init__( self, - timeout: float = 10.00, + timeout: float = 10.0, + docker: Optional[str] = None, + dockerContainer: Optional[str] = None, dockerExtraArgs: Optional[list] = None, dockerOpenModelicaPath: str | os.PathLike = "omc", dockerNetwork: Optional[str] = None, @@ -1436,11 +1613,21 @@ def __init__( self._docker_extra_args = dockerExtraArgs self._docker_open_modelica_path = pathlib.PurePosixPath(dockerOpenModelicaPath) self._docker_network = dockerNetwork + self._docker_container_id: str + self._docker_process: Optional[DockerPopen] - self._interactive_port = port + # start up omc executable in docker container waiting for the ZMQ connection + self._omc_process, self._docker_process, self._docker_container_id = self._docker_omc_start( + docker_image=docker, + docker_cid=dockerContainer, + omc_port=port, + ) + # connect to the running omc instance using ZMQ + self._omc_port = self._omc_port_get(docker_cid=self._docker_container_id) + if port is not None and not self._omc_port.endswith(f":{port}"): + raise OMCSessionException(f"Port mismatch: {self._omc_port} is not using the defined port {port}!") - self._docker_container_id: Optional[str] = None - self._docker_process: Optional[DockerPopen] = None + self._cmd_prefix = self.model_execution_prefix() def _docker_process_get(self, docker_cid: str) -> Optional[DockerPopen]: if sys.platform == 'win32': @@ -1466,6 +1653,15 @@ def _docker_process_get(self, docker_cid: str) -> Optional[DockerPopen]: return docker_process + @abc.abstractmethod + def _docker_omc_start( + self, + docker_image: Optional[str] = None, + docker_cid: Optional[str] = None, + omc_port: Optional[int] = None, + ) -> Tuple[subprocess.Popen, DockerPopen, str]: + pass + @staticmethod def _getuid() -> int: """ @@ -1477,11 +1673,14 @@ def _getuid() -> int: # Windows, hence the type: ignore comment. return 1000 if sys.platform == 'win32' else os.getuid() # type: ignore - def _omc_port_get(self) -> str: + def _omc_port_get( + self, + docker_cid: str, + ) -> str: port = None - if not isinstance(self._docker_container_id, str): - raise OMCSessionException(f"Invalid docker container ID: {self._docker_container_id}") + if not isinstance(docker_cid, str): + raise OMCSessionException(f"Invalid docker container ID: {docker_cid}") # See if the omc server is running loop = self._timeout_loop(timestep=0.1) @@ -1490,7 +1689,7 @@ def _omc_port_get(self) -> str: if omc_portfile_path is not None: try: output = subprocess.check_output(args=["docker", - "exec", self._docker_container_id, + "exec", docker_cid, "cat", omc_portfile_path.as_posix()], stderr=subprocess.DEVNULL) port = output.decode().strip() @@ -1513,7 +1712,10 @@ def get_server_address(self) -> Optional[str]: """ if self._docker_network == "separate" and isinstance(self._docker_container_id, str): output = subprocess.check_output(["docker", "inspect", self._docker_container_id]).decode().strip() - return json.loads(output)[0]["NetworkSettings"]["IPAddress"] + address = json.loads(output)[0]["NetworkSettings"]["IPAddress"] + if not isinstance(address, str): + raise OMCSessionException(f"Invalid docker server address: {address}!") + return address return None @@ -1560,27 +1762,16 @@ def __init__( super().__init__( timeout=timeout, + docker=docker, dockerExtraArgs=dockerExtraArgs, dockerOpenModelicaPath=dockerOpenModelicaPath, dockerNetwork=dockerNetwork, port=port, ) - if docker is None: - raise OMCSessionException("Argument docker must be set!") - - self._docker = docker - - # start up omc executable in docker container waiting for the ZMQ connection - self._omc_process, self._docker_process, self._docker_container_id = self._docker_omc_start() - # connect to the running omc instance using ZMQ - self._omc_port = self._omc_port_get() - def __del__(self) -> None: - super().__del__() - - if isinstance(self._docker_process, DockerPopen): + if hasattr(self, '_docker_process') and isinstance(self._docker_process, DockerPopen): try: self._docker_process.wait(timeout=2.0) except subprocess.TimeoutExpired: @@ -1592,29 +1783,37 @@ def __del__(self) -> None: finally: self._docker_process = None + super().__del__() + def _docker_omc_cmd( self, - omc_path_and_args_list: list[str], + docker_image: str, docker_cid_file: pathlib.Path, + omc_path_and_args_list: list[str], + omc_port: Optional[int | str] = None, ) -> list: """ Define the command that will be called by the subprocess module. """ + extra_flags = [] if sys.platform == "win32": extra_flags = ["-d=zmqDangerousAcceptConnectionsFromAnywhere"] - if not self._interactive_port: - raise OMCSessionException("docker on Windows requires knowing which port to connect to - " + if not self._omc_port: + raise OMCSessionException("Docker on Windows requires knowing which port to connect to - " "please set the interactivePort argument") + port: Optional[int] = None + if isinstance(omc_port, str): + port = int(omc_port) + elif isinstance(omc_port, int): + port = omc_port + if sys.platform == "win32": - if isinstance(self._interactive_port, str): - port = int(self._interactive_port) - elif isinstance(self._interactive_port, int): - port = self._interactive_port - else: - raise OMCSessionException("Missing or invalid interactive port!") + if not isinstance(port, int): + raise OMCSessionException("OMC on Windows needs the interactive port - " + f"missing or invalid value: {repr(omc_port)}!") docker_network_str = ["-p", f"127.0.0.1:{port}:{port}"] elif self._docker_network == "host" or self._docker_network is None: docker_network_str = ["--network=host"] @@ -1625,8 +1824,8 @@ def _docker_omc_cmd( raise OMCSessionException(f'dockerNetwork was set to {self._docker_network}, ' 'but only \"host\" or \"separate\" is allowed') - if isinstance(self._interactive_port, int): - extra_flags = extra_flags + [f"--interactivePort={int(self._interactive_port)}"] + if isinstance(port, int): + extra_flags = extra_flags + [f"--interactivePort={port}"] omc_command = ([ "docker", "run", @@ -1636,22 +1835,33 @@ def _docker_omc_cmd( ] + self._docker_extra_args + docker_network_str - + [self._docker, self._docker_open_modelica_path.as_posix()] + + [docker_image, self._docker_open_modelica_path.as_posix()] + omc_path_and_args_list + extra_flags) return omc_command - def _docker_omc_start(self) -> Tuple[subprocess.Popen, DockerPopen, str]: + def _docker_omc_start( + self, + docker_image: Optional[str] = None, + docker_cid: Optional[str] = None, + omc_port: Optional[int] = None, + ) -> Tuple[subprocess.Popen, DockerPopen, str]: + + if not isinstance(docker_image, str): + raise OMCSessionException("A docker image name must be provided!") + my_env = os.environ.copy() docker_cid_file = self._temp_dir / (self._omc_filebase + ".docker.cid") omc_command = self._docker_omc_cmd( + docker_image=docker_image, + docker_cid_file=docker_cid_file, omc_path_and_args_list=["--locale=C", "--interactive=zmq", f"-z={self._random_string}"], - docker_cid_file=docker_cid_file, + omc_port=omc_port, ) omc_process = subprocess.Popen(omc_command, @@ -1662,6 +1872,7 @@ def _docker_omc_start(self) -> Tuple[subprocess.Popen, DockerPopen, str]: if not isinstance(docker_cid_file, pathlib.Path): raise OMCSessionException(f"Invalid content for docker container ID file path: {docker_cid_file}") + # the provided value for docker_cid is not used docker_cid = None loop = self._timeout_loop(timestep=0.1) while next(loop): @@ -1672,10 +1883,12 @@ def _docker_omc_start(self) -> Tuple[subprocess.Popen, DockerPopen, str]: pass if docker_cid is not None: break - else: - logger.error(f"Docker did not start. Log-file says:\n{self.get_log()}") + time.sleep(self._timeout / 40.0) + + if docker_cid is None: raise OMCSessionException(f"Docker did not start (timeout={self._timeout} might be too short " - "especially if you did not docker pull the image before this command).") + "especially if you did not docker pull the image before this command). " + f"Log-file says:\n{self.get_log()}") docker_process = self._docker_process_get(docker_cid=docker_cid) if docker_process is None: @@ -1702,22 +1915,13 @@ def __init__( super().__init__( timeout=timeout, + dockerContainer=dockerContainer, dockerExtraArgs=dockerExtraArgs, dockerOpenModelicaPath=dockerOpenModelicaPath, dockerNetwork=dockerNetwork, port=port, ) - if not isinstance(dockerContainer, str): - raise OMCSessionException("Argument dockerContainer must be set!") - - self._docker_container_id = dockerContainer - - # start up omc executable in docker container waiting for the ZMQ connection - self._omc_process, self._docker_process = self._docker_omc_start() - # connect to the running omc instance using ZMQ - self._omc_port = self._omc_port_get() - def __del__(self) -> None: super().__del__() @@ -1725,7 +1929,12 @@ def __del__(self) -> None: # docker container ID was provided - do NOT kill the docker process! self._docker_process = None - def _docker_omc_cmd(self, omc_path_and_args_list) -> list: + def _docker_omc_cmd( + self, + docker_cid: str, + omc_path_and_args_list: list[str], + omc_port: Optional[int] = None, + ) -> list: """ Define the command that will be called by the subprocess module. """ @@ -1733,33 +1942,44 @@ def _docker_omc_cmd(self, omc_path_and_args_list) -> list: if sys.platform == "win32": extra_flags = ["-d=zmqDangerousAcceptConnectionsFromAnywhere"] - if not self._interactive_port: + if not isinstance(omc_port, int): raise OMCSessionException("Docker on Windows requires knowing which port to connect to - " "Please set the interactivePort argument. Furthermore, the container needs " "to have already manually exposed this port when it was started " "(-p 127.0.0.1:n:n) or you get an error later.") - if isinstance(self._interactive_port, int): - extra_flags = extra_flags + [f"--interactivePort={int(self._interactive_port)}"] + if isinstance(omc_port, int): + extra_flags = extra_flags + [f"--interactivePort={omc_port}"] omc_command = ([ "docker", "exec", "--user", str(self._getuid()), ] + self._docker_extra_args - + [self._docker_container_id, self._docker_open_modelica_path.as_posix()] + + [docker_cid, self._docker_open_modelica_path.as_posix()] + omc_path_and_args_list + extra_flags) return omc_command - def _docker_omc_start(self) -> Tuple[subprocess.Popen, DockerPopen]: + def _docker_omc_start( + self, + docker_image: Optional[str] = None, + docker_cid: Optional[str] = None, + omc_port: Optional[int] = None, + ) -> Tuple[subprocess.Popen, DockerPopen, str]: + + if not isinstance(docker_cid, str): + raise OMCSessionException("A docker container ID must be provided!") + my_env = os.environ.copy() omc_command = self._docker_omc_cmd( + docker_cid=docker_cid, omc_path_and_args_list=["--locale=C", "--interactive=zmq", f"-z={self._random_string}"], + omc_port=omc_port, ) omc_process = subprocess.Popen(omc_command, @@ -1768,14 +1988,14 @@ def _docker_omc_start(self) -> Tuple[subprocess.Popen, DockerPopen]: env=my_env) docker_process = None - if isinstance(self._docker_container_id, str): - docker_process = self._docker_process_get(docker_cid=self._docker_container_id) + if isinstance(docker_cid, str): + docker_process = self._docker_process_get(docker_cid=docker_cid) if docker_process is None: raise OMCSessionException(f"Docker top did not contain omc process {self._random_string} " - f"/ {self._docker_container_id}. Log-file says:\n{self.get_log()}") + f"/ {docker_cid}. Log-file says:\n{self.get_log()}") - return omc_process, docker_process + return omc_process, docker_process, docker_cid class OMCSessionWSL(OMCSessionABC): @@ -1803,6 +2023,8 @@ def __init__( # connect to the running omc instance using ZMQ self._omc_port = self._omc_port_get() + self._cmd_prefix = self.model_execution_prefix() + def model_execution_prefix(self, cwd: Optional[OMPathABC] = None) -> list[str]: """ Helper function which returns a command prefix needed for docker and WSL. It defaults to an empty list. @@ -1826,7 +2048,8 @@ def _omc_process_get(self) -> subprocess.Popen: self._wsl_omc, "--locale=C", "--interactive=zmq", - f"-z={self._random_string}"] + f"-z={self._random_string}", + ] omc_process = subprocess.Popen(omc_command, stdout=self._omc_loghandle, @@ -1870,13 +2093,26 @@ class OMSessionRunner(OMSessionABC): def __init__( self, - timeout: float = 10.00, - version: str = "1.27.0" + timeout: float = 10.0, + version: str = "1.27.0", + ompath_runner: Type[OMPathRunnerABC] = OMPathRunnerLocal, + cmd_prefix: Optional[list[str]] = None, + model_execution_local: bool = True, ) -> None: super().__init__(timeout=timeout) - self.model_execution_local = True self._version = version + if not issubclass(ompath_runner, OMPathRunnerABC): + raise OMCSessionException(f"Invalid OMPathRunner class: {type(ompath_runner)}!") + self._ompath_runner = ompath_runner + + self.model_execution_local = model_execution_local + if cmd_prefix is not None: + self._cmd_prefix = cmd_prefix + + # TODO: some checking?! + # if ompath_runner == Type[OMPathRunnerBash]: + def __post_init__(self) -> None: """ No connection to an OMC server is created by this class! @@ -1886,7 +2122,7 @@ def model_execution_prefix(self, cwd: Optional[OMPathABC] = None) -> list[str]: """ Helper function which returns a command prefix. """ - return [] + return self.get_cmd_prefix() def get_version(self) -> str: """ @@ -1897,15 +2133,15 @@ def get_version(self) -> str: def set_workdir(self, workdir: OMPathABC) -> None: """ - Set the workdir for this session. + Set the workdir for this session. For OMSessionRunner this is a nop. The workdir must be defined within the + definition of cmd_prefix. """ - os.chdir(workdir.as_posix()) def omcpath(self, *path) -> OMPathABC: """ Create an OMCPath object based on the given path segments and the current OMCSession* class. """ - return OMPathRunnerLocal(*path, session=self) + return self._ompath_runner(*path, session=self) def omcpath_tempdir(self, tempdir_base: Optional[OMPathABC] = None) -> OMPathABC: """ diff --git a/OMPython/__init__.py b/OMPython/__init__.py index d6016e53..4dc2f974 100644 --- a/OMPython/__init__.py +++ b/OMPython/__init__.py @@ -41,6 +41,10 @@ OMCSessionException, OMCSessionLocal, OMCSessionPort, + + OMPathRunnerBash, + OMPathRunnerLocal, + OMCSessionWSL, OMCSessionZMQ, ) @@ -77,6 +81,10 @@ 'OMCSessionException', 'OMCSessionPort', 'OMCSessionLocal', + + 'OMPathRunnerBash', + 'OMPathRunnerLocal', + 'OMCSessionWSL', 'OMCSessionZMQ', ]