diff --git a/README.md b/README.md index f21d3fa..29f19d2 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,13 @@ Welcome to the CIQ kernel-src-tree-tools package. This is a collection of script These are just scripts we have used internally and have decided to share with the community to encourage other to contribute to our requirements more easily. +## KT (Kernel Tools) +In this repo there is a directory called `KT` which contains a "kernel tools" +python package that should be installed into your python environment. Its purpose +is to consolidate all of the one off or single purpose scripts to share more +common code and functionality. Please see the KT directions in KT/kt.md for +setup directions. + ## ciq-cherry-pick This script is used to cherry-pick a commit from a remote repository to the current branch. It is a wrapper around `git cherry-pick -nsx ` command but sets up the incoming commit with the header information we require. diff --git a/kernel_install_dep.sh b/kernel_install_dep.sh index e65764c..56ef3c1 100755 --- a/kernel_install_dep.sh +++ b/kernel_install_dep.sh @@ -59,12 +59,13 @@ install_kselftest_deps_8() { traceroute \ wget - # Doesn't work for 8.6? + # Not available on all 8.x minor releases sudo dnf -y install --enablerepo=devel \ fuse-devel \ gcc-toolset-13-libasan-devel \ glibc-static \ - kernel-selftests-internal + kernel-selftests-internal \ + || echo "Optional packages not available or install failed; continuing." pip3 install --user \ jsonschema \ diff --git a/kt/KT.md b/kt/KT.md index 7f57db7..de6eb5e 100644 --- a/kt/KT.md +++ b/kt/KT.md @@ -11,16 +11,68 @@ By keeping this under the same repo, it will be easier to refactor things. ## Setup: -1. Install dependencies globally (you can also create a venv) : +1. Install dependencies globally (you can also create a venv) in the repository +root directory: ``` -$ python -m pip install -e ".[dev]" +[kernel-src-tree-tools]$ python -m pip install -e ".[dev]" ``` 2. The command above will install pre-commit. To setup the pre-commit tool before you commit something, run this: ``` -$ pre-commit install +[kernel-src-tree-tools]$ pre-commit install ``` +3. Set up the configuration file for kt. +There is a default one in kt/data/config.json but you can create your own by +defining the KTOOLS_CONFIG_FILE enviroment varriable to point to your own config +file. +```bash +$ echo $KTOOLS_CONFIG_FILE +/home/jmaple/.config/kt/test_config.json + +(.venv) [jmaple@devbox kernel-src-tree-tools]$ cat $KTOOLS_CONFIG_FILE +{ + "base_path": "~/workspace/kt_test", + "kernels_dir": "~/workspace/kt_test/kernels", + "images_source_dir": "~/workspace/kt_test/images_source", + "images_dir": "~/workspace/kt_test/images", + "ssh_key": "~/.ssh/test.pub", + "user": "USER" +} +``` +user defaults to $USER, set this if you wish for your user on the vms to be +different then define. + +4. Private Repos +By default, kt will use the public repos defined in kt/data/kernels.yaml. If +you have access to our private repos, you can create a .private_repos.yaml in +the base_path directory and define the private repos and branches should they +differ from the public ones. For example: +```yaml +private_repos: + dist-git-tree-lts: + +kernel_override: + lts-9.2: + dist_git_branch: + dist_git_root: dist-git-tree-lts +``` + +Optional is if you have an active depot account you can define the channels you +wish to use for each kernel. For example: +```yaml +kernel_override: + lts-9.2: + dist_git_branch: + dist_git_root: dist-git-tree-lts + depot_channels: + - +``` + +5. Setup needs to be run first. +```bash +$ kt setup +``` ## Implementation details: kt/ktlib is the place for common helpers that would be used for kt commands. diff --git a/kt/commands/list_kernels/impl.py b/kt/commands/list_kernels/impl.py index 5df5af4..08011aa 100644 --- a/kt/commands/list_kernels/impl.py +++ b/kt/commands/list_kernels/impl.py @@ -8,4 +8,5 @@ def main(automated: bool = False): for k in sorted(kernels.values(), key=lambda k: k.name): if not automated or k.automated: - print(k.name) + label = f"{k.name} (override)" if k.overridden else f"{k.name} (default)" + print(label) diff --git a/kt/data/cloud_init_centos7.yaml b/kt/data/cloud_init_centos7.yaml new file mode 100644 index 0000000..1333c02 --- /dev/null +++ b/kt/data/cloud_init_centos7.yaml @@ -0,0 +1,53 @@ +#cloud-config +users: + - name: USER_PLACEHOLDER + sudo: ALL=(ALL) NOPASSWD:ALL + groups: mock + ssh_authorized_keys: + - SSH_KEY_PLACEHOLDER +chpasswd: + expire: false + list: + - USER_PLACEHOLDER:test + +ssh_pwauth: true + +# Ensure the system does not update on boot. +package_upgrade: False + +mounts: + - - NFS_SOURCE_PLACEHOLDER + - SHARED_DIR_PLACEHOLDER + - nfs + - defaults + - - SHARED_DIR_PLACEHOLDER + - /var/lib/mock + - bind + - defaults,bind + +# Change working directory after boot +write_files: + - path: /etc/profile.d/change_dir.sh + content: | + cd WORKING_DIR_PLACEHOLDER + permissions: '0644' + +# if homedir already exists, cloud-init does nothing about it +# and then root is the owner +# workaround to change the owner to user +runcmd: + - - chown + - USER_PLACEHOLDER:USER_PLACEHOLDER + - HOMEDIR_PLACEHOLDER + - sudo sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/CentOS-*.repo + - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/CentOS-*.repo + - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/CentOS-*.repo + - [sudo, yum, install, -y, mock, git, vim] + - [sudo, yum, install, -y, ncurses-devel, make, gcc, bc, bison, flex, elfutils-libelf-devel, openssl-devel] + - [sudo, yum, update, -y] + +power_state: + mode: reboot + message: "Rebooting after install" + timeout: 30 + condition: true diff --git a/kt/data/kernels.yaml b/kt/data/kernels.yaml index 2095ee6..2d58a09 100644 --- a/kt/data/kernels.yaml +++ b/kt/data/kernels.yaml @@ -1,5 +1,6 @@ common_repos: dist-git-tree-fips: git@gitlab.com:ctrl-iq-public/fips/src/kernel.git + dist-git-tree: git@github.com:ciq-rocky-lts/kernel.git kernel-src-tree: https://github.com/ctrliq/kernel-src-tree.git kernel-src-tree-tools: https://github.com/ctrliq/kernel-src-tree-tools.git @@ -7,43 +8,40 @@ kernels: cbr-7.9: src_tree_root: kernel-src-tree src_tree_branch: ciqcbr7_9 - dist_git_root: dist-git-tree-cbr - dist_git_branch: cbr79-7 + dist_git_root: dist-git-tree + dist_git_branch: el-7.9 mock_config: centos-cbr79 automated: true + vm_image_url: https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2211.qcow2 + os_variant: centos7 + use_nfs: true lts-8.6: src_tree_root: kernel-src-tree src_tree_branch: ciqlts8_6 - dist_git_root: dist-git-tree-lts - dist_git_branch: lts86-8 + dist_git_root: dist-git-tree + dist_git_branch: el-8.6 mock_config: rocky-lts86 automated: true vm_image_url: https://dl.rockylinux.org/vault/rocky/8.6/images/Rocky-8-GenericCloud-8.6.20220702.0.x86_64.qcow2 - depot_channels: - - lts-8.6 lts-9.2: src_tree_root: kernel-src-tree src_tree_branch: ciqlts9_2 - dist_git_root: dist-git-tree-lts - dist_git_branch: lts92-9 + dist_git_root: dist-git-tree + dist_git_branch: el-9.2 mock_config: rocky-lts92 automated: true vm_image_url: https://dl.rockylinux.org/vault/rocky/9.2/images/x86_64/Rocky-9-GenericCloud-Base-9.2-20230513.0.x86_64.qcow2 - depot_channels: - - lts-9.2 lts-9.6: src_tree_root: kernel-src-tree src_tree_branch: ciqlts9_6 - dist_git_root: dist-git-tree-lts - dist_git_branch: lts96-9 + dist_git_root: dist-git-tree + dist_git_branch: el-9.6 mock_config: rocky-lts96 automated: true vm_image_url: https://dl.rockylinux.org/vault/rocky/9.6/images/x86_64/Rocky-9-GenericCloud-Base-9.6-20250531.0.x86_64.qcow2 - depot_channels: - - lts-9.6 fipslegacy-8.6: src_tree_root: kernel-src-tree diff --git a/kt/ktlib/config.py b/kt/ktlib/config.py index be3672e..994fdd2 100644 --- a/kt/ktlib/config.py +++ b/kt/ktlib/config.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings from dataclasses import dataclass from typing import ClassVar, Optional @@ -43,8 +44,21 @@ class Config: "user": os.environ["USER"], } + REQUIRED_KEYS: ClassVar = {"base_path", "kernels_dir", "images_source_dir", "images_dir", "ssh_key"} + @classmethod def from_str_dict(cls, data: dict[str, str]): + missing = cls.REQUIRED_KEYS - data.keys() + if missing: + example = json.dumps(cls.DEFAULT, indent=2) + sys.exit( + f"Error: config is missing required keys:\n\t{', '.join(sorted(missing))}" + f"\n\nExample config (note user is optional):\n{example}" + ) + + if "user" not in data: + data = {**data, "user": os.environ["USER"]} + # Transform the str values to Path except for user non_path_keys = {"user"} new_data = {k: (Path(v).expanduser() if k not in non_path_keys else v) for k, v in data.items()} diff --git a/kt/ktlib/kernels.py b/kt/ktlib/kernels.py index e64f192..6e2f5a1 100644 --- a/kt/ktlib/kernels.py +++ b/kt/ktlib/kernels.py @@ -41,6 +41,9 @@ class KernelInfo: vm_image_url: str | None = None depot_channels: list[str] | None = None + os_variant: str | None = None + use_nfs: bool = False + overridden: bool = False @dataclass @@ -49,17 +52,17 @@ class KernelsInfo: repos: dict[str, RepoInfo] @classmethod - def _load_private_repos(cls, config: Config) -> dict[str, str]: - """Load private repository URLs from local config""" + def _load_private_config(cls, config: Config) -> tuple[dict[str, str], dict[str, dict]]: + """Load private repository URLs and kernel overrides from local config""" private_repos_path = config.base_path / Constants.PRIVATE_REPOS_CONFIG_FILE if not private_repos_path.exists(): logging.info(f"{private_repos_path} does not exist") - return {} + return {}, {} with open(private_repos_path) as f: data = yaml.safe_load(f) - return data.get("private_repos", {}) + return data.get("private_repos", {}), data.get("kernel_overrides", {}) @classmethod def _get_repos(cls, data: dict, private_data: dict, config: Config): @@ -83,8 +86,9 @@ def _get_repos(cls, data: dict, private_data: dict, config: Config): return repos @classmethod - def _get_kernels_info(cls, data: dict, repos: dict[str, RepoInfo]): + def _get_kernels_info(cls, data: dict, repos: dict[str, RepoInfo], kernel_overrides: dict[str, dict] = None): kernels_info = {} + kernel_overrides = kernel_overrides or {} try: items = data[Constants.KERNELS].items() @@ -94,6 +98,10 @@ def _get_kernels_info(cls, data: dict, repos: dict[str, RepoInfo]): for kernel, info in items: k_info_dict = {"name": kernel, **info} + if kernel in kernel_overrides: + k_info_dict.update(kernel_overrides[kernel]) + k_info_dict["overridden"] = True + # Make the src_tree_root and dist_git_root absolute paths to the # local clone of these repos (transformation from src to Path) # repos dictionary contains the proper absolute paths @@ -129,13 +137,13 @@ def from_yaml(cls, config: Config): with open(KERNEL_INFO_YAML_PATH) as f: data = yaml.safe_load(f) - private_data = cls._load_private_repos(config=config) + private_data, kernel_overrides = cls._load_private_config(config=config) - return cls.from_dict(data=data, private_data=private_data, config=config) + return cls.from_dict(data=data, private_data=private_data, config=config, kernel_overrides=kernel_overrides) @classmethod - def from_dict(cls, data: dict, private_data: dict, config: Config): + def from_dict(cls, data: dict, private_data: dict, config: Config, kernel_overrides: dict[str, dict] = None): repos = cls._get_repos(data=data, private_data=private_data, config=config) - kernels_info = cls._get_kernels_info(data=data, repos=repos) + kernels_info = cls._get_kernels_info(data=data, repos=repos, kernel_overrides=kernel_overrides) return cls(kernels=kernels_info, repos=repos) diff --git a/kt/ktlib/util.py b/kt/ktlib/util.py index 8284970..9522129 100644 --- a/kt/ktlib/util.py +++ b/kt/ktlib/util.py @@ -12,6 +12,9 @@ class Constants: DEFAULT_VM_BASE = "Rocky" CLOUD_INIT = "cloud_init.yaml" + LIBVIRT_HOST_IP = "192.168.122.1" VM_STARTUP_WAIT_SECONDS = 60 VM_REBOOT_WAIT_SECONDS = 120 + VM_POLL_INTERVAL_SECONDS = 15 + VM_POLL_MAX_ATTEMPTS = 20 diff --git a/kt/ktlib/virt.py b/kt/ktlib/virt.py index 50aa9f9..7b83f6f 100644 --- a/kt/ktlib/virt.py +++ b/kt/ktlib/virt.py @@ -38,18 +38,19 @@ def install( cls, name: str, qcow2_path: Path, - vm_major_version: str, + os_variant: str, cloud_init_path: Path, common_dir: Path, vcpus: int = 12, memory: int = 32768, + use_nfs: bool = False, ): command = [ "--name", name, "--disk", f"{qcow2_path},device=disk,bus=virtio", - f"--os-variant=rocky{vm_major_version}", + f"--os-variant={os_variant}", "--virt-type", "kvm", "--vcpus", @@ -59,13 +60,18 @@ def install( "--vnc", "--cloud-init", f"user-data={cloud_init_path}", - "--filesystem", - f"source={common_dir},target=mount_tag_mock_scratch,accessmode=passthrough,driver.type=virtiofs,driver.queue=1024,binary.path=/usr/libexec/virtiofsd,binary.xattr=on", - "--memorybacking", - "source.type=memfd,access.mode=shared", - "--noautoconsole", ] + if not use_nfs: + command += [ + "--filesystem", + f"source={common_dir},target=mount_tag_mock_scratch,accessmode=passthrough,driver.type=virtiofs,driver.queue=1024,binary.path=/usr/libexec/virtiofsd,binary.xattr=on", + "--memorybacking", + "source.type=memfd,access.mode=shared", + ] + + command.append("--noautoconsole") + cls.run(command_type=VmCommandType.VIRT_INSTALL, command=command) @classmethod diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index d9125d8..c0af554 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -22,6 +22,7 @@ # TODO move this to a separate repo CLOUD_INIT_BASE_PATH = Path(__file__).parent.parent.joinpath("data/cloud_init.yaml") +CLOUD_INIT_CENTOS7_PATH = Path(__file__).parent.parent.joinpath("data/cloud_init_centos7.yaml") @dataclass @@ -48,6 +49,8 @@ class Vm: kernel_workspace: KernelWorkspace vm_image_url: str | None = None depot_channels: list[str] | None = None + os_variant: str | None = None + use_nfs: bool = False @classmethod def load( @@ -56,6 +59,8 @@ def load( kernel_workspace: KernelWorkspace, vm_image_url: str | None = None, depot_channels: list[str] | None = None, + os_variant: str | None = None, + use_nfs: bool = False, ): kernel_workspace_str = kernel_workspace.folder.name kernel_name = cls._extract_kernel_name(kernel_workspace_str) @@ -85,6 +90,8 @@ def load( kernel_workspace=kernel_workspace, vm_image_url=vm_image_url, depot_channels=depot_channels, + os_variant=os_variant, + use_nfs=use_nfs, ) @classmethod @@ -123,15 +130,24 @@ def load_from_workspace(cls, kernel_workspace_name: str): vm_image_url = None depot_channels = None + os_variant = None + use_nfs = False kernel_name = cls._extract_kernel_name(kernel_workspace_name) kernels_info = KernelsInfo.from_yaml(config=config) kernel_info = kernels_info.kernels.get(kernel_name) if kernel_info: vm_image_url = kernel_info.vm_image_url depot_channels = kernel_info.depot_channels + os_variant = kernel_info.os_variant + use_nfs = kernel_info.use_nfs return cls.load( - config=config, kernel_workspace=kernel_workspace, vm_image_url=vm_image_url, depot_channels=depot_channels + config=config, + kernel_workspace=kernel_workspace, + vm_image_url=vm_image_url, + depot_channels=depot_channels, + os_variant=os_variant, + use_nfs=use_nfs, ) @classmethod @@ -188,6 +204,9 @@ def _download_source_image(self, override_base: bool = False): logging.info(f"Downloading image from {self._get_vm_url()}") wget.download(self._get_vm_url(), out=str(self.qcow2_source_path)) + def _is_centos7(self): + return self.os_variant and self.os_variant.startswith("centos7") + def _setup_cloud_init(self, config: Config, no_depot: bool = False): yaml = YAML() yaml.preserve_quotes = True @@ -195,8 +214,8 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): yaml.default_flow_style = False yaml.best_sequence_indent = 2 - data = None - with open(CLOUD_INIT_BASE_PATH) as f: + template_path = CLOUD_INIT_CENTOS7_PATH if self._is_centos7() else CLOUD_INIT_BASE_PATH + with open(template_path) as f: data = yaml.load(f) # replace placeholders with user data @@ -210,8 +229,15 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): ssh_key_content = f.read().strip() data["users"][0]["ssh_authorized_keys"][0] = ssh_key_content - data["mounts"][0][1] = str(config.base_path.absolute()) - data["mounts"][1][0] = str(config.base_path.absolute()) + base_path_str = str(config.base_path.absolute()) + if self._is_centos7(): + nfs_source = f"{Constants.LIBVIRT_HOST_IP}:{base_path_str}" + data["mounts"][0][0] = nfs_source + data["mounts"][0][1] = base_path_str + data["mounts"][1][0] = base_path_str + else: + data["mounts"][0][1] = base_path_str + data["mounts"][1][0] = base_path_str # Go to the working directory of the kernel working_dir = config.kernels_dir / Path(self.name) @@ -223,20 +249,23 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): data["runcmd"][0][1] = f"{config.user}:{config.user}" data["runcmd"][0][2] = os.environ["HOME"] - # Pin dnf to vault for kernels with a pinned VM image (Rocky only, not CentOS/cbr) - if self.vm_image_url: - data["runcmd"].append(f'echo "{self.vm_major_minor_version}" > /etc/dnf/vars/releasever') - data["runcmd"].append('echo "vault/rocky" > /etc/dnf/vars/contentdir') - data["runcmd"].append( - "cd /etc/yum.repos.d/ && for f in *.repo; do " - 'sed -i -e "s/^mirrorlist=/#mirrorlist=/" -e "s/#baseurl=/baseurl=/" "$f"; done' - ) - data["runcmd"].append("dnf clean all") + if not self._is_centos7(): + # Pin dnf to vault for kernels with a pinned VM image (Rocky only) + if self.vm_image_url: + data["runcmd"].append(f'echo "{self.vm_major_minor_version}" > /etc/dnf/vars/releasever') + data["runcmd"].append('echo "vault/rocky" > /etc/dnf/vars/contentdir') + data["runcmd"].append( + "cd /etc/yum.repos.d/ && for f in *.repo; do " + 'sed -i -e "s/^mirrorlist=/#mirrorlist=/" -e "s/#baseurl=/baseurl=/" "$f"; done' + ) + data["runcmd"].append("dnf clean all") + + pkg_mgr = "yum" if self._is_centos7() else "dnf" # Depot: install the client and login+enable if credentials are present if not no_depot: data["runcmd"].append( - "dnf install -y https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm" + f"{pkg_mgr} install -y https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm" ) depot_user = os.environ.get("DEPOT_USER") depot_token = os.environ.get("DEPOT_TOKEN") @@ -244,9 +273,15 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): data["runcmd"].append(f"depot login -u {depot_user} -t {depot_token}") for channel in self.depot_channels: data["runcmd"].append(f"depot enable {channel} -y") + data["runcmd"].append(f"{pkg_mgr} clean all") + data["runcmd"].append(f"{pkg_mgr} update -y") + + # kernel_install_dep.sh only supports Rocky 8/9/10 + if not self._is_centos7(): + data["runcmd"].append( + [str(config.base_path / Path("kernel-src-tree-tools") / Path("kernel_install_dep.sh"))] + ) - # Install packages needed later - data["runcmd"].append([str(config.base_path / Path("kernel-src-tree-tools") / Path("kernel_install_dep.sh"))]) # Write this to image cloud_init with open(self.cloud_init_path, "w") as f: yaml.dump(data, f) @@ -266,14 +301,16 @@ def _create_image(self, config: Config, vcpus: int = 12, memory: int = 32768, no time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) def _virt_install(self, config: Config, vcpus: int = 12, memory: int = 32768): + os_variant = self.os_variant or f"rocky{self.vm_major_version}" return VmCommand.install( name=self.name, qcow2_path=self.qcow2_path, - vm_major_version=self.vm_major_version, + os_variant=os_variant, cloud_init_path=self.cloud_init_path, common_dir=config.base_path, vcpus=vcpus, memory=memory, + use_nfs=self.use_nfs, ) def _resize_disk(self): @@ -287,11 +324,33 @@ def _resize_disk(self): def setup(self, override_base: bool = False): self._download_source_image(override_base=override_base) + def _wait_for_running(self): + attempted_start = False + for attempt in range(Constants.VM_POLL_MAX_ATTEMPTS): + if VirtHelper.is_running(vm_name=self.name): + logging.info(f"VM {self.name} is running") + return + if not attempted_start and VirtHelper.exists(vm_name=self.name): + logging.info(f"VM {self.name} is shut off, attempting start...") + try: + VmCommand.start(vm_name=self.name) + attempted_start = True + except RuntimeError: + pass + logging.info( + f"Waiting for VM {self.name} to be running (attempt {attempt + 1}/{Constants.VM_POLL_MAX_ATTEMPTS})..." + ) + time.sleep(Constants.VM_POLL_INTERVAL_SECONDS) + raise RuntimeError( + f"VM {self.name} did not become running after {Constants.VM_POLL_MAX_ATTEMPTS * Constants.VM_POLL_INTERVAL_SECONDS}s" + ) + def spin_up(self, config: Config, vcpus: int = 12, memory: int = 32768, no_depot: bool = False) -> VmInstance: if not VirtHelper.exists(vm_name=self.name): logging.info(f"VM {self.name} does not exist, creating from scratch...") self._create_image(config=config, vcpus=vcpus, memory=memory, no_depot=no_depot) + self._wait_for_running() return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) logging.info(f"Vm {self.name} already exists") @@ -302,7 +361,7 @@ def spin_up(self, config: Config, vcpus: int = 12, memory: int = 32768, no_depot logging.info(f"Vm {self.name} is not running, starting it") VmCommand.start(vm_name=self.name) - time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) + self._wait_for_running() return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) diff --git a/tests/kt/ktlib/test_config.py b/tests/kt/ktlib/test_config.py index dc1cef9..f6a20c4 100644 --- a/tests/kt/ktlib/test_config.py +++ b/tests/kt/ktlib/test_config.py @@ -34,6 +34,15 @@ def test_config_load_from_filename_not_exists(): assert config.base_path == DEFAULT_CONFIG["base_path"] +def test_config_load_from_filename_valid_file(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(CONFIG_STR) + + config = Config.from_filename(str(config_file)) + assert config.base_path == Path("~/ciq").expanduser() + assert config.user == "testuser" + + def test_config_load_from_json_None(): config = Config.from_json(None) assert config.base_path == DEFAULT_CONFIG["base_path"] @@ -50,8 +59,56 @@ def test_config_load_from_json_data_None(): def test_config_load_from_json_data_empty(): json_data = "{}" - with pytest.raises(TypeError, match="missing 6 required positional arguments:"): - config = Config.from_json(json_data) # noqa F841 + with pytest.raises(SystemExit, match="config is missing"): + Config.from_json(json_data) + + +def test_config_load_from_json_partial_config_missing_keys(): + json_data = '{"base_path": "~/workspace/kt_pro"}' + + with pytest.raises(SystemExit, match="kernels_dir"): + Config.from_json(json_data) + + +def test_config_load_from_json_partial_config_lists_all_missing(): + json_data = '{"base_path": "~/workspace/kt_pro"}' + + with pytest.raises(SystemExit) as exc_info: + Config.from_json(json_data) + + msg = str(exc_info.value) + for key in ["images_dir", "images_source_dir", "kernels_dir", "ssh_key"]: + assert key in msg + + +def test_config_load_from_json_user_defaults_from_env(monkeypatch): + monkeypatch.setenv("USER", "testuser") + json_data = ( + "{" + '"base_path": "~/ciq",' + '"kernels_dir": "~/ciq/kernels",' + '"images_source_dir": "~/ciq/default_test_images",' + '"images_dir": "~/ciq/tmp/virt-images",' + '"ssh_key": "~/ciq/id_ed25519_generic.pub"' + "}" + ) + + config = Config.from_json(json_data) + assert config.user == "testuser" + + +def test_config_load_from_str_dict_relative_path_raises(): + data = { + "base_path": "relative/path", + "kernels_dir": "~/ciq/kernels", + "images_source_dir": "~/ciq/default_test_images", + "images_dir": "~/ciq/tmp/virt-images", + "ssh_key": "~/ciq/id_ed25519_generic.pub", + "user": "testuser", + } + + with pytest.raises(ValueError, match="all paths should be absolute"): + Config.from_str_dict(data) def test_config_load_from_json_proper_base_path(): diff --git a/tests/kt/ktlib/test_kernels.py b/tests/kt/ktlib/test_kernels.py index 4b6b4b0..9370929 100644 --- a/tests/kt/ktlib/test_kernels.py +++ b/tests/kt/ktlib/test_kernels.py @@ -4,13 +4,13 @@ from kt.ktlib.config import Config from kt.ktlib.kernels import KernelsInfo -common_repos = {"dist-git-tree-cbr": "dist-url", "kernel-src-tree": "src-url"} +common_repos = {"dist-git-tree": "dist-url", "kernel-src-tree": "src-url"} kernels = { "kernel1": { "src_tree_root": "kernel-src-tree", "src_tree_branch": "src-branch", - "dist_git_root": "dist-git-tree-cbr", + "dist_git_root": "dist-git-tree", "dist_git_branch": "dist-branch", "mock_config": "test-mock-config", "automated": True, @@ -91,7 +91,7 @@ def test_kernels_from_dict_check_dist_root(): kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) kernel_info = list(kernels_info.kernels.values())[0] - assert kernel_info.dist_git_root.folder == config.base_path / Path("dist-git-tree-cbr") + assert kernel_info.dist_git_root.folder == config.base_path / Path("dist-git-tree") def test_kernels_from_dict_check_src_root(): @@ -110,6 +110,43 @@ def test_kernels_vm_image_url_default_none(): assert kernel_info.vm_image_url is None +def test_kernels_override_dist_git_branch(): + config = Config.from_str_dict(Config.DEFAULT) + overrides = {"kernel1": {"dist_git_branch": "overridden-branch"}} + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config, kernel_overrides=overrides) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.dist_git_branch == "overridden-branch" + + +def test_kernels_override_dist_git_root(): + private_repos = {"dist-git-tree-private": "private-url"} + overrides = {"kernel1": {"dist_git_root": "dist-git-tree-private"}} + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict( + data=data, private_data=private_repos, config=config, kernel_overrides=overrides + ) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.dist_git_root.folder == config.base_path / Path("dist-git-tree-private") + + +def test_kernels_override_nonexistent_kernel_ignored(): + config = Config.from_str_dict(Config.DEFAULT) + overrides = {"nonexistent-kernel": {"dist_git_branch": "some-branch"}} + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config, kernel_overrides=overrides) + assert len(kernels_info.kernels) == 1 + + +def test_kernels_no_overrides_uses_defaults(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.dist_git_branch == "dist-branch" + + def test_kernels_vm_image_url_present(): pinned_url = ( "https://dl.rockylinux.org/vault/rocky/9.2/images/x86_64/Rocky-9-GenericCloud-Base-9.2-20230513.0.x86_64.qcow2"