diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c7b5b82..463f7a0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,14 +39,6 @@ jobs: with: string: ${{ github.repository }} - - name: set dockerhub repo name - uses: mad9000/actions-find-and-replace-string@5 - id: dockerhub_image - with: - source: ${{ steps.docker_image.outputs.lowercase }} - find: 'bb-ricardo' - replace: ${{ vars.DOCKER_HUB_USERNAME }} - - name: Log in to GitHub Container Registry uses: docker/login-action@v4 with: @@ -71,36 +63,3 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: Log in to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - - name: Docker hub meta - id: meta_dhub - uses: docker/metadata-action@v5 - with: - images: docker.io/${{ steps.dockerhub_image.outputs.value }} - tags: | - type=semver,pattern={{version}} - type=ref,event=branch - - - name: Build and push with tags - ${{ steps.dockerhub_image.outputs.value }} - uses: docker/build-push-action@v6 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_dhub.outputs.tags }} - labels: ${{ steps.meta_dhub.outputs.labels }} - -# - name: push README to Dockerhub -# uses: christian-korneck/update-container-description-action@v1 -# env: -# DOCKER_USER: ${{ vars.DOCKER_HUB_USERNAME }} -# DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }} -# with: -# destination_container_repo: ${{ steps.dockerhub_image.outputs.value }} -# provider: dockerhub diff --git a/CODEOWNERS b/CODEOWNERS index 3a5333e..176e27a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,5 +2,5 @@ * @semx @bb-ricardo -/sources/hetzner/ @SerhiiZahuba -docs/source_hetzner.md @SerhiiZahuba +/module/sources/hetzner/ @SerhiiZahuba +/docs/source_hetzner.md @SerhiiZahuba diff --git a/README.md b/README.md index cd83257..f395b1e 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.9.0 (2026-10-02) +Version: 1.9.1 (2026-10-05) Project URL: https://github.com/bb-ricardo/netbox-sync options: diff --git a/module/__init__.py b/module/__init__.py index f5e5c9c..138d096 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -7,8 +7,8 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -__version__ = "1.9.0" -__version_date__ = "2026-10-02" +__version__ = "1.9.1" +__version_date__ = "2026-10-05" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/module/netbox/connection.py b/module/netbox/connection.py index c4d686f..e32ebf6 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -489,8 +489,7 @@ def query_current_data(self, netbox_objects_to_query=None): cache_this_class = False if self.testing_cache is True and len(cached_nb_data) > 0: - for object_data in cached_nb_data: - self.inventory.add_object(nb_object_class, data=object_data, read_from_netbox=True) + self.inventory.load_from_netbox(nb_object_class, cached_nb_data) # mark this object class as retrieved self.resolved_dependencies.add(nb_object_class) @@ -561,8 +560,7 @@ def query_current_data(self, netbox_objects_to_query=None): log.debug(f"Processing %s returned {nb_object_class.name}%s" % (len(nb_objects), plural(len(nb_objects)))) - for object_data in nb_objects: - self.inventory.add_object(nb_object_class, data=object_data, read_from_netbox=True) + self.inventory.load_from_netbox(nb_object_class, nb_objects) # mark this object class as retrieved self.resolved_dependencies.add(nb_object_class) diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 4d13740..af39be8 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -40,6 +40,9 @@ def __new__(cls): def init(self): + # object types read from NetBox in this run + self.loaded_types = set() + for object_type in NetBoxObject.__subclasses__(): self.base_structure[object_type.name] = list() @@ -211,6 +214,23 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None return new_object + def load_from_netbox(self, object_type, objects_data): + """ + Adds all objects of $object_type read from NetBox to the inventory. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type which was read from NetBox + objects_data: list + data of every object of this type NetBox returned + """ + + for object_data in objects_data: + self.add_object(object_type, data=object_data, read_from_netbox=True) + + self.loaded_types.add(object_type) + def add_update_object(self, object_type, data=None, read_from_netbox=False, source=None): """ Adds new object or updates existing object with data, based on the content of data. diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 2c05f1b..e6e8c81 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -900,7 +900,8 @@ def resolve_relations(self): if resolved_data is not None: self.data[key] = resolved_data - else: + # a type this run did not read from NetBox is expected to be unresolvable, keep the NetBox value + elif data_type in self.inventory.loaded_types: log.error(f"Problems resolving relation '{key}' for object '{self.get_display_name()}' and " f"value '{data_value}'") @@ -1727,7 +1728,8 @@ def resolve_relations(self): def matches_site(self, site=None) -> bool: """ - tries to figure out if this prefix matches a certain site or site group + tries to figure out if this prefix matches a certain site or site group. + Without a site only a prefix without a scope matches. Parameters ---------- @@ -1739,6 +1741,11 @@ def matches_site(self, site=None) -> bool: bool: True if matches one of the params """ + if site is None: + # a scope whose object is not in the inventory resolves to None as well + return not self.data.get("scope_type") and self.data.get("scope_id") is None and \ + self.data.get("site") is None + if isinstance(site, NBSite): if isinstance(self.data.get("scope_id"), NBSite) and self.data.get("scope_id") == site: return True diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 34119bc..00db850 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -92,6 +92,15 @@ def __init__(self): syncing.""", default_value=False), + ConfigOption("skip_os_reported_ips", + bool, + description="""define if IP addresses which the BMC reports for the ports of the server + itself are skipped. Some BMCs (i.e. HPE iLO with AMS) report the IP addresses of the + operating system on the physical NICs. If the operating system is synced by another + source (i.e. VMware ESXi hosts) these IPs would be moved back and forth between + interfaces. If True only the IP addresses of the BMC ports are synced.""", + default_value=False), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), ] diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index b79ccf2..91add4e 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -981,6 +981,9 @@ def update_network_interface(self): # collect ip addresses nic_ips[port_name] = list() + if self.settings.skip_os_reported_ips is True and mgmt_only is False: + continue + for ipv4_address in grab(nic_port, "ipv4_addresses", fallback=list()): if self.settings.permitted_subnets.permitted(ipv4_address, interface_name=port_name) is False: continue diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index df098e7..7537f51 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -165,7 +165,12 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac # grab current data for interface in self.inventory.get_all_interfaces(device_vm_object): - int_mac = grab(interface, "data.mac_address") + # from NetBox 4.2 on 'mac_address' is a read-only leftover of the last NetBox read which sources + # don't update. Use the current primary MAC address object instead. + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + int_mac = grab(grab(interface, "data.primary_mac_address"), "data.mac_address") + else: + int_mac = grab(interface, "data.mac_address") int_name = grab(interface, "data.name") if interface_exclude_filter is not None and int_name is not None and \ diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 27ea23c..60cf56d 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2433,10 +2433,10 @@ def add_host(self, obj): pnic_hints = dict() # noinspection PyBroadException try: - for hint in obj.configManager.networkSystem.QueryNetworkHint(""): + for hint in obj.configManager.networkSystem.QueryNetworkHint(): pnic_hints[hint.device] = hint - except Exception: - pass + except Exception as e: + log.debug(f"Unable to query network hints (CDP/LLDP) of host '{name}': {e}") pnic_list = grab(obj, "config.network.pnic", fallback=list()) if self.settings.skip_host_nics is True: diff --git a/pyproject.toml b/pyproject.toml index 0dfaa3f..9c284b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "netbox-sync" -version = "1.9.0" +version = "1.9.1" description = "Sync objects from various sources to NetBox" authors = [ { name = "Ricardo Bartels", email = "ricardo.bartels@telekom.de" } diff --git a/settings-example.ini b/settings-example.ini index b1c71fa..75b725f 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.9.0 (2026-10-02) +;;; Version: 1.9.1 (2026-10-05) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config @@ -584,6 +584,13 @@ inventory_file_path = /full/path/to/inventory/files ; incorrect syncing. ;skip_fhrp_group_ips = False +; define if IP addresses which the BMC reports for the ports of the server itself are +; skipped. Some BMCs (i.e. HPE iLO with AMS) report the IP addresses of the operating +; system on the physical NICs. If the operating system is synced by another source (i.e. +; VMware ESXi hosts) these IPs would be moved back and forth between interfaces. If True +; only the IP addresses of the BMC ports are synced. +;skip_os_reported_ips = False + ; define in which order the IP address tenant will be assigned if tenant is undefined. ; possible values: ; * device : host or VM tenant will be assigned to the IP address diff --git a/tests/test_check_redfish_interface_ips.py b/tests/test_check_redfish_interface_ips.py index 182d5de..577721c 100644 --- a/tests/test_check_redfish_interface_ips.py +++ b/tests/test_check_redfish_interface_ips.py @@ -13,7 +13,9 @@ NBIPAddress objects. """ +from module.common.misc import grab from module.netbox.object_classes import NBInterface, NBIPAddress +from module.sources.common.permitted_subnets import PermittedSubnets def seed_interface_with_ip(context, name="pnet0", address="172.10.10.12/24"): @@ -71,3 +73,51 @@ def test_ip_is_still_removed_when_the_discovered_ips_are_unusable(check_redfish_ keep_undiscovered_ips=True) assert "assigned_object_id" in ip.unset_items + + +def run_network_interface_sync(check_redfish_source, **settings): + """Sync a server NIC port and a BMC port which both report IP addresses, like an HPE iLO + with AMS which reports the IPs of the operating system on the physical NICs.""" + + context = check_redfish_source(**settings) + source = context.source + context.inventory.netbox_api_version = "4.2.0" + source.interface_adapter_type_dict = {} + source.manager_name = "iLO 5" + source.settings.permitted_subnets = PermittedSubnets("10.0.0.0/8") + source.settings.ip_tenant_inheritance_order = [] + + source.inventory_file_content = { + "inventory": { + "network_port": [ + {"id": "2.2", "addresses": ["8C:DC:D4:0F:D7:84"], + "operation_status": "Enabled", "link_status": "Up", "manager_ids": [], + "ipv4_addresses": ["10.40.232.163/22"], "ipv6_addresses": []}, + {"id": "1:1", "name": "Manager Dedicated Network Interface", + "addresses": ["08:F1:EA:93:F3:D6"], "operation_status": "Enabled", + "link_status": "Up", "manager_ids": ["1"], + "ipv4_addresses": ["10.44.102.102/22"], "ipv6_addresses": []}, + ], + } + } + + source.update_network_interface() + + return {grab(ip, "data.assigned_object_id.data.name"): str(grab(ip, "data.address")) + for ip in context.inventory.get_all_items(NBIPAddress)} + + +def test_os_reported_ips_are_synced_by_default(check_redfish_source): + """Without skip_os_reported_ips the IPs of all ports are synced, as before.""" + + assigned_ips = run_network_interface_sync(check_redfish_source) + + assert assigned_ips == {"2.2": "10.40.232.163/22", "iLO 5 (1:1)": "10.44.102.102/22"} + + +def test_os_reported_ips_are_skipped(check_redfish_source): + """With skip_os_reported_ips only the IPs of the BMC ports are synced.""" + + assigned_ips = run_network_interface_sync(check_redfish_source, skip_os_reported_ips=True) + + assert assigned_ips == {"iLO 5 (1:1)": "10.44.102.102/22"} diff --git a/tests/test_interface_matching_primary_mac.py b/tests/test_interface_matching_primary_mac.py new file mode 100644 index 0000000..fa29309 --- /dev/null +++ b/tests/test_interface_matching_primary_mac.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +"""Interface matching by MAC must use the current primary MAC address object (NetBox >= 4.2). + +The legacy interface 'mac_address' field is only what NetBox returned at read time. Once a +previous source moved the primary MAC to another address, matching on the stale value merged +a BMC port (i.e. Lenovo XCC) into the ESXi vmk0 interface on every run. +""" + +from module.netbox.object_classes import NBInterface, NBMACAddress + +BMC_MAC = "38:68:DD:34:33:C5" +VMK_MAC = "00:50:56:67:39:DA" + + +def seed_vmk0(context, legacy_mac, primary_mac): + interface = context.inventory.add_object( + NBInterface, data={"name": "vmk0", "device": context.device, "type": "virtual", "mac_address": legacy_mac}, + source=context.source) + mac_object = context.inventory.add_object( + NBMACAddress, data={"mac_address": primary_mac, "assigned_object_id": interface, + "assigned_object_type": NBInterface}, source=context.source) + interface.update(data={"primary_mac_address": mac_object}, source=context.source) + return interface + + +def bmc_port(): + return {"XCC (1:NIC:abc)": {"name": "XCC (1:NIC:abc)", "mac_address": BMC_MAC, "type": "other", + "mgmt_only": True}} + + +def test_bmc_port_ignores_stale_legacy_mac(check_redfish_source): + + context = check_redfish_source() + seed_vmk0(context, legacy_mac=BMC_MAC, primary_mac=VMK_MAC) + + data = context.source.map_object_interfaces_to_current_interfaces(context.device, bmc_port(), True) + + assert data["XCC (1:NIC:abc)"] is None + + +def test_bmc_port_still_matches_by_primary_mac(check_redfish_source): + + context = check_redfish_source() + vmk0 = seed_vmk0(context, legacy_mac=VMK_MAC, primary_mac=BMC_MAC) + + data = context.source.map_object_interfaces_to_current_interfaces(context.device, bmc_port(), True) + + assert data["XCC (1:NIC:abc)"] is vmk0 diff --git a/tests/test_prefix_matching.py b/tests/test_prefix_matching.py new file mode 100644 index 0000000..d057cb2 --- /dev/null +++ b/tests/test_prefix_matching.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Prefix lookup for an IP (issue #579). + +An IP is matched against the prefixes of its site first and, if none fits, +against the prefixes without a scope. NBPrefix.matches_site(None) returned +False unconditionally, so a prefix without a scope was never found. + +The prefixes are fed in the shape the NetBox API returns them and resolved +like a real run does, so the scope handling of NBPrefix is part of the test. +""" +from ipaddress import ip_address +from types import SimpleNamespace + +import pytest + +from module.netbox.object_classes import ( + NBCluster, NBClusterType, NBIPAddress, NBLocation, NBPrefix, NBRegion, NBSite, NBSiteGroup, NBVM, NBVMInterface, + NBVRF, +) +from module.sources.common.source_base import SourceBase + + +@pytest.fixture +def source(inventory): + src = SourceBase() + src.inventory = inventory + src.name = "test" + return src + + +@pytest.fixture +def site(inventory): + return inventory.add_object(NBSite, data={"id": 1, "name": "site1"}, read_from_netbox=True) + + +def add_prefix(inventory, prefix, scope=None, **data): + """a prefix as read from /api/ipam/prefixes/ on NetBox 4.2+, scope given as (type, id)""" + scope_type, scope_id = scope or (None, None) + return inventory.add_object(NBPrefix, data={"prefix": prefix, "scope_type": scope_type, "scope_id": scope_id, + **data}, read_from_netbox=True) + + +def find(source, ip, site_name=None): + source.inventory.resolve_relations() + return source.return_longest_matching_prefix_for_ip(ip_address(ip), site_name) + + +def test_a_prefix_without_a_scope_is_found_without_a_site(inventory, source, site): + prefix = add_prefix(inventory, "10.40.232.0/22") + + assert find(source, "10.40.232.163") is prefix + + +def test_a_prefix_without_a_scope_is_not_a_site_prefix(inventory, source, site): + add_prefix(inventory, "10.40.232.0/22") + + assert find(source, "10.40.232.163", "site1") is None + + +def test_a_site_prefix_is_not_a_global_one(inventory, source, site): + prefix = add_prefix(inventory, "10.40.232.0/22", scope=("dcim.site", site.nb_id)) + + assert find(source, "10.40.232.163", "site1") is prefix + assert find(source, "10.40.232.163") is None + + +def test_the_site_prefix_and_the_global_one_are_told_apart(inventory, source, site): + global_prefix = add_prefix(inventory, "10.40.0.0/16") + site_prefix = add_prefix(inventory, "10.40.232.0/22", scope=("dcim.site", site.nb_id)) + + assert find(source, "10.40.232.163", "site1") is site_prefix + assert find(source, "10.40.1.1", "site1") is None + assert find(source, "10.40.1.1") is global_prefix + + +def test_the_longest_global_prefix_wins(inventory, source): + add_prefix(inventory, "192.168.0.0/16") + longer = add_prefix(inventory, "192.168.10.0/23") + + assert find(source, "192.168.11.214") is longer + + +def test_a_site_group_prefix_matches_the_sites_of_the_group(inventory, source): + group = inventory.add_object(NBSiteGroup, data={"id": 5, "name": "dc"}, read_from_netbox=True) + inventory.add_object(NBSite, data={"id": 1, "name": "site1", "group": {"id": 5, "name": "dc"}}, + read_from_netbox=True) + prefix = add_prefix(inventory, "10.40.232.0/22", scope=("dcim.sitegroup", group.nb_id)) + + assert find(source, "10.40.232.163", "site1") is prefix + assert find(source, "10.40.232.163") is None + + +def test_a_prefix_of_a_region_or_location_is_not_global(inventory, source, site): + # not matched by site either; that has not changed + region = inventory.add_object(NBRegion, data={"id": 7, "name": "eu"}, read_from_netbox=True) + location = inventory.add_object(NBLocation, data={"id": 3, "name": "row 1", "site": {"id": 1, "name": "site1"}}, + read_from_netbox=True) + add_prefix(inventory, "10.40.232.0/22", scope=("dcim.region", region.nb_id)) + add_prefix(inventory, "10.40.0.0/16", scope=("dcim.location", location.nb_id)) + + assert find(source, "10.40.232.163", "site1") is None + assert find(source, "10.40.232.163") is None + + +def test_a_prefix_whose_scope_is_not_in_the_inventory_is_not_global(inventory, source, site): + # the scope resolves to None then, the scope type is still set + add_prefix(inventory, "10.40.232.0/22", scope=("dcim.site", 99)) + add_prefix(inventory, "10.40.0.0/16", scope=("dcim.region", 7)) + + assert find(source, "10.40.232.163", "site1") is None + assert find(source, "10.40.232.163") is None + + +def test_a_prefix_with_a_site_of_an_old_netbox_is_not_global(inventory, source, site): + # NetBox before 4.2 reports the site of a prefix directly + prefix = inventory.add_object(NBPrefix, data={"prefix": "10.40.232.0/22", "site": {"id": 1, "name": "site1"}}, + read_from_netbox=True) + + assert find(source, "10.40.232.163", "site1") is prefix + assert find(source, "10.40.232.163") is None + + +def vm_interface_at_site(inventory, source, site): + """a VM interface on a cluster of the given site, and a source ready to sync its IPs""" + cluster_type = inventory.add_object(NBClusterType, data={"name": "vmware"}, read_from_netbox=True) + cluster = inventory.add_object(NBCluster, data={"name": "c1", "type": cluster_type, "scope_type": "dcim.site", + "scope_id": site.nb_id}, read_from_netbox=True) + vm = inventory.add_object(NBVM, data={"name": "vm1", "cluster": cluster}, read_from_netbox=True) + nic = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": vm, "enabled": True}, + read_from_netbox=True) + inventory.resolve_relations() + assert cluster.get_site_name() == "site1" + source.source_tag = "Source: test" + source.settings = SimpleNamespace( + skip_fhrp_group_ips=False, + ip_tenant_inheritance_order=["disabled"], + disable_vlan_sync=True, + vlan_group_relation_by_id=None, + vlan_group_relation_by_name=None, + vlan_sync_exclude_by_id=None, + vlan_sync_exclude_by_name=None, + ) + return vm, nic + + +def sync_ip(source, vm, nic, address): + _, ip_objects = source.add_update_interface( + interface_object=nic, device_object=vm, interface_data={"name": "eth0"}, interface_ips=[address], + vmware_object=SimpleNamespace(guest=SimpleNamespace(toolsRunningStatus="guestToolsRunning")), + ) + return ip_objects + + +def test_an_ip_without_a_length_is_added_with_the_length_of_the_global_prefix(inventory, source, site): + # the symptom from the report: the IP was skipped with "Unable to add IP address to NetBox" + vrf = inventory.add_object(NBVRF, data={"id": 2, "name": "lab"}, read_from_netbox=True) + add_prefix(inventory, "10.40.232.0/22", vrf={"id": 2, "name": "lab"}) + vm, nic = vm_interface_at_site(inventory, source, site) + + ip_objects = sync_ip(source, vm, nic, "10.40.232.163") + + assert [str(ip.data.get("address")) for ip in ip_objects] == ["10.40.232.163/22"] + assert ip_objects[0].data.get("vrf") is vrf + assert inventory.get_by_data(NBIPAddress, data={"address": "10.40.232.163/22"}) is ip_objects[0] + + +def test_the_prefix_of_the_own_site_wins_over_the_global_one(inventory, source, site): + site_vrf = inventory.add_object(NBVRF, data={"id": 2, "name": "site-vrf"}, read_from_netbox=True) + inventory.add_object(NBVRF, data={"id": 3, "name": "global-vrf"}, read_from_netbox=True) + add_prefix(inventory, "10.40.232.0/22", scope=("dcim.site", site.nb_id), vrf={"id": 2, "name": "site-vrf"}) + add_prefix(inventory, "10.40.232.0/22", vrf={"id": 3, "name": "global-vrf"}) + vm, nic = vm_interface_at_site(inventory, source, site) + + ip_objects = sync_ip(source, vm, nic, "10.40.232.163/22") + + assert ip_objects[0].data.get("vrf") is site_vrf diff --git a/tests/test_unloaded_relations.py b/tests/test_unloaded_relations.py new file mode 100644 index 0000000..57b94da --- /dev/null +++ b/tests/test_unloaded_relations.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +A run reads only the object types its sources work with. An object can still point at a +type the run did not read, e.g. the module of an interface in a VMware only setup. Such a +relation stays as NetBox returned it: it is not an error and the sync must not change it. +""" +import logging + +import pytest + +from module.netbox.object_classes import NBDevice, NBInterface, NBMACAddress, NBModule, NBPowerPort +from module.sources import instantiate_sources + +# how NetBox returns the module of a component +NETBOX_MODULE = {"id": 5, "display": "Embedded LOM"} + + +def relation_errors(caplog): + return [record.getMessage() for record in caplog.records + if record.levelno == logging.ERROR and "Problems resolving relation" in record.getMessage()] + + +@pytest.mark.parametrize("component_type", [NBInterface, NBPowerPort]) +def test_relation_to_a_type_the_run_did_not_read_is_kept(inventory, caplog, component_type): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(component_type, [{"id": 10, "name": "port0", "device": {"id": 1}, + "module": dict(NETBOX_MODULE)}]) + + inventory.resolve_relations() + + component = inventory.get_by_id(component_type, nb_id=10) + assert relation_errors(caplog) == [] + assert component.data["device"] is inventory.get_by_id(NBDevice, nb_id=1) + assert component.data["module"] == NETBOX_MODULE + + +def test_relation_to_an_object_the_run_created_is_resolved(inventory, caplog): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + # a source can create objects of a type it did not read, NetBox then returns them as a reference + mac_address = inventory.add_object(NBMACAddress, data={"mac_address": "00:11:22:33:44:55"}) + mac_address.nb_id = 7 + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(NBInterface, [{"id": 10, "name": "port0", "device": {"id": 1}, + "primary_mac_address": {"id": 7}}]) + + inventory.resolve_relations() + + assert relation_errors(caplog) == [] + assert inventory.get_by_id(NBInterface, nb_id=10).data["primary_mac_address"] is mac_address + + +def test_relation_to_a_missing_object_of_a_read_type_is_an_error(inventory, caplog): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(NBModule, []) + inventory.load_from_netbox(NBInterface, [{"id": 10, "name": "port0", "device": {"id": 1}, + "module": dict(NETBOX_MODULE)}]) + + inventory.resolve_relations() + + assert relation_errors(caplog) == [ + f"Problems resolving relation 'module' for object 'port0 (esx01)' and value '{NETBOX_MODULE}'"] + + +def test_vmware_sync_keeps_the_module_of_host_interfaces(vcsim, inventory, load_config, vmware_settings, caplog): + load_config(vmware_settings) + first_run = instantiate_sources()[0] + inventory.resolve_relations() + first_run.apply() + + # the second run finds the objects of the first one in NetBox, the host interfaces with a module + host_interfaces = [i for i in inventory.get_all_items(NBInterface) if "virtual" not in str(i.data.get("type"))] + if not host_interfaces: + pytest.skip(f"vcsim dump '{vcsim.name}' has no physical host interfaces") + for nb_id, this_object in enumerate(inventory.get_all_items(NBInterface) + inventory.get_all_items(NBDevice), + start=1): + this_object.nb_id = nb_id + this_object.is_new = False + this_object.updated_items = list() + this_object.unset_items = list() + for interface in host_interfaces: + interface.data["module"] = dict(NETBOX_MODULE) + + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.source_list = [] + load_config(vmware_settings) + second_run = instantiate_sources()[0] + inventory.resolve_relations() + second_run.apply() + + assert relation_errors(caplog) == [] + for interface in host_interfaces: + assert interface.data["module"] == NETBOX_MODULE, interface.get_display_name() + assert "module" not in interface.updated_items + interface.unset_items, interface.get_display_name() diff --git a/tests/test_vmware_host_cables.py b/tests/test_vmware_host_cables.py index c140921..4af11a6 100644 --- a/tests/test_vmware_host_cables.py +++ b/tests/test_vmware_host_cables.py @@ -10,14 +10,18 @@ """ Cables from the CDP/LLDP neighbors an ESXi host reports for its physical interfaces. -The vcsim captures carry no CDP/LLDP data, so the parts which turn a reported neighbor -into a cable are tested against a hand built inventory. What the vcsim run has to prove -is that the feature stays completely out of the way while `sync_host_cables` is disabled. +The vcsim captures report CDP neighbors whose switches are not in NetBox, so the parts +which turn a reported neighbor into a cable are tested against a hand built inventory. +What the vcsim run has to prove is that the feature stays completely out of the way +while `sync_host_cables` is disabled. """ from types import SimpleNamespace import pytest +from pyVmomi import vim + +from module.common.misc import grab from module.netbox.object_classes import NBCable, NBDevice, NBInterface, NBTag from module.sources import instantiate_sources from module.sources.vmware.connection import VMWareHandler @@ -477,5 +481,37 @@ def test_a_sync_with_the_option_enabled_still_works(vcsim, inventory, load_confi sources[0].apply() assert list(inventory.get_all_items(NBDevice)), "hosts must still be synced" - # none of the captured vcsim inventories reports a CDP/LLDP neighbor + # the neighbors the captures report are switches which are not in NetBox assert list(inventory.get_all_items(NBCable)) == [] + + +def test_network_hints_are_queried_without_a_device_name(vcsim, inventory, load_config, vmware_settings, + monkeypatch): + # QueryNetworkHint() takes an optional list of device names. vCenter raises NotFound + # for a name it does not know, "" included, and no neighbor was ever read that way + # (issue #580). vcsim ignores the argument, so it is checked here. + query_network_hint = vim.host.NetworkSystem.QueryNetworkHint + queried = list() + rejected = list() + + def like_vcenter(network_system, device=None): + # the caller swallows every exception, so a broken fake must show up in the lists, not raise + queried.append(device) + names = [device] if isinstance(device, str) else list(device or []) + known = [pnic.device for pnic in grab(network_system, "networkInfo.pnic", fallback=list())] + if any(name not in known for name in names): + rejected.append(device) + raise vim.fault.NotFound(msg="The object or item referred to could not be found.") + return query_network_hint(network_system, device) + + monkeypatch.setattr(vim.host.NetworkSystem, "QueryNetworkHint", like_vcenter) + + load_config(vmware_settings + "\nsync_host_cables = True\n") + sources = instantiate_sources() + assert sources and sources[0].init_successful + + inventory.resolve_relations() + sources[0].apply() + + assert len(queried) == len(list(inventory.get_all_items(NBDevice))) + assert rejected == [], "vCenter would have answered these queries with NotFound"