From 07fb2c08c61ee9861820196a17fbc5296d1bb39e Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 24 Apr 2026 12:26:20 +0200 Subject: [PATCH] Add sonic validate command for YANG-based config_db.json validation Validates generated SONiC ConfigDB JSON against the bundled SONiC YANG models so config drift, type errors and broken leafrefs are caught before configs are pushed to switches. Sources can be a local file, NetBox local_context, the export directory or a fresh in-memory generation. Pulls in sonic-yang-mgmt as a VCS install (not on PyPI), pinned to sonic-buildimage release branch 202511. Renders the three previously missing YANG models (sonic-extension, sonic-types, sonic-policer) from their upstream Jinja templates so the bundled model set is loadable by libyang. AI-assisted: Claude Code Signed-off-by: Christian Berendt --- .flake8 | 2 +- Containerfile | 1 + Pipfile | 1 + files/sonic/yang_models/sonic-extension.yang | 25 + files/sonic/yang_models/sonic-passwh.yang | 0 files/sonic/yang_models/sonic-policer.yang | 125 +++++ files/sonic/yang_models/sonic-types.yang | 524 +++++++++++++++++++ osism/commands/sonic.py | 245 +++++++++ osism/settings.py | 7 + osism/tasks/conductor/sonic/validator.py | 136 +++++ playbooks/test-setup.yml | 2 +- requirements.sonic.txt | 1 + requirements.txt | 3 +- setup.cfg | 1 + 14 files changed, 1070 insertions(+), 3 deletions(-) create mode 100644 files/sonic/yang_models/sonic-extension.yang mode change 100755 => 100644 files/sonic/yang_models/sonic-passwh.yang create mode 100644 files/sonic/yang_models/sonic-policer.yang create mode 100644 files/sonic/yang_models/sonic-types.yang create mode 100644 osism/tasks/conductor/sonic/validator.py create mode 100644 requirements.sonic.txt diff --git a/.flake8 b/.flake8 index a83721ad..5ed3c01f 100644 --- a/.flake8 +++ b/.flake8 @@ -5,4 +5,4 @@ exclude = files/disable files/generate files/import -ignore = F841 W503 E721 E722 E226 +ignore = F841 W503 E721 E722 E226 E203 diff --git a/Containerfile b/Containerfile index 13f9b7a1..b35dc834 100644 --- a/Containerfile +++ b/Containerfile @@ -58,6 +58,7 @@ uv pip install --no-cache --system -r /src/requirements.ansible.txt uv pip install --no-cache --system -r /src/requirements.openstack-image-manager.txt uv pip install --no-cache --system -r /src/requirements.openstack-flavor-manager.txt uv pip install --no-cache --system -r /src/requirements.netbox-manager.txt +uv pip install --no-cache --system -r /src/requirements.sonic.txt # install python-osism uv pip install --no-cache --system /src diff --git a/Pipfile b/Pipfile index 56774cac..412b04e1 100644 --- a/Pipfile +++ b/Pipfile @@ -31,6 +31,7 @@ pottery = "==3.0.1" prompt-toolkit = "==3.0.52" pyang = "==2.7.1" pynetbox = "==7.6.1" +sonic-yang-mgmt = {git = "https://github.com/sonic-net/sonic-buildimage.git", ref = "9ef3658e5315002006ea1d6da2524ad14f77c928", subdirectory = "src/sonic-yang-mgmt"} pytest-testinfra = "==10.2.2" python-dateutil = "==2.9.0.post0" redfish = "==3.3.5" diff --git a/files/sonic/yang_models/sonic-extension.yang b/files/sonic/yang_models/sonic-extension.yang new file mode 100644 index 00000000..79e80f86 --- /dev/null +++ b/files/sonic/yang_models/sonic-extension.yang @@ -0,0 +1,25 @@ +module sonic-extension { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-extension"; + prefix sonic-extension; + + description "Extension yang Module for SONiC OS"; + + revision 2019-07-01 { + description "First Revision"; + } + + /* For complete guide of using these extensions in SONiC yangs, refer + SONiC yang guidelines at + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md */ + + extension db-name { + description "DB name, e.g. APPL_DB, CONFIG_DB"; + argument "value"; + } + + + +} diff --git a/files/sonic/yang_models/sonic-passwh.yang b/files/sonic/yang_models/sonic-passwh.yang old mode 100755 new mode 100644 diff --git a/files/sonic/yang_models/sonic-policer.yang b/files/sonic/yang_models/sonic-policer.yang new file mode 100644 index 00000000..6c88d866 --- /dev/null +++ b/files/sonic/yang_models/sonic-policer.yang @@ -0,0 +1,125 @@ + +/* this is sonic py yang model */ + +module sonic-policer { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-policer"; + prefix policer; + + import sonic-types { + prefix stypes; + } + + description "Policer YANG Module for SONiC OS"; + + revision 2022-02-03 { + description + "First Revision"; + } + + container sonic-policer { + container POLICER { + list POLICER_LIST { + key name; + + leaf name { + type string; + description "Policer name"; + } + + leaf meter_type { + mandatory true; + type stypes:meter_type; + description "Policer meter type"; + } + + leaf mode { + mandatory true; + type stypes:policer_mode; + description "Policer mode"; + } + + leaf color { + type stypes:policer_color_source; + description "Policer color Source"; + } + + leaf cir { + type uint64; + description + "Committed information rate for the dual-rate token + bucket policer. This value represents the rate at which + tokens are added to the primary bucket. Unit is bytes/sec + or packets/sec based on meter_type"; + } + + leaf cbs { + must "((current()/../cir) and (current()/../cir > 0))" { + error-message "cbs can't be configured without cir."; + } + must "(current() >= current()/../cir)" { + error-message "cbs must be greater than or equal to cir"; + } + type uint64; + description + "Committed burst size for the dual-rate token bucket + policer. This value represents the depth of the token + bucket. Unit is bytes or packets based on meter_type"; + } + + leaf pir { + when "current()/../mode = 'tr_tcm'"; + must "((current()/../cir) and (current()/../cir > 0))" { + error-message "pir can't be configured without cir."; + } + must "(current() >= current()/../cir)" { + error-message "pir must be greater than or equal to cir"; + } + type uint64; + description + "Peak information rate for the dual-rate token bucket + policer. This value represents the rate at which tokens + are added to the secondary bucket. Unit is bytes/sec or + packets/sec based on meter_type"; + } + + leaf pbs { + when "((current()/../mode = 'sr_tcm') or (current()/../mode = 'tr_tcm'))"; + must "((not(current()/../cbs)) or (current() >= current()/../cbs))" { + error-message "pbs must be greater than or equal to cbs"; + } + type uint64; + description + "Excess burst size for the dual-rate token bucket policer. + This value represents the depth of the secondary bucket. Unit + is bytes or packets based on meter_type"; + } + + leaf green_packet_action { + type stypes:policer_packet_action; + default "forward"; + description "Green action"; + } + + leaf yellow_packet_action { + when "((current()/../mode = 'sr_tcm') or (current()/../mode = 'tr_tcm'))"; + type stypes:policer_packet_action; + default "forward"; + description "Yellow action"; + } + + leaf red_packet_action { + type stypes:policer_packet_action; + default "forward"; + description "Red action"; + } + } + /* end of list POLICER_LIST */ + } + /* end of container POLICER */ + } + /* end of top level container */ +} +/* end of module sonic-policer */ diff --git a/files/sonic/yang_models/sonic-types.yang b/files/sonic/yang_models/sonic-types.yang new file mode 100644 index 00000000..8edaa3cd --- /dev/null +++ b/files/sonic/yang_models/sonic-types.yang @@ -0,0 +1,524 @@ +module sonic-types { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-head"; + prefix sonic-types; + + description "SONiC type for yang Models of SONiC OS"; + /* + * Try to define only sonic specific types here. Rest can be written in + * respective YANG files. + */ + + revision 2019-07-01 { + description "First Revision"; + } + + typedef ip-family { + type enumeration { + enum IPv4; + enum IPv6; + } + } + + typedef sonic-ip4-prefix { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '/(([0-9])|([1-2][0-9])|(3[0-2]))'; + } + } + + typedef sonic-ip6-prefix { + type string { + pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}' + + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|' + + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}' + + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))' + + '(/(([0-9])|([0-9]{2})|(1[0-1][0-9])|(12[0-8])))'; + pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|' + + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)' + + '(/.+)'; + } + } + + typedef sonic-ip-prefix { + type union { + type sonic-ip4-prefix; + type sonic-ip6-prefix; + } + } + + typedef ipv4-address-list { + type string { + pattern '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '(%[\p{N}\p{L}]+)?' + + '(,(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '(%[\p{N}\p{L}]+)?)*'; + } + description + "Comma-separated IPv4 addresses"; + } + + typedef admin_status { + type enumeration { + enum up; + enum down; + } + } + + typedef packet_action{ + type enumeration { + enum DROP; + enum ACCEPT; + enum FORWARD; + enum REDIRECT; + enum DO_NOT_NAT; + enum DISABLE_TRIM; + } + } + + typedef ip_type { + type enumeration { + enum ANY; + enum IP; + enum NON_IP; + enum IPV4; + enum IPV6; + enum IPv4ANY; + enum IPV4ANY; + enum NON_IP4; + enum IPv6ANY; + enum IPV6ANY; + enum NON_IPv6; + enum ARP; + } + } + + typedef acl_table_type { + type enumeration { + enum L2; + enum L3; + enum L3V6; + enum L3V4V6; + enum MIRROR; + enum MIRRORV6; + enum MIRROR_DSCP; + enum CTRLPLANE; + } + } + + typedef hwsku { + type string { + length 1..255; + /* Should we list all hwsku here */ + } + } + + typedef vlan_tagging_mode { + type enumeration { + enum tagged; + enum untagged; + enum priority_tagged; + } + } + + typedef crm_threshold_type { + type string { + length 1..64; + pattern "percentage|used|free|PERCENTAGE|USED|FREE"; + } + } + + typedef loopback_action { + type string { + pattern "drop|forward"; + } + } + + typedef admin_mode { + type enumeration { + enum enabled; + enum disabled; + } + } + + typedef ip-protocol-type { + type enumeration { + enum TCP; + enum UDP; + } + } + + typedef interface_type { + type enumeration { + enum CR; + enum CR2; + enum CR4; + enum CR8; + enum SR; + enum SR2; + enum SR4; + enum SR8; + enum LR; + enum LR4; + enum LR8; + enum KR; + enum KR4; + enum KR8; + enum CAUI; + enum GMII; + enum SFI; + enum XLAUI; + enum KR2; + enum CAUI4; + enum XAUI; + enum XFI; + enum XGMII; + enum none; + } + } + + typedef oper-status { + type enumeration { + enum unknown; + enum up; + enum down; + } + description "Operational status of an entity such as Port, MCLAG etc"; + } + + typedef mode-status { + type enumeration { + enum enable; + enum disable; + } + description + "This type can be used where toggle functionality required. + For ex. IPv6-link-local-only, Dhcp-replay-link-select, SNMP traps etc"; + } + + typedef dhcp-relay-policy-action { + type enumeration { + enum discard; + enum append; + enum replace; + } + description "DHCP relay policy action value"; + } + + typedef percentage { + type uint8 { + range "0..100"; + } + description + "Integer indicating a percentage value"; + } + + typedef tpid_type { + type string { + pattern "0x8100|0x9100|0x9200|0x88a8|0x88A8"; + } + } + + typedef switchport_mode { + type string { + pattern "routed|access|trunk"; + } + description + "SwitchPort Modes for Port & PortChannel"; + } + + typedef meter_type { + type enumeration { + enum packets; + enum bytes; + } + } + + typedef policer_mode { + type enumeration { + enum sr_tcm; + enum tr_tcm; + enum storm; + } + } + + typedef policer_color_source { + type enumeration { + enum aware; + enum blind; + } + } + + + typedef policer_packet_action { + type enumeration { + enum drop; + enum forward; + enum copy; + enum copy_cancel; + enum trap; + enum log; + enum deny; + enum transit; + } + } + + typedef boolean_type { + type string { + pattern "false|true|False|True"; + } + } + + typedef mac-addr-and-mask { + type string { + pattern "[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}|[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}/[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}"; + } + } + + typedef mac-address-list { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5})(,([0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}))*'; + } + description + "Comma-separated MAC addresses"; + } + + typedef hostname { + type string { + length 1..63; + } + } + + typedef vnid_type { + type uint32 { + range "1..16777215"; + } + description + "VXLAN Network Identifier"; + } + + typedef vnid-list { + type string { + pattern '([1-9][0-9]{0,6}|1[0-5][0-9]{6}|16[0-6][0-9]{5}|167[0-6][0-9]{4}|1677[0-6][0-9]{3}|16777[01][0-9]{2}|1677720[0-9]|1677721[0-5])' + + '(,([1-9][0-9]{0,6}|1[0-5][0-9]{6}|16[0-6][0-9]{5}|167[0-6][0-9]{4}|1677[0-6][0-9]{3}|16777[01][0-9]{2}|1677720[0-9]|1677721[0-5]))*'; + } + description + "Comma-separated VNIs"; + } + + typedef tc_type { + type uint8 { + range "0..15" { + error-message "Invalid Traffic Class"; + error-app-tag tc-invalid; + } + } + } + + typedef process_name { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,31})' { + error-message "Invalid process_name."; + error-app-tag invalid-process-name; + } + length 1..32 { + error-message "Invalid length for process_name."; + error-app-tag invalid-process-name-length; + } + } + } + + typedef ctr_name { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,31})' { + error-message "Invalid ctr_name."; + error-app-tag invalid-ctr-name; + } + length 1..32 { + error-message "Invalid length for ctr_name."; + error-app-tag invalid-ctr-name-length; + } + } + } + + typedef interface_name { + description "Represents IFNAMSIZ defined in "; + type string { + length 1..15 { + error-message "Invalid interface name length, it must not exceed 16 characters."; + error-app-tag invalid-interface-name-length; + } + } + } + + typedef hash-field { + description "Represents native hash field"; + type enumeration { + enum IN_PORT; + enum DST_MAC; + enum SRC_MAC; + enum ETHERTYPE; + enum VLAN_ID; + enum IP_PROTOCOL; + enum DST_IP; + enum SRC_IP; + enum L4_DST_PORT; + enum L4_SRC_PORT; + enum INNER_DST_MAC; + enum INNER_SRC_MAC; + enum INNER_ETHERTYPE; + enum INNER_IP_PROTOCOL; + enum INNER_DST_IP; + enum INNER_DST_IPV4; + enum INNER_DST_IPV6; + enum INNER_SRC_IP; + enum INNER_SRC_IPV4; + enum INNER_SRC_IPV6; + enum INNER_L4_DST_PORT; + enum INNER_L4_SRC_PORT; + enum IPV6_FLOW_LABEL; + } + } + + typedef hash-algorithm { + description "Represents hash algorithm"; + type enumeration { + enum CRC; + enum XOR; + enum RANDOM; + enum CRC_32LO; + enum CRC_32HI; + enum CRC_CCITT; + enum CRC_XOR; + } + } + + typedef timezone-name-type { + type string; + description + "A time zone name as used by the Time Zone Database, + sometimes referred to as the 'Olson Database'. + + The exact set of valid values is an implementation-specific + matter. Client discovery of the exact set of time zone names + for a particular server is out of scope."; + reference + "BCP 175: Procedures for Maintaining the Time Zone Database"; + } + + typedef yes-no { + description "Yes/No configuration"; + type enumeration { + enum yes; + enum no; + } + } + + typedef on-off { + description "On/Off configuration"; + type enumeration { + enum on; + enum off; + } + } + + typedef asic_name { + type string { + pattern '[Aa][Ss][Ii][Cc][0-9]{1,2}'; + } + } + + typedef debug_counter_type { + type enumeration { + enum PORT_INGRESS_DROPS; + enum PORT_EGRESS_DROPS; + enum SWITCH_INGRESS_DROPS; + enum SWITCH_EGRESS_DROPS; + } + } + + typedef counter_drop_reason { + type enumeration { + enum L2_ANY; + enum SMAC_MULTICAST; + enum SMAC_EQUALS_DMAC; + enum DMAC_RESERVED; + enum VLAN_TAG_NOT_ALLOWED; + enum INGRESS_VLAN_FILTER; + enum INGRESS_STP_FILTER; + enum FDB_UC_DISCARD; + enum FDB_MC_DISCARD; + enum L2_LOOPBACK_FILTER; + enum EXCEEDS_L2_MTU; + enum L3_ANY; + enum EXCEEDS_L3_MTU; + enum TTL; + enum L3_LOOPBACK_FILTER; + enum NON_ROUTABLE; + enum NO_L3_HEADER; + enum IP_HEADER_ERROR; + enum UC_DIP_MC_DMAC; + enum DIP_LOOPBACK; + enum SIP_LOOPBACK; + enum SIP_MC; + enum SIP_CLASS_E; + enum SIP_UNSPECIFIED; + enum MC_DMAC_MISMATCH; + enum SIP_EQUALS_DIP; + enum SIP_BC; + enum DIP_LOCAL; + enum DIP_LINK_LOCAL; + enum SIP_LINK_LOCAL; + enum IPV6_MC_SCOPE0; + enum IPV6_MC_SCOPE1; + enum IRIF_DISABLED; + enum ERIF_DISABLED; + enum LPM4_MISS; + enum LPM6_MISS; + enum BLACKHOLE_ROUTE; + enum BLACKHOLE_ARP; + enum UNRESOLVED_NEXT_HOP; + enum L3_EGRESS_LINK_DOWN; + enum DECAP_ERROR; + enum ACL_ANY; + enum ACL_INGRESS_PORT; + enum ACL_INGRESS_LAG; + enum ACL_INGRESS_VLAN; + enum ACL_INGRESS_RIF; + enum ACL_INGRESS_SWITCH; + enum ACL_EGRESS_PORT; + enum ACL_EGRESS_LAG; + enum ACL_EGRESS_VLAN; + enum ACL_EGRESS_RIF; + enum ACL_EGRESS_SWITCH; + enum EGRESS_VLAN_FILTER; + } + } + + typedef relay-agent-mode { + type enumeration { + enum forward_and_append { + description "Forward and append our own relay option"; + } + enum forward_and_replace { + description "Forward, but replace theirs with ours"; + } + enum forward_untouched { + description "Forward without changes"; + } + enum discard { + description "Discard the packet"; + } + } + description "This enumeration type defines what to do about a dhcp packets that already has a relay option."; + } + + + +} diff --git a/osism/commands/sonic.py b/osism/commands/sonic.py index b73fa81d..b322ccf5 100644 --- a/osism/commands/sonic.py +++ b/osism/commands/sonic.py @@ -1136,3 +1136,248 @@ def take_action(self, parsed_args): except Exception as e: logger.error(f"Error listing SONiC devices: {e}") return 1 + + +class Validate(SonicCommandBase): + """Validate SONiC config_db.json against the bundled YANG models. + + Configurations can be sourced from a local file, NetBox local context, + the on-disk export directory, or generated on-the-fly from NetBox. + """ + + def get_parser(self, prog_name): + parser = super(Validate, self).get_parser(prog_name) + parser.add_argument( + "hostname", + nargs="*", + default=[], + type=str, + help=( + "One or more hostnames. Required for --from-netbox / --generate; " + "optional for --from-export-dir (filters by filename)." + ), + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--file", + dest="file", + type=str, + help="Validate a single config_db.json file at this path.", + ) + source.add_argument( + "--from-netbox", + dest="from_netbox", + action="store_true", + help="Validate the sonic_config stored in NetBox local_context_data.", + ) + source.add_argument( + "--from-export-dir", + dest="from_export_dir", + nargs="?", + const="", + default=None, + help=( + "Validate config files from the SONiC export directory " + "(default: SONIC_EXPORT_DIR setting)." + ), + ) + source.add_argument( + "--generate", + dest="generate", + action="store_true", + help="Generate config from NetBox in-memory and validate it.", + ) + parser.add_argument( + "--yang-dir", + dest="yang_dir", + type=str, + default=None, + help="Override YANG model directory (default: SONIC_YANG_MODELS_DIR).", + ) + parser.add_argument( + "--format", + dest="output_format", + choices=["text", "json"], + default="text", + help="Output format (default: text).", + ) + return parser + + def take_action(self, parsed_args): + try: + from osism.tasks.conductor.sonic.validator import ( + ValidatorUnavailable, + load_yang_context, + validate_config, + ) + except ImportError as exc: + logger.error(f"Validator module unavailable: {exc}") + return 2 + + try: + ctx = load_yang_context(parsed_args.yang_dir) + except ValidatorUnavailable as exc: + logger.error(str(exc)) + return 2 + except Exception as exc: + logger.error(f"Failed to load YANG models: {exc}") + return 2 + + try: + sources = self._collect_sources(parsed_args) + except ValueError as exc: + logger.error(str(exc)) + return 2 + + if not sources: + logger.error("No configurations found to validate.") + return 2 + + results = [] + worst_rc = 0 + for label, config in sources: + if config is None: + results.append((label, None)) + worst_rc = max(worst_rc, 2) + continue + result = validate_config(config, ctx=ctx) + results.append((label, result)) + if not result.valid: + worst_rc = max(worst_rc, 1) + + if parsed_args.output_format == "json": + payload = { + label: ( + result.to_dict() + if result + else { + "valid": False, + "errors": [{"message": "config not available"}], + } + ) + for label, result in results + } + print(json.dumps(payload, indent=2)) + else: + self._print_text_report(results) + + return worst_rc + + def _collect_sources(self, parsed_args): + """Return a list of (label, config_dict_or_None) tuples to validate.""" + if parsed_args.file: + with open(parsed_args.file, "r") as fh: + return [(parsed_args.file, json.load(fh))] + + if parsed_args.from_netbox: + if not parsed_args.hostname: + raise ValueError("--from-netbox requires at least one hostname.") + return [ + (hostname, self._config_from_netbox(hostname)) + for hostname in parsed_args.hostname + ] + + if parsed_args.from_export_dir is not None: + from osism import settings + + export_dir = parsed_args.from_export_dir or settings.SONIC_EXPORT_DIR + return self._configs_from_export_dir(export_dir, parsed_args.hostname) + + if parsed_args.generate: + if not parsed_args.hostname: + raise ValueError("--generate requires at least one hostname.") + return [ + (f"{hostname} (generated)", self._config_from_generate(hostname)) + for hostname in parsed_args.hostname + ] + + raise ValueError("No configuration source specified.") + + def _config_from_netbox(self, hostname): + device = self._get_device_from_netbox(hostname) + if not device: + return None + ctx = self._get_config_context(device, hostname) + if not ctx: + return None + config = ctx.get("sonic_config") + if not config: + logger.error( + f"Device {hostname} has no 'sonic_config' in local_context_data." + ) + return None + return config + + def _configs_from_export_dir(self, export_dir, hostnames): + if not os.path.isdir(export_dir): + raise ValueError(f"Export directory not found: {export_dir}") + + from osism import settings + + prefix = settings.SONIC_EXPORT_PREFIX + suffix = settings.SONIC_EXPORT_SUFFIX + + wanted = set(hostnames) if hostnames else None + sources = [] + for name in sorted(os.listdir(export_dir)): + if not (name.startswith(prefix) and name.endswith(suffix)): + continue + identifier = ( + name[len(prefix) : -len(suffix)] if suffix else name[len(prefix) :] + ) + if wanted is not None and identifier not in wanted: + continue + path = os.path.join(export_dir, name) + try: + with open(path, "r") as fh: + sources.append((path, json.load(fh))) + except (OSError, json.JSONDecodeError) as exc: + logger.error(f"Could not read {path}: {exc}") + sources.append((path, None)) + return sources + + def _config_from_generate(self, hostname): + from osism.tasks.conductor.sonic.config_generator import generate_sonic_config + from osism.tasks.conductor.sonic.constants import SUPPORTED_HWSKUS + + device = self._get_device_from_netbox(hostname) + if not device: + return None + hwsku = None + if ( + hasattr(device, "custom_fields") + and device.custom_fields.get("sonic_parameters") + and device.custom_fields["sonic_parameters"].get("hwsku") + ): + hwsku = device.custom_fields["sonic_parameters"]["hwsku"] + if not hwsku: + logger.error(f"Device {hostname} has no HWSKU configured.") + return None + if hwsku not in SUPPORTED_HWSKUS: + logger.error( + f"Device {hostname} HWSKU '{hwsku}' is not supported " + f"(supported: {', '.join(SUPPORTED_HWSKUS)})." + ) + return None + config_version = None + if device.custom_fields.get("sonic_parameters", {}).get("config_version"): + config_version = device.custom_fields["sonic_parameters"]["config_version"] + return generate_sonic_config(device, hwsku, None, config_version) + + def _print_text_report(self, results): + ok = sum(1 for _, r in results if r and r.valid) + fail = sum(1 for _, r in results if r is None or not r.valid) + for label, result in results: + if result is None: + print(f"[ERROR] {label}: configuration not available") + continue + if result.valid: + print(f"[OK] {label}") + else: + print(f"[FAIL] {label}: {len(result.errors)} error(s)") + for err in result.errors: + if err.path: + print(f" - {err.message} ({err.path})") + else: + print(f" - {err.message}") + print(f"\nSummary: {ok} valid, {fail} failed, {len(results)} total") diff --git a/osism/settings.py b/osism/settings.py index 3e893829..44ff6541 100644 --- a/osism/settings.py +++ b/osism/settings.py @@ -58,6 +58,13 @@ def read_secret(secret_name): SONIC_EXPORT_SUFFIX = os.getenv("SONIC_EXPORT_SUFFIX", "_config_db.json") SONIC_EXPORT_IDENTIFIER = os.getenv("SONIC_EXPORT_IDENTIFIER", "serial-number") +SONIC_YANG_MODELS_DIR = os.getenv( + "SONIC_YANG_MODELS_DIR", + os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "files", "sonic", "yang_models") + ), +) + NETBOX_SECONDARIES = ( os.getenv("NETBOX_SECONDARIES", read_secret("NETBOX_SECONDARIES")) or "[]" ) diff --git a/osism/tasks/conductor/sonic/validator.py b/osism/tasks/conductor/sonic/validator.py new file mode 100644 index 00000000..69081cc3 --- /dev/null +++ b/osism/tasks/conductor/sonic/validator.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""YANG-based validation for SONiC config_db.json configurations. + +Uses sonic-yang-mgmt (which wraps libyang) to validate that a generated +SONiC ConfigDB JSON conforms to the bundled SONiC YANG models. The library +performs the ConfigDB-table → YANG-tree translation internally and then +runs full schema validation including types, leafrefs, must/when constraints +and mandatory leaves. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from loguru import logger + +from osism import settings + + +@dataclass +class ValidationError: + message: str + path: Optional[str] = None + table: Optional[str] = None + + +@dataclass +class ValidationResult: + valid: bool + errors: List[ValidationError] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return { + "valid": self.valid, + "errors": [ + {"message": e.message, "path": e.path, "table": e.table} + for e in self.errors + ], + "warnings": list(self.warnings), + } + + +class ValidatorUnavailable(RuntimeError): + """Raised when the underlying sonic-yang-mgmt library cannot be imported.""" + + +def _import_sonic_yang(): + try: + import sonic_yang # type: ignore + except ImportError as exc: + raise ValidatorUnavailable( + "sonic-yang-mgmt is not importable. It is pinned in " + "requirements.sonic.txt as a VCS install from sonic-buildimage; " + "ensure pip installed it and that the libyang C library plus its " + "Python binding (apt: libyang-dev + python3-yang) are present on " + "the system." + ) from exc + return sonic_yang + + +def load_yang_context(yang_dir: Optional[str] = None): + """Load all SONiC YANG models from yang_dir into a SonicYang context. + + Args: + yang_dir: Directory containing sonic-*.yang files. Defaults to + settings.SONIC_YANG_MODELS_DIR. + + Returns: + sonic_yang.SonicYang: a context with all models loaded. + """ + sonic_yang = _import_sonic_yang() + + yang_dir = yang_dir or settings.SONIC_YANG_MODELS_DIR + logger.debug(f"Loading SONiC YANG models from {yang_dir}") + + ctx = sonic_yang.SonicYang(yang_dir, print_log_enabled=False) + ctx.loadYangModel() + return ctx + + +def validate_config( + config: Dict[str, Any], ctx=None, yang_dir: Optional[str] = None +) -> ValidationResult: + """Validate a SONiC config_db.json dict against the SONiC YANG models. + + Args: + config: The ConfigDB JSON as a dict (top-level keys are tables). + ctx: Optional pre-loaded SonicYang context (reuse for batches). + yang_dir: Override the YANG model directory; ignored when ctx is given. + + Returns: + ValidationResult with valid flag and any collected errors. + """ + sonic_yang = _import_sonic_yang() + + if ctx is None: + ctx = load_yang_context(yang_dir) + + try: + ctx.loadData(configdbJson=config) + ctx.validate_data_tree() + except sonic_yang.SonicYangException as exc: + message = str(exc) + errors = _split_libyang_errors(message) + if not errors: + errors = [ValidationError(message=message)] + return ValidationResult(valid=False, errors=errors) + except Exception as exc: + return ValidationResult( + valid=False, + errors=[ValidationError(message=f"Unexpected validator error: {exc}")], + ) + + return ValidationResult(valid=True) + + +def _split_libyang_errors(raw: str) -> List[ValidationError]: + """Best-effort parse of the multi-line error blob returned by libyang. + + libyang concatenates multiple errors into one string via SonicYangException; + we split on newlines and try to extract a path hint when present. + """ + errors: List[ValidationError] = [] + for line in (line.strip() for line in raw.splitlines()): + if not line: + continue + path: Optional[str] = None + message = line + marker = "Schema location:" + if marker in line: + head, tail = line.split(marker, 1) + message = head.strip().rstrip(",;.") + path = tail.strip().split(",", 1)[0].strip() + errors.append(ValidationError(message=message, path=path)) + return errors diff --git a/playbooks/test-setup.yml b/playbooks/test-setup.yml index 448e3fd2..a7542d42 100644 --- a/playbooks/test-setup.yml +++ b/playbooks/test-setup.yml @@ -15,4 +15,4 @@ export PATH=$PATH:$HOME/.local/bin pipenv install - pipenv run pip install . + pipenv run pip install --no-deps . diff --git a/requirements.sonic.txt b/requirements.sonic.txt new file mode 100644 index 00000000..c72fe0ca --- /dev/null +++ b/requirements.sonic.txt @@ -0,0 +1 @@ +sonic-yang-mgmt @ git+https://github.com/sonic-net/sonic-buildimage.git@9ef3658e5315002006ea1d6da2524ad14f77c928#subdirectory=src/sonic-yang-mgmt diff --git a/requirements.txt b/requirements.txt index 296c3f01..8c455c0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,8 @@ redfish==3.3.5 setuptools==82.0.1 sqlmodel==0.0.38 sushy==5.10.0 -tabulate==0.10.0 +# pinned to 0.9.0 because sonic-yang-mgmt requires tabulate==0.9.0 +tabulate==0.9.0 transitions==0.9.3 uvicorn[standard]==0.42.0 validators==0.35.0 diff --git a/setup.cfg b/setup.cfg index 5393e530..7b865e3b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -143,6 +143,7 @@ osism.commands: sonic reset = osism.commands.sonic:Reset sonic show = osism.commands.sonic:Show sonic sync= osism.commands.sync:Sonic + sonic validate = osism.commands.sonic:Validate sonic ztp = osism.commands.sonic:Ztp sync ceph-keys = osism.commands.sync:CephKeys sync configuration = osism.commands.configuration:Sync