diff --git a/.flake8 b/.flake8 index a83721ad..5ed3c01f 100644 --- a/.flake8 +++ b/.flake8 @@ -5,4 +5,4 @@ exclude = files/disable files/generate files/import -ignore = F841 W503 E721 E722 E226 +ignore = F841 W503 E721 E722 E226 E203 diff --git a/Containerfile b/Containerfile index 13f9b7a1..b35dc834 100644 --- a/Containerfile +++ b/Containerfile @@ -58,6 +58,7 @@ uv pip install --no-cache --system -r /src/requirements.ansible.txt uv pip install --no-cache --system -r /src/requirements.openstack-image-manager.txt uv pip install --no-cache --system -r /src/requirements.openstack-flavor-manager.txt uv pip install --no-cache --system -r /src/requirements.netbox-manager.txt +uv pip install --no-cache --system -r /src/requirements.sonic.txt # install python-osism uv pip install --no-cache --system /src diff --git a/Pipfile b/Pipfile index 56774cac..412b04e1 100644 --- a/Pipfile +++ b/Pipfile @@ -31,6 +31,7 @@ pottery = "==3.0.1" prompt-toolkit = "==3.0.52" pyang = "==2.7.1" pynetbox = "==7.6.1" +sonic-yang-mgmt = {git = "https://github.com/sonic-net/sonic-buildimage.git", ref = "9ef3658e5315002006ea1d6da2524ad14f77c928", subdirectory = "src/sonic-yang-mgmt"} pytest-testinfra = "==10.2.2" python-dateutil = "==2.9.0.post0" redfish = "==3.3.5" diff --git a/files/sonic/yang_models/sonic-extension.yang b/files/sonic/yang_models/sonic-extension.yang new file mode 100644 index 00000000..79e80f86 --- /dev/null +++ b/files/sonic/yang_models/sonic-extension.yang @@ -0,0 +1,25 @@ +module sonic-extension { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-extension"; + prefix sonic-extension; + + description "Extension yang Module for SONiC OS"; + + revision 2019-07-01 { + description "First Revision"; + } + + /* For complete guide of using these extensions in SONiC yangs, refer + SONiC yang guidelines at + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md */ + + extension db-name { + description "DB name, e.g. APPL_DB, CONFIG_DB"; + argument "value"; + } + + + +} diff --git a/files/sonic/yang_models/sonic-passwh.yang b/files/sonic/yang_models/sonic-passwh.yang old mode 100755 new mode 100644 diff --git a/files/sonic/yang_models/sonic-policer.yang b/files/sonic/yang_models/sonic-policer.yang new file mode 100644 index 00000000..6c88d866 --- /dev/null +++ b/files/sonic/yang_models/sonic-policer.yang @@ -0,0 +1,125 @@ + +/* this is sonic py yang model */ + +module sonic-policer { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-policer"; + prefix policer; + + import sonic-types { + prefix stypes; + } + + description "Policer YANG Module for SONiC OS"; + + revision 2022-02-03 { + description + "First Revision"; + } + + container sonic-policer { + container POLICER { + list POLICER_LIST { + key name; + + leaf name { + type string; + description "Policer name"; + } + + leaf meter_type { + mandatory true; + type stypes:meter_type; + description "Policer meter type"; + } + + leaf mode { + mandatory true; + type stypes:policer_mode; + description "Policer mode"; + } + + leaf color { + type stypes:policer_color_source; + description "Policer color Source"; + } + + leaf cir { + type uint64; + description + "Committed information rate for the dual-rate token + bucket policer. This value represents the rate at which + tokens are added to the primary bucket. Unit is bytes/sec + or packets/sec based on meter_type"; + } + + leaf cbs { + must "((current()/../cir) and (current()/../cir > 0))" { + error-message "cbs can't be configured without cir."; + } + must "(current() >= current()/../cir)" { + error-message "cbs must be greater than or equal to cir"; + } + type uint64; + description + "Committed burst size for the dual-rate token bucket + policer. This value represents the depth of the token + bucket. Unit is bytes or packets based on meter_type"; + } + + leaf pir { + when "current()/../mode = 'tr_tcm'"; + must "((current()/../cir) and (current()/../cir > 0))" { + error-message "pir can't be configured without cir."; + } + must "(current() >= current()/../cir)" { + error-message "pir must be greater than or equal to cir"; + } + type uint64; + description + "Peak information rate for the dual-rate token bucket + policer. This value represents the rate at which tokens + are added to the secondary bucket. Unit is bytes/sec or + packets/sec based on meter_type"; + } + + leaf pbs { + when "((current()/../mode = 'sr_tcm') or (current()/../mode = 'tr_tcm'))"; + must "((not(current()/../cbs)) or (current() >= current()/../cbs))" { + error-message "pbs must be greater than or equal to cbs"; + } + type uint64; + description + "Excess burst size for the dual-rate token bucket policer. + This value represents the depth of the secondary bucket. Unit + is bytes or packets based on meter_type"; + } + + leaf green_packet_action { + type stypes:policer_packet_action; + default "forward"; + description "Green action"; + } + + leaf yellow_packet_action { + when "((current()/../mode = 'sr_tcm') or (current()/../mode = 'tr_tcm'))"; + type stypes:policer_packet_action; + default "forward"; + description "Yellow action"; + } + + leaf red_packet_action { + type stypes:policer_packet_action; + default "forward"; + description "Red action"; + } + } + /* end of list POLICER_LIST */ + } + /* end of container POLICER */ + } + /* end of top level container */ +} +/* end of module sonic-policer */ diff --git a/files/sonic/yang_models/sonic-types.yang b/files/sonic/yang_models/sonic-types.yang new file mode 100644 index 00000000..8edaa3cd --- /dev/null +++ b/files/sonic/yang_models/sonic-types.yang @@ -0,0 +1,524 @@ +module sonic-types { + + yang-version 1.1; + + namespace "http://github.com/sonic-net/sonic-head"; + prefix sonic-types; + + description "SONiC type for yang Models of SONiC OS"; + /* + * Try to define only sonic specific types here. Rest can be written in + * respective YANG files. + */ + + revision 2019-07-01 { + description "First Revision"; + } + + typedef ip-family { + type enumeration { + enum IPv4; + enum IPv6; + } + } + + typedef sonic-ip4-prefix { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '/(([0-9])|([1-2][0-9])|(3[0-2]))'; + } + } + + typedef sonic-ip6-prefix { + type string { + pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}' + + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|' + + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}' + + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))' + + '(/(([0-9])|([0-9]{2})|(1[0-1][0-9])|(12[0-8])))'; + pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|' + + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)' + + '(/.+)'; + } + } + + typedef sonic-ip-prefix { + type union { + type sonic-ip4-prefix; + type sonic-ip6-prefix; + } + } + + typedef ipv4-address-list { + type string { + pattern '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '(%[\p{N}\p{L}]+)?' + + '(,(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '(%[\p{N}\p{L}]+)?)*'; + } + description + "Comma-separated IPv4 addresses"; + } + + typedef admin_status { + type enumeration { + enum up; + enum down; + } + } + + typedef packet_action{ + type enumeration { + enum DROP; + enum ACCEPT; + enum FORWARD; + enum REDIRECT; + enum DO_NOT_NAT; + enum DISABLE_TRIM; + } + } + + typedef ip_type { + type enumeration { + enum ANY; + enum IP; + enum NON_IP; + enum IPV4; + enum IPV6; + enum IPv4ANY; + enum IPV4ANY; + enum NON_IP4; + enum IPv6ANY; + enum IPV6ANY; + enum NON_IPv6; + enum ARP; + } + } + + typedef acl_table_type { + type enumeration { + enum L2; + enum L3; + enum L3V6; + enum L3V4V6; + enum MIRROR; + enum MIRRORV6; + enum MIRROR_DSCP; + enum CTRLPLANE; + } + } + + typedef hwsku { + type string { + length 1..255; + /* Should we list all hwsku here */ + } + } + + typedef vlan_tagging_mode { + type enumeration { + enum tagged; + enum untagged; + enum priority_tagged; + } + } + + typedef crm_threshold_type { + type string { + length 1..64; + pattern "percentage|used|free|PERCENTAGE|USED|FREE"; + } + } + + typedef loopback_action { + type string { + pattern "drop|forward"; + } + } + + typedef admin_mode { + type enumeration { + enum enabled; + enum disabled; + } + } + + typedef ip-protocol-type { + type enumeration { + enum TCP; + enum UDP; + } + } + + typedef interface_type { + type enumeration { + enum CR; + enum CR2; + enum CR4; + enum CR8; + enum SR; + enum SR2; + enum SR4; + enum SR8; + enum LR; + enum LR4; + enum LR8; + enum KR; + enum KR4; + enum KR8; + enum CAUI; + enum GMII; + enum SFI; + enum XLAUI; + enum KR2; + enum CAUI4; + enum XAUI; + enum XFI; + enum XGMII; + enum none; + } + } + + typedef oper-status { + type enumeration { + enum unknown; + enum up; + enum down; + } + description "Operational status of an entity such as Port, MCLAG etc"; + } + + typedef mode-status { + type enumeration { + enum enable; + enum disable; + } + description + "This type can be used where toggle functionality required. + For ex. IPv6-link-local-only, Dhcp-replay-link-select, SNMP traps etc"; + } + + typedef dhcp-relay-policy-action { + type enumeration { + enum discard; + enum append; + enum replace; + } + description "DHCP relay policy action value"; + } + + typedef percentage { + type uint8 { + range "0..100"; + } + description + "Integer indicating a percentage value"; + } + + typedef tpid_type { + type string { + pattern "0x8100|0x9100|0x9200|0x88a8|0x88A8"; + } + } + + typedef switchport_mode { + type string { + pattern "routed|access|trunk"; + } + description + "SwitchPort Modes for Port & PortChannel"; + } + + typedef meter_type { + type enumeration { + enum packets; + enum bytes; + } + } + + typedef policer_mode { + type enumeration { + enum sr_tcm; + enum tr_tcm; + enum storm; + } + } + + typedef policer_color_source { + type enumeration { + enum aware; + enum blind; + } + } + + + typedef policer_packet_action { + type enumeration { + enum drop; + enum forward; + enum copy; + enum copy_cancel; + enum trap; + enum log; + enum deny; + enum transit; + } + } + + typedef boolean_type { + type string { + pattern "false|true|False|True"; + } + } + + typedef mac-addr-and-mask { + type string { + pattern "[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}|[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}/[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}"; + } + } + + typedef mac-address-list { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5})(,([0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}))*'; + } + description + "Comma-separated MAC addresses"; + } + + typedef hostname { + type string { + length 1..63; + } + } + + typedef vnid_type { + type uint32 { + range "1..16777215"; + } + description + "VXLAN Network Identifier"; + } + + typedef vnid-list { + type string { + pattern '([1-9][0-9]{0,6}|1[0-5][0-9]{6}|16[0-6][0-9]{5}|167[0-6][0-9]{4}|1677[0-6][0-9]{3}|16777[01][0-9]{2}|1677720[0-9]|1677721[0-5])' + + '(,([1-9][0-9]{0,6}|1[0-5][0-9]{6}|16[0-6][0-9]{5}|167[0-6][0-9]{4}|1677[0-6][0-9]{3}|16777[01][0-9]{2}|1677720[0-9]|1677721[0-5]))*'; + } + description + "Comma-separated VNIs"; + } + + typedef tc_type { + type uint8 { + range "0..15" { + error-message "Invalid Traffic Class"; + error-app-tag tc-invalid; + } + } + } + + typedef process_name { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,31})' { + error-message "Invalid process_name."; + error-app-tag invalid-process-name; + } + length 1..32 { + error-message "Invalid length for process_name."; + error-app-tag invalid-process-name-length; + } + } + } + + typedef ctr_name { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,31})' { + error-message "Invalid ctr_name."; + error-app-tag invalid-ctr-name; + } + length 1..32 { + error-message "Invalid length for ctr_name."; + error-app-tag invalid-ctr-name-length; + } + } + } + + typedef interface_name { + description "Represents IFNAMSIZ defined in "; + type string { + length 1..15 { + error-message "Invalid interface name length, it must not exceed 16 characters."; + error-app-tag invalid-interface-name-length; + } + } + } + + typedef hash-field { + description "Represents native hash field"; + type enumeration { + enum IN_PORT; + enum DST_MAC; + enum SRC_MAC; + enum ETHERTYPE; + enum VLAN_ID; + enum IP_PROTOCOL; + enum DST_IP; + enum SRC_IP; + enum L4_DST_PORT; + enum L4_SRC_PORT; + enum INNER_DST_MAC; + enum INNER_SRC_MAC; + enum INNER_ETHERTYPE; + enum INNER_IP_PROTOCOL; + enum INNER_DST_IP; + enum INNER_DST_IPV4; + enum INNER_DST_IPV6; + enum INNER_SRC_IP; + enum INNER_SRC_IPV4; + enum INNER_SRC_IPV6; + enum INNER_L4_DST_PORT; + enum INNER_L4_SRC_PORT; + enum IPV6_FLOW_LABEL; + } + } + + typedef hash-algorithm { + description "Represents hash algorithm"; + type enumeration { + enum CRC; + enum XOR; + enum RANDOM; + enum CRC_32LO; + enum CRC_32HI; + enum CRC_CCITT; + enum CRC_XOR; + } + } + + typedef timezone-name-type { + type string; + description + "A time zone name as used by the Time Zone Database, + sometimes referred to as the 'Olson Database'. + + The exact set of valid values is an implementation-specific + matter. Client discovery of the exact set of time zone names + for a particular server is out of scope."; + reference + "BCP 175: Procedures for Maintaining the Time Zone Database"; + } + + typedef yes-no { + description "Yes/No configuration"; + type enumeration { + enum yes; + enum no; + } + } + + typedef on-off { + description "On/Off configuration"; + type enumeration { + enum on; + enum off; + } + } + + typedef asic_name { + type string { + pattern '[Aa][Ss][Ii][Cc][0-9]{1,2}'; + } + } + + typedef debug_counter_type { + type enumeration { + enum PORT_INGRESS_DROPS; + enum PORT_EGRESS_DROPS; + enum SWITCH_INGRESS_DROPS; + enum SWITCH_EGRESS_DROPS; + } + } + + typedef counter_drop_reason { + type enumeration { + enum L2_ANY; + enum SMAC_MULTICAST; + enum SMAC_EQUALS_DMAC; + enum DMAC_RESERVED; + enum VLAN_TAG_NOT_ALLOWED; + enum INGRESS_VLAN_FILTER; + enum INGRESS_STP_FILTER; + enum FDB_UC_DISCARD; + enum FDB_MC_DISCARD; + enum L2_LOOPBACK_FILTER; + enum EXCEEDS_L2_MTU; + enum L3_ANY; + enum EXCEEDS_L3_MTU; + enum TTL; + enum L3_LOOPBACK_FILTER; + enum NON_ROUTABLE; + enum NO_L3_HEADER; + enum IP_HEADER_ERROR; + enum UC_DIP_MC_DMAC; + enum DIP_LOOPBACK; + enum SIP_LOOPBACK; + enum SIP_MC; + enum SIP_CLASS_E; + enum SIP_UNSPECIFIED; + enum MC_DMAC_MISMATCH; + enum SIP_EQUALS_DIP; + enum SIP_BC; + enum DIP_LOCAL; + enum DIP_LINK_LOCAL; + enum SIP_LINK_LOCAL; + enum IPV6_MC_SCOPE0; + enum IPV6_MC_SCOPE1; + enum IRIF_DISABLED; + enum ERIF_DISABLED; + enum LPM4_MISS; + enum LPM6_MISS; + enum BLACKHOLE_ROUTE; + enum BLACKHOLE_ARP; + enum UNRESOLVED_NEXT_HOP; + enum L3_EGRESS_LINK_DOWN; + enum DECAP_ERROR; + enum ACL_ANY; + enum ACL_INGRESS_PORT; + enum ACL_INGRESS_LAG; + enum ACL_INGRESS_VLAN; + enum ACL_INGRESS_RIF; + enum ACL_INGRESS_SWITCH; + enum ACL_EGRESS_PORT; + enum ACL_EGRESS_LAG; + enum ACL_EGRESS_VLAN; + enum ACL_EGRESS_RIF; + enum ACL_EGRESS_SWITCH; + enum EGRESS_VLAN_FILTER; + } + } + + typedef relay-agent-mode { + type enumeration { + enum forward_and_append { + description "Forward and append our own relay option"; + } + enum forward_and_replace { + description "Forward, but replace theirs with ours"; + } + enum forward_untouched { + description "Forward without changes"; + } + enum discard { + description "Discard the packet"; + } + } + description "This enumeration type defines what to do about a dhcp packets that already has a relay option."; + } + + + +} diff --git a/osism/commands/sonic.py b/osism/commands/sonic.py index b73fa81d..b322ccf5 100644 --- a/osism/commands/sonic.py +++ b/osism/commands/sonic.py @@ -1136,3 +1136,248 @@ def take_action(self, parsed_args): except Exception as e: logger.error(f"Error listing SONiC devices: {e}") return 1 + + +class Validate(SonicCommandBase): + """Validate SONiC config_db.json against the bundled YANG models. + + Configurations can be sourced from a local file, NetBox local context, + the on-disk export directory, or generated on-the-fly from NetBox. + """ + + def get_parser(self, prog_name): + parser = super(Validate, self).get_parser(prog_name) + parser.add_argument( + "hostname", + nargs="*", + default=[], + type=str, + help=( + "One or more hostnames. Required for --from-netbox / --generate; " + "optional for --from-export-dir (filters by filename)." + ), + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--file", + dest="file", + type=str, + help="Validate a single config_db.json file at this path.", + ) + source.add_argument( + "--from-netbox", + dest="from_netbox", + action="store_true", + help="Validate the sonic_config stored in NetBox local_context_data.", + ) + source.add_argument( + "--from-export-dir", + dest="from_export_dir", + nargs="?", + const="", + default=None, + help=( + "Validate config files from the SONiC export directory " + "(default: SONIC_EXPORT_DIR setting)." + ), + ) + source.add_argument( + "--generate", + dest="generate", + action="store_true", + help="Generate config from NetBox in-memory and validate it.", + ) + parser.add_argument( + "--yang-dir", + dest="yang_dir", + type=str, + default=None, + help="Override YANG model directory (default: SONIC_YANG_MODELS_DIR).", + ) + parser.add_argument( + "--format", + dest="output_format", + choices=["text", "json"], + default="text", + help="Output format (default: text).", + ) + return parser + + def take_action(self, parsed_args): + try: + from osism.tasks.conductor.sonic.validator import ( + ValidatorUnavailable, + load_yang_context, + validate_config, + ) + except ImportError as exc: + logger.error(f"Validator module unavailable: {exc}") + return 2 + + try: + ctx = load_yang_context(parsed_args.yang_dir) + except ValidatorUnavailable as exc: + logger.error(str(exc)) + return 2 + except Exception as exc: + logger.error(f"Failed to load YANG models: {exc}") + return 2 + + try: + sources = self._collect_sources(parsed_args) + except ValueError as exc: + logger.error(str(exc)) + return 2 + + if not sources: + logger.error("No configurations found to validate.") + return 2 + + results = [] + worst_rc = 0 + for label, config in sources: + if config is None: + results.append((label, None)) + worst_rc = max(worst_rc, 2) + continue + result = validate_config(config, ctx=ctx) + results.append((label, result)) + if not result.valid: + worst_rc = max(worst_rc, 1) + + if parsed_args.output_format == "json": + payload = { + label: ( + result.to_dict() + if result + else { + "valid": False, + "errors": [{"message": "config not available"}], + } + ) + for label, result in results + } + print(json.dumps(payload, indent=2)) + else: + self._print_text_report(results) + + return worst_rc + + def _collect_sources(self, parsed_args): + """Return a list of (label, config_dict_or_None) tuples to validate.""" + if parsed_args.file: + with open(parsed_args.file, "r") as fh: + return [(parsed_args.file, json.load(fh))] + + if parsed_args.from_netbox: + if not parsed_args.hostname: + raise ValueError("--from-netbox requires at least one hostname.") + return [ + (hostname, self._config_from_netbox(hostname)) + for hostname in parsed_args.hostname + ] + + if parsed_args.from_export_dir is not None: + from osism import settings + + export_dir = parsed_args.from_export_dir or settings.SONIC_EXPORT_DIR + return self._configs_from_export_dir(export_dir, parsed_args.hostname) + + if parsed_args.generate: + if not parsed_args.hostname: + raise ValueError("--generate requires at least one hostname.") + return [ + (f"{hostname} (generated)", self._config_from_generate(hostname)) + for hostname in parsed_args.hostname + ] + + raise ValueError("No configuration source specified.") + + def _config_from_netbox(self, hostname): + device = self._get_device_from_netbox(hostname) + if not device: + return None + ctx = self._get_config_context(device, hostname) + if not ctx: + return None + config = ctx.get("sonic_config") + if not config: + logger.error( + f"Device {hostname} has no 'sonic_config' in local_context_data." + ) + return None + return config + + def _configs_from_export_dir(self, export_dir, hostnames): + if not os.path.isdir(export_dir): + raise ValueError(f"Export directory not found: {export_dir}") + + from osism import settings + + prefix = settings.SONIC_EXPORT_PREFIX + suffix = settings.SONIC_EXPORT_SUFFIX + + wanted = set(hostnames) if hostnames else None + sources = [] + for name in sorted(os.listdir(export_dir)): + if not (name.startswith(prefix) and name.endswith(suffix)): + continue + identifier = ( + name[len(prefix) : -len(suffix)] if suffix else name[len(prefix) :] + ) + if wanted is not None and identifier not in wanted: + continue + path = os.path.join(export_dir, name) + try: + with open(path, "r") as fh: + sources.append((path, json.load(fh))) + except (OSError, json.JSONDecodeError) as exc: + logger.error(f"Could not read {path}: {exc}") + sources.append((path, None)) + return sources + + def _config_from_generate(self, hostname): + from osism.tasks.conductor.sonic.config_generator import generate_sonic_config + from osism.tasks.conductor.sonic.constants import SUPPORTED_HWSKUS + + device = self._get_device_from_netbox(hostname) + if not device: + return None + hwsku = None + if ( + hasattr(device, "custom_fields") + and device.custom_fields.get("sonic_parameters") + and device.custom_fields["sonic_parameters"].get("hwsku") + ): + hwsku = device.custom_fields["sonic_parameters"]["hwsku"] + if not hwsku: + logger.error(f"Device {hostname} has no HWSKU configured.") + return None + if hwsku not in SUPPORTED_HWSKUS: + logger.error( + f"Device {hostname} HWSKU '{hwsku}' is not supported " + f"(supported: {', '.join(SUPPORTED_HWSKUS)})." + ) + return None + config_version = None + if device.custom_fields.get("sonic_parameters", {}).get("config_version"): + config_version = device.custom_fields["sonic_parameters"]["config_version"] + return generate_sonic_config(device, hwsku, None, config_version) + + def _print_text_report(self, results): + ok = sum(1 for _, r in results if r and r.valid) + fail = sum(1 for _, r in results if r is None or not r.valid) + for label, result in results: + if result is None: + print(f"[ERROR] {label}: configuration not available") + continue + if result.valid: + print(f"[OK] {label}") + else: + print(f"[FAIL] {label}: {len(result.errors)} error(s)") + for err in result.errors: + if err.path: + print(f" - {err.message} ({err.path})") + else: + print(f" - {err.message}") + print(f"\nSummary: {ok} valid, {fail} failed, {len(results)} total") diff --git a/osism/settings.py b/osism/settings.py index 3e893829..44ff6541 100644 --- a/osism/settings.py +++ b/osism/settings.py @@ -58,6 +58,13 @@ def read_secret(secret_name): SONIC_EXPORT_SUFFIX = os.getenv("SONIC_EXPORT_SUFFIX", "_config_db.json") SONIC_EXPORT_IDENTIFIER = os.getenv("SONIC_EXPORT_IDENTIFIER", "serial-number") +SONIC_YANG_MODELS_DIR = os.getenv( + "SONIC_YANG_MODELS_DIR", + os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "files", "sonic", "yang_models") + ), +) + NETBOX_SECONDARIES = ( os.getenv("NETBOX_SECONDARIES", read_secret("NETBOX_SECONDARIES")) or "[]" ) diff --git a/osism/tasks/conductor/sonic/validator.py b/osism/tasks/conductor/sonic/validator.py new file mode 100644 index 00000000..69081cc3 --- /dev/null +++ b/osism/tasks/conductor/sonic/validator.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""YANG-based validation for SONiC config_db.json configurations. + +Uses sonic-yang-mgmt (which wraps libyang) to validate that a generated +SONiC ConfigDB JSON conforms to the bundled SONiC YANG models. The library +performs the ConfigDB-table → YANG-tree translation internally and then +runs full schema validation including types, leafrefs, must/when constraints +and mandatory leaves. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from loguru import logger + +from osism import settings + + +@dataclass +class ValidationError: + message: str + path: Optional[str] = None + table: Optional[str] = None + + +@dataclass +class ValidationResult: + valid: bool + errors: List[ValidationError] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return { + "valid": self.valid, + "errors": [ + {"message": e.message, "path": e.path, "table": e.table} + for e in self.errors + ], + "warnings": list(self.warnings), + } + + +class ValidatorUnavailable(RuntimeError): + """Raised when the underlying sonic-yang-mgmt library cannot be imported.""" + + +def _import_sonic_yang(): + try: + import sonic_yang # type: ignore + except ImportError as exc: + raise ValidatorUnavailable( + "sonic-yang-mgmt is not importable. It is pinned in " + "requirements.sonic.txt as a VCS install from sonic-buildimage; " + "ensure pip installed it and that the libyang C library plus its " + "Python binding (apt: libyang-dev + python3-yang) are present on " + "the system." + ) from exc + return sonic_yang + + +def load_yang_context(yang_dir: Optional[str] = None): + """Load all SONiC YANG models from yang_dir into a SonicYang context. + + Args: + yang_dir: Directory containing sonic-*.yang files. Defaults to + settings.SONIC_YANG_MODELS_DIR. + + Returns: + sonic_yang.SonicYang: a context with all models loaded. + """ + sonic_yang = _import_sonic_yang() + + yang_dir = yang_dir or settings.SONIC_YANG_MODELS_DIR + logger.debug(f"Loading SONiC YANG models from {yang_dir}") + + ctx = sonic_yang.SonicYang(yang_dir, print_log_enabled=False) + ctx.loadYangModel() + return ctx + + +def validate_config( + config: Dict[str, Any], ctx=None, yang_dir: Optional[str] = None +) -> ValidationResult: + """Validate a SONiC config_db.json dict against the SONiC YANG models. + + Args: + config: The ConfigDB JSON as a dict (top-level keys are tables). + ctx: Optional pre-loaded SonicYang context (reuse for batches). + yang_dir: Override the YANG model directory; ignored when ctx is given. + + Returns: + ValidationResult with valid flag and any collected errors. + """ + sonic_yang = _import_sonic_yang() + + if ctx is None: + ctx = load_yang_context(yang_dir) + + try: + ctx.loadData(configdbJson=config) + ctx.validate_data_tree() + except sonic_yang.SonicYangException as exc: + message = str(exc) + errors = _split_libyang_errors(message) + if not errors: + errors = [ValidationError(message=message)] + return ValidationResult(valid=False, errors=errors) + except Exception as exc: + return ValidationResult( + valid=False, + errors=[ValidationError(message=f"Unexpected validator error: {exc}")], + ) + + return ValidationResult(valid=True) + + +def _split_libyang_errors(raw: str) -> List[ValidationError]: + """Best-effort parse of the multi-line error blob returned by libyang. + + libyang concatenates multiple errors into one string via SonicYangException; + we split on newlines and try to extract a path hint when present. + """ + errors: List[ValidationError] = [] + for line in (line.strip() for line in raw.splitlines()): + if not line: + continue + path: Optional[str] = None + message = line + marker = "Schema location:" + if marker in line: + head, tail = line.split(marker, 1) + message = head.strip().rstrip(",;.") + path = tail.strip().split(",", 1)[0].strip() + errors.append(ValidationError(message=message, path=path)) + return errors diff --git a/playbooks/test-setup.yml b/playbooks/test-setup.yml index 448e3fd2..a7542d42 100644 --- a/playbooks/test-setup.yml +++ b/playbooks/test-setup.yml @@ -15,4 +15,4 @@ export PATH=$PATH:$HOME/.local/bin pipenv install - pipenv run pip install . + pipenv run pip install --no-deps . diff --git a/requirements.sonic.txt b/requirements.sonic.txt new file mode 100644 index 00000000..c72fe0ca --- /dev/null +++ b/requirements.sonic.txt @@ -0,0 +1 @@ +sonic-yang-mgmt @ git+https://github.com/sonic-net/sonic-buildimage.git@9ef3658e5315002006ea1d6da2524ad14f77c928#subdirectory=src/sonic-yang-mgmt diff --git a/requirements.txt b/requirements.txt index 296c3f01..8c455c0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,8 @@ redfish==3.3.5 setuptools==82.0.1 sqlmodel==0.0.38 sushy==5.10.0 -tabulate==0.10.0 +# pinned to 0.9.0 because sonic-yang-mgmt requires tabulate==0.9.0 +tabulate==0.9.0 transitions==0.9.3 uvicorn[standard]==0.42.0 validators==0.35.0 diff --git a/setup.cfg b/setup.cfg index 5393e530..7b865e3b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -143,6 +143,7 @@ osism.commands: sonic reset = osism.commands.sonic:Reset sonic show = osism.commands.sonic:Show sonic sync= osism.commands.sync:Sonic + sonic validate = osism.commands.sonic:Validate sonic ztp = osism.commands.sonic:Ztp sync ceph-keys = osism.commands.sync:CephKeys sync configuration = osism.commands.configuration:Sync