diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68ba6ab..336322b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -226,7 +226,7 @@ integration-test-posix-extdispatch: - while ! [ -r ./ud3tn.aap2.socket ]; do sleep 0.1; done - aap2-bdm-ud3tn-routing --insecure-config -vv & DISPATCHER_PID=$! - sleep 1 # give the dispatcher some time to start - - TEST_QUERY=1 pytest test/integration + - TEST_QUERY=1 TEST_JSON_CONFIG=1 pytest test/integration - CLA=tcpspp pytest test/integration - kill -TERM $UD3TN_PID - echo "Waiting for uD3TN to exit gracefully - if it doesn't, check for sanitizer warnings." diff --git a/nix/packages.nix b/nix/packages.nix index 560c482..149ba09 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -49,7 +49,7 @@ rec { src = ../python-ud3tn-utils; format = "pyproject"; nativeBuildInputs = [ setuptools ]; - propagatedBuildInputs = [ cbor2 protobuf pyd3tn ]; + propagatedBuildInputs = [ cbor2 jsonschema protobuf pyd3tn ]; }; mkdocs-html = pkgs.stdenv.mkDerivation { diff --git a/python-ud3tn-utils/requirements.txt b/python-ud3tn-utils/requirements.txt index 6d4c43e..72629ef 100644 --- a/python-ud3tn-utils/requirements.txt +++ b/python-ud3tn-utils/requirements.txt @@ -1,3 +1,4 @@ cbor2==5.6.5 +jsonschema==4.23.0 protobuf==6.30.2 pyd3tn==0.14.2 diff --git a/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_config.py b/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_config.py index 76870c1..c4f1b85 100644 --- a/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_config.py +++ b/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_config.py @@ -12,7 +12,10 @@ from ud3tn_utils.aap.bin.helpers import ( initialize_logger, get_config_eid, ) -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) logger = logging.getLogger(__name__) diff --git a/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_contact_plan_reader.py b/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_contact_plan_reader.py index 2e594bb..aa8c14c 100644 --- a/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_contact_plan_reader.py +++ b/python-ud3tn-utils/ud3tn_utils/aap/bin/aap_contact_plan_reader.py @@ -12,7 +12,11 @@ from datetime import datetime import sys from ud3tn_utils.aap import AAPTCPClient, AAPUnixClient -from ud3tn_utils.config import ConfigMessage, unix2dtn, Contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + unix2dtn, + Contact, +) from ud3tn_utils.aap.bin.helpers import get_config_eid diff --git a/python-ud3tn-utils/ud3tn_utils/aap2/bin/aap2_config.py b/python-ud3tn-utils/ud3tn_utils/aap2/bin/aap2_config.py index 1d106c9..a2c9bb3 100644 --- a/python-ud3tn-utils/ud3tn_utils/aap2/bin/aap2_config.py +++ b/python-ud3tn-utils/ud3tn_utils/aap2/bin/aap2_config.py @@ -14,7 +14,7 @@ from ud3tn_utils.aap2 import ( ResponseStatus, ) from ud3tn_utils.config import ( - ConfigMessage, + LegacyConfigMessage as ConfigMessage, make_contact, RouterCommand, ) diff --git a/python-ud3tn-utils/ud3tn_utils/config.py b/python-ud3tn-utils/ud3tn_utils/config.py index f2933b8..ab25c8c 100644 --- a/python-ud3tn-utils/ud3tn_utils/config.py +++ b/python-ud3tn-utils/ud3tn_utils/config.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause OR Apache-2.0 import dataclasses import enum +import json +import jsonschema import time import re @@ -12,6 +14,57 @@ DTN_EPOCH = datetime(2000, 1, 1, tzinfo=timezone.utc) UNIX_TO_DTN_OFFSET = (DTN_EPOCH - UNIX_EPOCH).total_seconds() assert UNIX_TO_DTN_OFFSET == 946684800 +# This is a quite permissive regex to check EIDs in configuration commands, +# just enforcing the general structure of : specified by RFC9171. +CONFIG_EID_VALIDATION_REGEX = r"^[a-zA-Z0-9]+:.+$" + +# Schema for the new JSON format for configuration messages. +CONFIG_MSG_JSON_SCHEMA = { + "type": "object", + "properties": { + "command": { + "type": "string", + "pattern": r"^(ADD|UPDATE|DELETE|QUERY)$", + }, + "node_id": { + "type": "string", + "pattern": CONFIG_EID_VALIDATION_REGEX, + }, + "cla_addr": { + "type": "string", + "minLength": 1, + }, + "reachable_eids": { + "type": "array", + "items": { + "type": "string", + "pattern": CONFIG_EID_VALIDATION_REGEX, + }, + }, + "contact_list": { + "type": "array", + "items": { + "type": "object", + "properties": { + "start": {"type": "integer", "minimum": 0}, + "end": {"type": "integer", "minimum": 0}, + "data_rate": {"type": "integer", "minimum": 0}, + "reachable_eids": { + "type": "array", + "items": { + "type": "string", + "pattern": CONFIG_EID_VALIDATION_REGEX, + }, + }, + }, + "required": ["start", "end", "data_rate"], + } + }, + }, + "additionalProperties": False, + "required": ["command", "node_id"], +} + def unix2dtn(unix_timestamp): """Converts a given Unix timestamp into a DTN timestamp @@ -42,6 +95,7 @@ class Contact: start (int): DTN timestamp when the contact starts end (int): DTN timestamp when the contact is over bitrate (int): Bitrate of the contact, in bytes per second + reachable_eids (Set[str]): EIDs reachable during contact; not hashed """ start: int end: int @@ -83,6 +137,7 @@ def make_contact(start_offset, duration, bitrate, reachable_eids=None): start_offset (int): Start point of the contact in seconds from now duration (int): Duration of the contact in seconds bitrate (int): Bitrate of the contact, in bytes per second + reachable_eids (Set[str]): EIDs reachable during contact; not hashed Returns: Contact: contact tuple with DTN timestamps """ @@ -105,8 +160,9 @@ def make_contact(start_offset, duration, bitrate, reachable_eids=None): class ConfigMessage(object): - """uD3TN configuration message that can be processed by its config agent. - These messages are used to configure contacts in uD3TN. + """Base class for the uD3TN configuration messages, which can be processed + by the config endpoint of its deterministic first-contact forwarding BDM + (either integrated or external). Args: eid (Optional[str]): The endpoint identifier of a contact @@ -132,8 +188,122 @@ class ConfigMessage(object): self.eid, self.cla_address, self.reachable_eids, self.contacts ) + @staticmethod + def parse(config_str: str, schema_validate: bool = True): + """Parse the provided configuration string representation, + automatically decising whether it is a JSON or legacy representation. + + Args: + config_str (str): The serialized string representation, obtained + e.g. via str() applied on a LegacyConfigMessage or + JSONConfigMessage object. + + Return: + A ConfigMessage instance of the correct subclass representing the + data parsed from config_str. + """ + + if config_str[0] == "{": + return JSONConfigMessage.parse(config_str, schema_validate) + else: + return LegacyConfigMessage.parse(config_str) + + +class JSONConfigMessage(ConfigMessage): + """uD3TN configuration message using the JSON encoding. + + NOTE: At the moment (v0.15.0), for this to be supported, the external + (Python) forwarding module has to be used. + """ + def __str__(self): - # missing escaping has to be addresses in uD3TN + ret_obj = { + "command": { + 1: "ADD", + 2: "UPDATE", + 3: "DELETE", + 4: "QUERY", + }[self.type], + "node_id": self.eid, + } + + if self.cla_address: + ret_obj["cla_addr"] = self.cla_address + + if self.reachable_eids: + ret_obj["reachable_eids"] = list(self.reachable_eids) + + if self.contacts: + ret_obj["contact_list"] = [ + { + "start": c.start, + "end": c.end, + "data_rate": c.bitrate, + "reachable_eids": ( + list(c.reachable_eids) if c.reachable_eids else [] + ), + } + for c in self.contacts + ] + + return json.dumps(ret_obj) + + def __bytes__(self): + return str(self).encode('utf-8') + + def to_legacy_format(self) -> "LegacyConfigMessage": + return LegacyConfigMessage( + self.eid, + self.cla_address, + self.reachable_eids, + self.contacts, + type=self.type, + ) + + @staticmethod + def parse(config_str: str, schema_validate: bool = True): + """Parse the provided JSON configuration string representation. + + Args: + config_str (str): The serialized string representation, obtained + e.g. via str() applied on a JSONConfigMessage object. + + Return: + A JSONConfigMessage instance representing the data parsed from + config_str. + """ + + obj = json.loads(config_str) + if schema_validate: + jsonschema.validate(obj, CONFIG_MSG_JSON_SCHEMA) + + return JSONConfigMessage( + obj["node_id"], + obj["cla_addr"], + obj.get("reachable_eids", []), + [ + Contact( + start=c["start"], + end=c["end"], + bitrate=c["data_rate"], + reachable_eids=c.get("reachable_eids", None), + ) + for c in obj.get("contact_list", []) + ], + type=RouterCommand({ + "ADD": 1, + "UPDATE": 2, + "DELETE": 3, + "QUERY": 4, + }[obj["command"]]), + ) + + +class LegacyConfigMessage(ConfigMessage): + """uD3TN configuration message using the legacy custom format.""" + + def __str__(self): + # missing escaping has to be addressed in uD3TN for part in ({str(self.eid), str(self.cla_address)} | self.reachable_eids): assert "(" not in part, "unsupported character in string" @@ -181,22 +351,31 @@ class ConfigMessage(object): def __bytes__(self): return str(self).encode('ascii') + def to_json_format(self) -> JSONConfigMessage: + return JSONConfigMessage( + self.eid, + self.cla_address, + self.reachable_eids, + self.contacts, + type=self.type, + ) + @staticmethod def parse(config_str: str): """Parse the provided configuration string representation. Args: config_str (str): The serialized string representation, obtained - e.g. via str() applied on a ConfigMessage object. + e.g. via str() applied on a LegacyConfigMessage object. Return: - A ConfigMessage instance representing to the data parsed from + A LegacyConfigMessage instance representing the data parsed from config_str. """ type, node_id, cla_addr, reachable_eids, contacts = _parse_config( config_str, ) - return ConfigMessage( + return LegacyConfigMessage( node_id, cla_addr, reachable_eids, @@ -389,11 +568,30 @@ def test_parse_and_serialize(): "{1401519506972,1401519516972,1200,[(dtn://89326/),(dtn://12349/)]}];" ) cm1 = ConfigMessage.parse(config_str1) + assert isinstance(cm1, LegacyConfigMessage) # NOTE: We cannot `assert config_str1 == str(cm1)` here as the set # ordering may differ. cm2 = ConfigMessage.parse(str(cm1)) + assert isinstance(cm2, LegacyConfigMessage) assert cm1.type == cm2.type == 1 assert cm1.eid == cm2.eid == "dtn://ud3tn2.dtn/" assert cm1.cla_address == cm2.cla_address == "mtcp:127.0.0.1:4223" assert cm1.reachable_eids == cm2.reachable_eids == {"ipn:1.0"} assert cm1.contacts == cm2.contacts + # Test JSON conversion and serialization + jo = cm1.to_json_format() + assert isinstance(jo, JSONConfigMessage) + jstr = str(jo) + jo2 = ConfigMessage.parse(jstr, True) + assert cm1.type == jo2.type == 1 + assert cm1.eid == jo2.eid == "dtn://ud3tn2.dtn/" + assert cm1.cla_address == jo2.cla_address == "mtcp:127.0.0.1:4223" + assert cm1.reachable_eids == jo2.reachable_eids == {"ipn:1.0"} + assert cm1.contacts == jo2.contacts + cm3 = jo2.to_legacy_format() + assert isinstance(cm3, LegacyConfigMessage) + assert cm1.type == cm3.type == 1 + assert cm1.eid == cm3.eid == "dtn://ud3tn2.dtn/" + assert cm1.cla_address == cm3.cla_address == "mtcp:127.0.0.1:4223" + assert cm1.reachable_eids == cm3.reachable_eids == {"ipn:1.0"} + assert cm1.contacts == cm3.contacts diff --git a/test/functional/sqlite_storage_test/storage_agent_bundle_id_test.py b/test/functional/sqlite_storage_test/storage_agent_bundle_id_test.py index 0f9609d..3fc8c4b 100644 --- a/test/functional/sqlite_storage_test/storage_agent_bundle_id_test.py +++ b/test/functional/sqlite_storage_test/storage_agent_bundle_id_test.py @@ -25,7 +25,10 @@ from ud3tn_utils.aap2 import ( ResponseStatus, ) from ud3tn_utils.aap2.bin.aap2_receive import run_aap_recv -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from ud3tn_utils.storage_agent import StorageCall, StorageOperation diff --git a/test/integration/helpers.py b/test/integration/helpers.py index d2e0b24..6f9ef3e 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -2,7 +2,7 @@ import os import time -from ud3tn_utils.config import ConfigMessage, RouterCommand +from ud3tn_utils.config import LegacyConfigMessage, RouterCommand from pyd3tn.bundle7 import Bundle, CRCType USER_SELECTED_CLA = os.environ.get("CLA", None) @@ -35,6 +35,7 @@ TEST_AAP2 = os.environ.get("TEST_AAP2", "1") == "1" TEST_AAP2_ASYNC = TEST_AAP2 and os.environ.get("TEST_AAP2_ASYNC", "1") == "1" AAP2_BDM_SECRET = os.environ.get("TEST_AAP2_BDM_SECRET", None) +TEST_JSON_CONFIG = os.environ.get("TEST_JSON_CONFIG", "0") == "1" TEST_QUERY = os.environ.get("TEST_QUERY", "0") == "1" UD3TN_EID = "dtn://ud3tn.dtn/" @@ -88,7 +89,7 @@ def send_delete_gs(conn, serialize_func, gs_iterable): conn.send_bundle(serialize_func( TEST_SCRIPT_EID, UD3TN_CONFIG_EP, - bytes(ConfigMessage( + bytes(LegacyConfigMessage( eid, "NULL", type=RouterCommand.DELETE, diff --git a/test/integration/test_bundle_age.py b/test/integration/test_bundle_age.py index 22b85c4..4339dd4 100644 --- a/test/integration/test_bundle_age.py +++ b/test/integration/test_bundle_age.py @@ -4,7 +4,10 @@ import time import cbor2 -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from pyd3tn.bundle7 import serialize_bundle7, create_bundle7, BlockType from pyd3tn.mtcp import MTCPConnection diff --git a/test/integration/test_bundle_routing.py b/test/integration/test_bundle_routing.py index 319c0cb..bf7f79e 100644 --- a/test/integration/test_bundle_routing.py +++ b/test/integration/test_bundle_routing.py @@ -5,7 +5,10 @@ import copy import pytest -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from pyd3tn.bundle7 import serialize_bundle7, BundleProcFlag from pyd3tn.mtcp import MTCPConnection diff --git a/test/integration/test_bundle_send_receive.py b/test/integration/test_bundle_send_receive.py index ad986b2..96bef63 100644 --- a/test/integration/test_bundle_send_receive.py +++ b/test/integration/test_bundle_send_receive.py @@ -6,7 +6,7 @@ import math import pytest from ud3tn_utils.config import ( - ConfigMessage, + LegacyConfigMessage as ConfigMessage, make_contact, ) diff --git a/test/integration/test_contact_configuration.py b/test/integration/test_contact_configuration.py index e67ccc2..b6be465 100644 --- a/test/integration/test_contact_configuration.py +++ b/test/integration/test_contact_configuration.py @@ -13,7 +13,8 @@ from ud3tn_utils.aap2 import ( ) from ud3tn_utils.config import ( - ConfigMessage, + JSONConfigMessage, + LegacyConfigMessage, make_contact, RouterCommand, ) @@ -23,6 +24,7 @@ from .helpers import ( AAP2_SECRET, AAP2_SOCKET, TEST_AAP2, + TEST_JSON_CONFIG, TEST_QUERY, UD3TN_CONFIG_EP, ) @@ -49,13 +51,16 @@ CONTACT_JSON_SCHEMA = { "type": "array", "items": {"type": "string", "minLength": 4}, }, - } + }, + "additionalProperties": False, + "required": ["start", "end", "data_rate"], } } def _send_config(rpc_client, cmd, eid, reachable_eids=None, contacts=None): - payload = bytes(ConfigMessage( + MSG_TYPE = JSONConfigMessage if TEST_JSON_CONFIG else LegacyConfigMessage + payload = bytes(MSG_TYPE( eid, CLA_STR, reachable_eids=reachable_eids, diff --git a/test/integration/test_re_scheduling.py b/test/integration/test_re_scheduling.py index 4029cd7..e1f5471 100644 --- a/test/integration/test_re_scheduling.py +++ b/test/integration/test_re_scheduling.py @@ -5,7 +5,7 @@ import time import pytest from ud3tn_utils.config import ( - ConfigMessage, + LegacyConfigMessage as ConfigMessage, make_contact, RouterCommand, ) diff --git a/test/integration/test_send_receive_fragment.py b/test/integration/test_send_receive_fragment.py index 4aa9e6d..39d34af 100644 --- a/test/integration/test_send_receive_fragment.py +++ b/test/integration/test_send_receive_fragment.py @@ -6,7 +6,7 @@ import time import pytest from ud3tn_utils.config import ( - ConfigMessage, + LegacyConfigMessage as ConfigMessage, make_contact, ) diff --git a/tools/cla/mtcp_test.py b/tools/cla/mtcp_test.py index 5efd2b0..e92c26d 100644 --- a/tools/cla/mtcp_test.py +++ b/tools/cla/mtcp_test.py @@ -4,7 +4,10 @@ import time -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from pyd3tn.bundle7 import serialize_bundle7, Bundle from pyd3tn.bundle6 import serialize_bundle6 diff --git a/tools/cla/tcpcl_test.py b/tools/cla/tcpcl_test.py index 2ab5266..f914434 100644 --- a/tools/cla/tcpcl_test.py +++ b/tools/cla/tcpcl_test.py @@ -6,7 +6,10 @@ import time import socket import argparse -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from pyd3tn.tcpcl import ( serialize_tcpcl_contact_header, diff --git a/tools/cla/tcpspp_test.py b/tools/cla/tcpspp_test.py index 1c2585d..9f12796 100644 --- a/tools/cla/tcpspp_test.py +++ b/tools/cla/tcpspp_test.py @@ -3,7 +3,10 @@ import socket -from ud3tn_utils.config import ConfigMessage, make_contact +from ud3tn_utils.config import ( + LegacyConfigMessage as ConfigMessage, + make_contact, +) from pyd3tn.bundle7 import serialize_bundle7, Bundle from pyd3tn.bundle6 import serialize_bundle6