python-ud3tn-utils: Introduce JSON configuration format

This introduces a JSON contact configuration format for the
deterministic first-contact forwarding (DFCF) implementation as a modern
alternative to our homebrewn configuration messages. For now, it is only
supported in the external DFCF BDM, but integration into the "router
agent" is planned.

Example JSON configuration string (from our tests):

```
{

    "command": "ADD",
    "node_id": "dtn://ud3tn2.dtn/",
    "cla_addr": "mtcp:127.0.0.1:4223",
    "reachable_eids": [
        "ipn:1.0"
    ],
    "contact_list": [
        {
            "start": 1401519306972,
            "end": 1401519316972,
            "data_rate": 2400,
            "reachable_eids": [
                "dtn://66553/",
                "dtn://89326/"
            ]
        },
        {
            "start": 1401519506972,
            "end": 1401519516972,
            "data_rate": 1200,
            "reachable_eids": [
                "dtn://12349/",
                "dtn://89326/"
            ]
        }
    ]
}
```

Closes: #15

Signed-off-by: Felix Walter <felix.walter@d3tn.com>

.gitlab-ci.yml

@@ -226,7 +226,7 @@ integration-test-posix-extdispatch:
         - while ! [ -r ./ud3tn.aap2.socket ]; do sleep 0.1; done
         - aap2-bdm-ud3tn-routing --insecure-config -vv & DISPATCHER_PID=$!
         - sleep 1 # give the dispatcher some time to start
-        - TEST_QUERY=1 pytest test/integration
+        - TEST_QUERY=1 TEST_JSON_CONFIG=1 pytest test/integration
         - CLA=tcpspp pytest test/integration
         - kill -TERM $UD3TN_PID
         - echo "Waiting for uD3TN to exit gracefully - if it doesn't, check for sanitizer warnings."

nix/packages.nix

@@ -49,7 +49,7 @@ rec {
     src = ../python-ud3tn-utils;
     format = "pyproject";
     nativeBuildInputs = [ setuptools ];
-    propagatedBuildInputs = [ cbor2 protobuf pyd3tn ];
+    propagatedBuildInputs = [ cbor2 jsonschema protobuf pyd3tn ];
   };
 
   mkdocs-html = pkgs.stdenv.mkDerivation {

python-ud3tn-utils/requirements.txt

@@ -1,3 +1,4 @@
 cbor2==5.6.5
+jsonschema==4.23.0
 protobuf==6.30.2
 pyd3tn==0.14.2

python-ud3tn-utils/ud3tn_utils/aap/bin/aap_config.py

@@ -12,7 +12,10 @@ from ud3tn_utils.aap.bin.helpers import (
     initialize_logger,
     get_config_eid,
 )
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 
 
 logger = logging.getLogger(__name__)

python-ud3tn-utils/ud3tn_utils/aap/bin/aap_contact_plan_reader.py

@@ -12,7 +12,11 @@ from datetime import datetime
 import sys
 
 from ud3tn_utils.aap import AAPTCPClient, AAPUnixClient
-from ud3tn_utils.config import ConfigMessage, unix2dtn, Contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    unix2dtn,
+    Contact,
+)
 from ud3tn_utils.aap.bin.helpers import get_config_eid
 
 

python-ud3tn-utils/ud3tn_utils/aap2/bin/aap2_config.py

@@ -14,7 +14,7 @@ from ud3tn_utils.aap2 import (
     ResponseStatus,
 )
 from ud3tn_utils.config import (
-    ConfigMessage,
+    LegacyConfigMessage as ConfigMessage,
     make_contact,
     RouterCommand,
 )

python-ud3tn-utils/ud3tn_utils/config.py

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause OR Apache-2.0
 import dataclasses
 import enum
+import json
+import jsonschema
 import time
 import re
 
@@ -12,6 +14,57 @@ DTN_EPOCH = datetime(2000, 1, 1, tzinfo=timezone.utc)
 UNIX_TO_DTN_OFFSET = (DTN_EPOCH - UNIX_EPOCH).total_seconds()
 assert UNIX_TO_DTN_OFFSET == 946684800
 
+# This is a quite permissive regex to check EIDs in configuration commands,
+# just enforcing the general structure of <scheme>:<ssp> specified by RFC9171.
+CONFIG_EID_VALIDATION_REGEX = r"^[a-zA-Z0-9]+:.+$"
+
+# Schema for the new JSON format for configuration messages.
+CONFIG_MSG_JSON_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "command": {
+            "type": "string",
+            "pattern": r"^(ADD|UPDATE|DELETE|QUERY)$",
+        },
+        "node_id": {
+            "type": "string",
+            "pattern": CONFIG_EID_VALIDATION_REGEX,
+        },
+        "cla_addr": {
+            "type": "string",
+            "minLength": 1,
+        },
+        "reachable_eids": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": CONFIG_EID_VALIDATION_REGEX,
+            },
+        },
+        "contact_list": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "start": {"type": "integer", "minimum": 0},
+                    "end": {"type": "integer", "minimum": 0},
+                    "data_rate": {"type": "integer", "minimum": 0},
+                    "reachable_eids": {
+                        "type": "array",
+                        "items": {
+                            "type": "string",
+                            "pattern": CONFIG_EID_VALIDATION_REGEX,
+                        },
+                    },
+                },
+                "required": ["start", "end", "data_rate"],
+            }
+        },
+    },
+    "additionalProperties": False,
+    "required": ["command", "node_id"],
+}
+
 
 def unix2dtn(unix_timestamp):
     """Converts a given Unix timestamp into a DTN timestamp
@@ -42,6 +95,7 @@ class Contact:
         start (int): DTN timestamp when the contact starts
         end (int): DTN timestamp when the contact is over
         bitrate (int): Bitrate of the contact, in bytes per second
+        reachable_eids (Set[str]): EIDs reachable during contact; not hashed
     """
     start: int
     end: int
@@ -83,6 +137,7 @@ def make_contact(start_offset, duration, bitrate, reachable_eids=None):
         start_offset (int): Start point of the contact in seconds from now
         duration (int): Duration of the contact in seconds
         bitrate (int): Bitrate of the contact, in bytes per second
+        reachable_eids (Set[str]): EIDs reachable during contact; not hashed
     Returns:
         Contact: contact tuple with DTN timestamps
     """
@@ -105,8 +160,9 @@ def make_contact(start_offset, duration, bitrate, reachable_eids=None):
 
 
 class ConfigMessage(object):
-    """uD3TN configuration message that can be processed by its config agent.
-    These messages are used to configure contacts in uD3TN.
+    """Base class for the uD3TN configuration messages, which can be processed
+    by the config endpoint of its deterministic first-contact forwarding BDM
+    (either integrated or external).
 
     Args:
         eid (Optional[str]): The endpoint identifier of a contact
@@ -132,8 +188,122 @@ class ConfigMessage(object):
             self.eid, self.cla_address, self.reachable_eids, self.contacts
         )
 
+    @staticmethod
+    def parse(config_str: str, schema_validate: bool = True):
+        """Parse the provided configuration string representation,
+        automatically decising whether it is a JSON or legacy representation.
+
+        Args:
+            config_str (str): The serialized string representation, obtained
+                e.g. via str() applied on a LegacyConfigMessage or
+                JSONConfigMessage object.
+
+        Return:
+            A ConfigMessage instance of the correct subclass representing the
+            data parsed from config_str.
+        """
+
+        if config_str[0] == "{":
+            return JSONConfigMessage.parse(config_str, schema_validate)
+        else:
+            return LegacyConfigMessage.parse(config_str)
+
+
+class JSONConfigMessage(ConfigMessage):
+    """uD3TN configuration message using the JSON encoding.
+
+    NOTE: At the moment (v0.15.0), for this to be supported, the external
+        (Python) forwarding module has to be used.
+    """
+
     def __str__(self):
-        # missing escaping has to be addresses in uD3TN
+        ret_obj = {
+            "command": {
+                1: "ADD",
+                2: "UPDATE",
+                3: "DELETE",
+                4: "QUERY",
+            }[self.type],
+            "node_id": self.eid,
+        }
+
+        if self.cla_address:
+            ret_obj["cla_addr"] = self.cla_address
+
+        if self.reachable_eids:
+            ret_obj["reachable_eids"] = list(self.reachable_eids)
+
+        if self.contacts:
+            ret_obj["contact_list"] = [
+                {
+                    "start": c.start,
+                    "end": c.end,
+                    "data_rate": c.bitrate,
+                    "reachable_eids": (
+                        list(c.reachable_eids) if c.reachable_eids else []
+                    ),
+                }
+                for c in self.contacts
+            ]
+
+        return json.dumps(ret_obj)
+
+    def __bytes__(self):
+        return str(self).encode('utf-8')
+
+    def to_legacy_format(self) -> "LegacyConfigMessage":
+        return LegacyConfigMessage(
+            self.eid,
+            self.cla_address,
+            self.reachable_eids,
+            self.contacts,
+            type=self.type,
+        )
+
+    @staticmethod
+    def parse(config_str: str, schema_validate: bool = True):
+        """Parse the provided JSON configuration string representation.
+
+        Args:
+            config_str (str): The serialized string representation, obtained
+                e.g. via str() applied on a JSONConfigMessage object.
+
+        Return:
+            A JSONConfigMessage instance representing the data parsed from
+            config_str.
+        """
+
+        obj = json.loads(config_str)
+        if schema_validate:
+            jsonschema.validate(obj, CONFIG_MSG_JSON_SCHEMA)
+
+        return JSONConfigMessage(
+            obj["node_id"],
+            obj["cla_addr"],
+            obj.get("reachable_eids", []),
+            [
+                Contact(
+                    start=c["start"],
+                    end=c["end"],
+                    bitrate=c["data_rate"],
+                    reachable_eids=c.get("reachable_eids", None),
+                )
+                for c in obj.get("contact_list", [])
+            ],
+            type=RouterCommand({
+                "ADD": 1,
+                "UPDATE": 2,
+                "DELETE": 3,
+                "QUERY": 4,
+            }[obj["command"]]),
+        )
+
+
+class LegacyConfigMessage(ConfigMessage):
+    """uD3TN configuration message using the legacy custom format."""
+
+    def __str__(self):
+        # missing escaping has to be addressed in uD3TN
         for part in ({str(self.eid), str(self.cla_address)} |
                      self.reachable_eids):
             assert "(" not in part, "unsupported character in string"
@@ -181,22 +351,31 @@ class ConfigMessage(object):
     def __bytes__(self):
         return str(self).encode('ascii')
 
+    def to_json_format(self) -> JSONConfigMessage:
+        return JSONConfigMessage(
+            self.eid,
+            self.cla_address,
+            self.reachable_eids,
+            self.contacts,
+            type=self.type,
+        )
+
     @staticmethod
     def parse(config_str: str):
         """Parse the provided configuration string representation.
 
         Args:
             config_str (str): The serialized string representation, obtained
-                e.g. via str() applied on a ConfigMessage object.
+                e.g. via str() applied on a LegacyConfigMessage object.
 
         Return:
-            A ConfigMessage instance representing to the data parsed from
+            A LegacyConfigMessage instance representing the data parsed from
             config_str.
         """
         type, node_id, cla_addr, reachable_eids, contacts = _parse_config(
             config_str,
         )
-        return ConfigMessage(
+        return LegacyConfigMessage(
             node_id,
             cla_addr,
             reachable_eids,
@@ -389,11 +568,30 @@ def test_parse_and_serialize():
         "{1401519506972,1401519516972,1200,[(dtn://89326/),(dtn://12349/)]}];"
     )
     cm1 = ConfigMessage.parse(config_str1)
+    assert isinstance(cm1, LegacyConfigMessage)
     # NOTE: We cannot `assert config_str1 == str(cm1)` here as the set
     # ordering may differ.
     cm2 = ConfigMessage.parse(str(cm1))
+    assert isinstance(cm2, LegacyConfigMessage)
     assert cm1.type == cm2.type == 1
     assert cm1.eid == cm2.eid == "dtn://ud3tn2.dtn/"
     assert cm1.cla_address == cm2.cla_address == "mtcp:127.0.0.1:4223"
     assert cm1.reachable_eids == cm2.reachable_eids == {"ipn:1.0"}
     assert cm1.contacts == cm2.contacts
+    # Test JSON conversion and serialization
+    jo = cm1.to_json_format()
+    assert isinstance(jo, JSONConfigMessage)
+    jstr = str(jo)
+    jo2 = ConfigMessage.parse(jstr, True)
+    assert cm1.type == jo2.type == 1
+    assert cm1.eid == jo2.eid == "dtn://ud3tn2.dtn/"
+    assert cm1.cla_address == jo2.cla_address == "mtcp:127.0.0.1:4223"
+    assert cm1.reachable_eids == jo2.reachable_eids == {"ipn:1.0"}
+    assert cm1.contacts == jo2.contacts
+    cm3 = jo2.to_legacy_format()
+    assert isinstance(cm3, LegacyConfigMessage)
+    assert cm1.type == cm3.type == 1
+    assert cm1.eid == cm3.eid == "dtn://ud3tn2.dtn/"
+    assert cm1.cla_address == cm3.cla_address == "mtcp:127.0.0.1:4223"
+    assert cm1.reachable_eids == cm3.reachable_eids == {"ipn:1.0"}
+    assert cm1.contacts == cm3.contacts

test/functional/sqlite_storage_test/storage_agent_bundle_id_test.py

@@ -25,7 +25,10 @@ from ud3tn_utils.aap2 import (
     ResponseStatus,
 )
 from ud3tn_utils.aap2.bin.aap2_receive import run_aap_recv
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 from ud3tn_utils.storage_agent import StorageCall, StorageOperation
 
 

test/integration/helpers.py

@@ -2,7 +2,7 @@
 import os
 import time
 
-from ud3tn_utils.config import ConfigMessage, RouterCommand
+from ud3tn_utils.config import LegacyConfigMessage, RouterCommand
 from pyd3tn.bundle7 import Bundle, CRCType
 
 USER_SELECTED_CLA = os.environ.get("CLA", None)
@@ -35,6 +35,7 @@ TEST_AAP2 = os.environ.get("TEST_AAP2", "1") == "1"
 TEST_AAP2_ASYNC = TEST_AAP2 and os.environ.get("TEST_AAP2_ASYNC", "1") == "1"
 AAP2_BDM_SECRET = os.environ.get("TEST_AAP2_BDM_SECRET", None)
 
+TEST_JSON_CONFIG = os.environ.get("TEST_JSON_CONFIG", "0") == "1"
 TEST_QUERY = os.environ.get("TEST_QUERY", "0") == "1"
 
 UD3TN_EID = "dtn://ud3tn.dtn/"
@@ -88,7 +89,7 @@ def send_delete_gs(conn, serialize_func, gs_iterable):
         conn.send_bundle(serialize_func(
             TEST_SCRIPT_EID,
             UD3TN_CONFIG_EP,
-            bytes(ConfigMessage(
+            bytes(LegacyConfigMessage(
                 eid,
                 "NULL",
                 type=RouterCommand.DELETE,

test/integration/test_bundle_age.py

@@ -4,7 +4,10 @@ import time
 
 import cbor2
 
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 from pyd3tn.bundle7 import serialize_bundle7, create_bundle7, BlockType
 from pyd3tn.mtcp import MTCPConnection
 

test/integration/test_bundle_routing.py

@@ -5,7 +5,10 @@ import copy
 
 import pytest
 
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 
 from pyd3tn.bundle7 import serialize_bundle7, BundleProcFlag
 from pyd3tn.mtcp import MTCPConnection

test/integration/test_bundle_send_receive.py

@@ -6,7 +6,7 @@ import math
 import pytest
 
 from ud3tn_utils.config import (
-    ConfigMessage,
+    LegacyConfigMessage as ConfigMessage,
     make_contact,
 )
 

test/integration/test_contact_configuration.py

@@ -13,7 +13,8 @@ from ud3tn_utils.aap2 import (
 )
 
 from ud3tn_utils.config import (
-    ConfigMessage,
+    JSONConfigMessage,
+    LegacyConfigMessage,
     make_contact,
     RouterCommand,
 )
@@ -23,6 +24,7 @@ from .helpers import (
     AAP2_SECRET,
     AAP2_SOCKET,
     TEST_AAP2,
+    TEST_JSON_CONFIG,
     TEST_QUERY,
     UD3TN_CONFIG_EP,
 )
@@ -49,13 +51,16 @@ CONTACT_JSON_SCHEMA = {
                 "type": "array",
                 "items": {"type": "string", "minLength": 4},
             },
-        }
+        },
+        "additionalProperties": False,
+        "required": ["start", "end", "data_rate"],
     }
 }
 
 
 def _send_config(rpc_client, cmd, eid, reachable_eids=None, contacts=None):
-    payload = bytes(ConfigMessage(
+    MSG_TYPE = JSONConfigMessage if TEST_JSON_CONFIG else LegacyConfigMessage
+    payload = bytes(MSG_TYPE(
         eid,
         CLA_STR,
         reachable_eids=reachable_eids,

test/integration/test_re_scheduling.py

@@ -5,7 +5,7 @@ import time
 import pytest
 
 from ud3tn_utils.config import (
-    ConfigMessage,
+    LegacyConfigMessage as ConfigMessage,
     make_contact,
     RouterCommand,
 )

test/integration/test_send_receive_fragment.py

@@ -6,7 +6,7 @@ import time
 import pytest
 
 from ud3tn_utils.config import (
-    ConfigMessage,
+    LegacyConfigMessage as ConfigMessage,
     make_contact,
 )
 

tools/cla/mtcp_test.py

@@ -4,7 +4,10 @@
 
 import time
 
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 
 from pyd3tn.bundle7 import serialize_bundle7, Bundle
 from pyd3tn.bundle6 import serialize_bundle6

tools/cla/tcpcl_test.py

@@ -6,7 +6,10 @@ import time
 import socket
 import argparse
 
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 
 from pyd3tn.tcpcl import (
     serialize_tcpcl_contact_header,

tools/cla/tcpspp_test.py

@@ -3,7 +3,10 @@
 
 import socket
 
-from ud3tn_utils.config import ConfigMessage, make_contact
+from ud3tn_utils.config import (
+    LegacyConfigMessage as ConfigMessage,
+    make_contact,
+)
 
 from pyd3tn.bundle7 import serialize_bundle7, Bundle
 from pyd3tn.bundle6 import serialize_bundle6