<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article article-type="research-article" dtd-version="2.3" xml:lang="EN" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Commun. Netw.</journal-id>
<journal-title>Frontiers in Communications and Networks</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Commun. Netw.</abbrev-journal-title>
<issn pub-type="epub">2673-530X</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="publisher-id">1635982</article-id>
<article-id pub-id-type="doi">10.3389/frcmn.2025.1635982</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Communications and Networks</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Intelligent path selection algorithm for tactical communication networks enhanced by link state awareness</article-title>
<alt-title alt-title-type="left-running-head">Shen et al.</alt-title>
<alt-title alt-title-type="right-running-head">
<ext-link ext-link-type="uri" xlink:href="https://doi.org/10.3389/frcmn.2025.1635982">10.3389/frcmn.2025.1635982</ext-link>
</alt-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name>
<surname>Shen</surname>
<given-names>Yigang</given-names>
</name>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Xie</surname>
<given-names>Lei</given-names>
</name>
<uri xlink:href="https://loop.frontiersin.org/people/3081037/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/Writing - review &#x26; editing/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name>
<surname>Li</surname>
<given-names>Ming</given-names>
</name>
<xref ref-type="corresp" rid="c001">&#x2a;</xref>
<role content-type="https://credit.niso.org/contributor-roles/Writing - review &#x26; editing/"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
</contrib>
</contrib-group>
<aff>
<institution>China Telecom Corporation Limited Zhejiang Branch</institution>, <addr-line>Hangzhou</addr-line>, <country>China</country>
</aff>
<author-notes>
<fn fn-type="edited-by">
<p>
<bold>Edited by:</bold> <ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/994157/overview">Mark Eisen</ext-link>, Johns Hopkins University, United States</p>
</fn>
<fn fn-type="edited-by">
<p>
<bold>Reviewed by:</bold> <ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/3020381/overview">Domingos F. Oliveira</ext-link>, Mandume Ya Ndemufayo University, Angola</p>
<p>
<ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/3034541/overview">Sourajit Das</ext-link>, University of Pennsylvania, United States</p>
<p>
<ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/3117165/overview">Nur Alamsyah</ext-link>, Universitas Informatika dan Bisnis Indonesia, Indonesia</p>
</fn>
<corresp id="c001">&#x2a;Correspondence: Ming Li, <email>lim.zj@chinatelecom.cn</email>
</corresp>
</author-notes>
<pub-date pub-type="epub">
<day>17</day>
<month>10</month>
<year>2025</year>
</pub-date>
<pub-date pub-type="collection">
<year>2025</year>
</pub-date>
<volume>6</volume>
<elocation-id>1635982</elocation-id>
<history>
<date date-type="received">
<day>04</day>
<month>06</month>
<year>2025</year>
</date>
<date date-type="accepted">
<day>15</day>
<month>09</month>
<year>2025</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#xa9; 2025 Shen, Xie and Li.</copyright-statement>
<copyright-year>2025</copyright-year>
<copyright-holder>Shen, Xie and Li</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p>
</license>
</permissions>
<abstract>
<p>In tactical communication networks, highly dynamic topologies and frequent data exchanges create complex spatiotemporal dependencies among link states. However, most existing intelligent routing algorithms rely on simplified model architectures and fail to capture these spatiotemporal correlations, resulting in limited situational awareness and poor adaptability under dynamic network conditions. To address these challenges, this study proposes an intelligent path selection method&#x2014;Deep Reinforcement Learning with Spatiotemporal-aware Link State Guidance Algorithm (DRLSGA). The algorithm builds upon the Proximal Policy Optimization (PPO) framework to develop an intelligent decision-making model and integrates a link state feature extraction module that combines Gated Recurrent Units (GRU) and a Graph Attention Network (GAT). This design enables the model to learn long-term temporal dependencies and spatial structural relationships from sequential link state data, thereby enhancing perception and decision-making capability. An attention mechanism is further introduced to highlight salient features within link state sequences, while an optimal routing strategy is derived through a deep reinforcement learning-based training process. Experimental results demonstrate that, compared with the existing DRL-ST algorithm, DRLSGA reduces average end-to-end latency by at least 2.07%, lowers the packet loss rate by 1.65%, and increases average throughput by up to 2.59% under high-traffic conditions. Moreover, the proposed algorithm exhibits stronger adaptability to highly dynamic network topologies.</p>
</abstract>
<kwd-group>
<kwd>link state information</kwd>
<kwd>spatiotemporal characteristics</kwd>
<kwd>intelligent path decision-making</kwd>
<kwd>deep reinforcement learning</kwd>
<kwd>graph attention network (GAT)</kwd>
</kwd-group>
<counts>
<page-count count="21"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Networks</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec id="s1">
<title>1 Introduction</title>
<p>The evolution of modern warfare has driven battlefield communication systems toward greater efficiency, flexibility, and intelligence. Routing technology plays a critical role in ensuring dependable data transmission within tactical communication networks, enabling coordination among units during missions, and improving overall quality of service (QoS) (<xref ref-type="bibr" rid="B25">Suomalainen et al., 2021</xref>; <xref ref-type="bibr" rid="B20">Liu et al., 2024</xref>). Under intelligent warfare conditions, however, the increasing number, heterogeneity, and autonomy of battlefield elements have created a highly complex and adversarial communication environment (<xref ref-type="bibr" rid="B22">Patel et al., 2023</xref>).</p>
<p>Tactical communication networks designed for intelligent combat must meet several demanding requirements. First, more than five types of heterogeneous services&#x2014;such as voice, image, and video&#x2014;are typically transmitted, each with distinct QoS demands concerning delay, bandwidth, and packet-loss tolerance (<xref ref-type="bibr" rid="B23">Pourranjbar et al., 2023</xref>). Second, the network infrastructure incorporates more than ten types of heterogeneous links, including shortwave, microwave, and optical fiber, with data rates ranging from Kbps to hundreds of Mbps. These diverse and dynamic link conditions significantly influence network performance and complicate routing decisions. As a result, the design and deployment of routing algorithms face new challenges in highly mobile and adversarial battlefield scenarios (<xref ref-type="bibr" rid="B2">Bajracharya et al., 2023</xref>).</p>
<p>Conventional routing approaches generally compute optimal paths using partial or limited link state information. While effective under stable conditions, these methods adapt poorly to rapid traffic fluctuations and often fail to meet diverse QoS requirements. The explosive growth of data traffic and applications in tactical communication networks further amplifies these limitations, leading to inefficiency and reduced accuracy (<xref ref-type="bibr" rid="B19">Liu et al., 2023</xref>). Deep reinforcement learning (DRL), a branch of artificial intelligence, has emerged as a promising solution to these challenges. DRL offers self-learning capabilities without requiring prior environmental knowledge and is well-suited for optimizing high-dimensional state spaces (<xref ref-type="bibr" rid="B27">Wang et al., 2023</xref>). Consequently, DRL-based routing strategies have attracted significant attention in path decision-making research.</p>
<p>Despite their potential, conventional DRL algorithms exhibit notable drawbacks. They often require lengthy trial-and-error exploration, during which valuable samples are underutilized while uninformative samples are repeatedly processed. This inefficiency leads to slow convergence and suboptimal routing decisions, which may aggravate congestion (<xref ref-type="bibr" rid="B10">He et al., 2023</xref>). Moreover, the frequent topological changes and intensive data exchanges in tactical communication networks give rise to complex temporal and spatial dependencies in link state information (<xref ref-type="bibr" rid="B28">Yang et al., 2024</xref>). Most existing DRL-based routing algorithms overlook these spatiotemporal correlations, thereby limiting situational awareness and adaptability. As a result, end-to-end QoS requirements are often not met.</p>
<p>To address these limitations, this study proposes a deep reinforcement learning with spatiotemporal-aware link state guidance algorithm (DRLSGA) for tactical communication networks. The algorithm leverages proximal policy optimization (PPO) to construct an intelligent path decision model while integrating a link state feature extraction module. Specifically, a gated recurrent unit (GRU) is employed to capture temporal dependencies in link state sequences, and a graph attention network (GAT) is used to model spatial correlations. An additional attention mechanism highlights critical features within the state sequence. Together, these components enable the DRL framework to generate an optimal path forwarding policy with improved adaptability to dynamic network topologies. The contributions of this work can be concluded as:<list list-type="simple">
<list-item>
<p>&#x2009;&#x2022; The study identifies critical limitations of existing DRL-based routing algorithms in tactical communication networks, including slow convergence, inefficient sample utilization, and the inability to capture spatiotemporal correlations in link state information.</p>
</list-item>
<list-item>
<p>&#x2009;&#x2022; A novel deep reinforcement learning with spatiotemporal-aware link state guidance algorithm is proposed. The algorithm integrates PPO-based decision-making with a specialized link state feature extraction module.</p>
</list-item>
<list-item>
<p>&#x2009;&#x2022; Temporal dependencies of link state sequences are captured using GRU, while spatial correlations are modeled through GAT. An attention mechanism is further introduced to highlight critical features, thereby enhancing situational awareness.</p>
</list-item>
</list>
</p>
<p>To further enhance transparency and reproducibility, we commit to releasing the full source code upon acceptance of this paper. All resources will be hosted on GitHub, enabling researchers to readily reproduce our results and adapt the proposed method to other recommendation tasks.</p>
</sec>
<sec id="s2">
<title>2 Related works</title>
<p>Global network path planning algorithms can be broadly categorized into three classes based on their underlying principles and decision-making mechanisms: traditional routing algorithms, heuristic routing algorithms, and DRL-based routing algorithms. Traditional routing algorithms are typically divided into static and dynamic approaches. Heuristic routing algorithms utilize experience-based and rule-driven strategies for path planning. DRL-based routing algorithms, depending on their learning strategies, fall into value function-based and policy gradient-based methods.</p>
<sec id="s2-1">
<title>2.1 Traditional routing algorithms</title>
<p>Conventional routing algorithms are typically divided into static and dynamic categories. Static routing involves manually configured routing tables and is incapable of adapting to variations in network topology or traffic conditions. As networks grow in scale and complexity, manual configuration becomes increasingly cumbersome and error-prone, rendering static routing unsuitable for dynamic environments. In contrast, dynamic routing algorithms autonomously adjust routing tables in response to network conditions, making them more adaptable and suitable for complex, dynamic communication scenarios (<xref ref-type="bibr" rid="B30">Zhou et al., 2023</xref>).</p>
<p>Traditional routing algorithms are supported by well-established theoretical foundations and have been widely applied across various domains. For instance, Mehraban et al. introduced an intelligent node placement strategy that integrates both node connectivity and traffic features, demonstrating notable improvements in reducing maximum link utilization over existing methods (<xref ref-type="bibr" rid="B12">Hussain and Sinha Roy, 2021</xref>). Similarly, Zhu et al. introduced a routing algorithm tailored for low Earth orbit satellite <italic>ad hoc</italic> networks by enhancing open shortest path first (OSPF) to accommodate dynamic satellite topologies, incorporating a routing reconstruction mechanism to ensure reliable data transmission (<xref ref-type="bibr" rid="B31">Zhu and Jin, 2023</xref>). Hussain et al. introduced a smart flow steering agent that leverages real-time network conditions to dynamically balance traffic in SDN, thereby enhancing routing efficiency and overall performance (<xref ref-type="bibr" rid="B13">Hussain et al., 2021</xref>). However, as network traffic and scale continue to grow exponentially, conventional routing algorithms face mounting limitations due to their restricted ability to obtain real-time link state information. This limitation results in slower convergence and delayed responsiveness to dynamic traffic changes, thereby reducing transmission efficiency. Consequently, Traditional routing strategies are inadequate for addressing the complex and dynamic demands of tactical communication networks in intelligent combat scenarios, thereby hindering the formulation of globally optimal routing decisions.</p>
</sec>
<sec id="s2-2">
<title>2.2 Heuristic routing algorithms</title>
<p>Heuristic algorithms are well-suited for solving complex optimization problems and offer notable advantages over traditional methods when addressing large-scale and dynamic network environments. As such, they have increasingly become mainstream solutions for routing optimization tasks. Tabatabaei et al. combined the bacterial foraging optimization algorithm with a mobile sink approach to propose a novel routing algorithm for wireless sensor networks. By incorporating node energy information in the path selection process, the algorithm effectively reduced control overhead and bandwidth consumption while improving routing accuracy (<xref ref-type="bibr" rid="B26">Tabatabaei, 2022</xref>). Alnajjar et al. developed a genetic algorithm to optimize cluster head placement in wireless sensor networks, achieving balanced distribution and stable energy usage. By employing customized crossover and mutation operations, the method gradually improved sensor-to-cluster assignments, confirming the efficiency and reliability of the approach over multiple generations (<xref ref-type="bibr" rid="B1">Alnajjar et al., 2022</xref>). <xref ref-type="bibr" rid="B7">Deng et al. (2022)</xref> introduced an ant colony optimization (ACO)-based routing algorithm for low Earth orbit (LEO) satellite networks. Their method jointly considers path distance, transmission direction, and link load to identify routes with minimal delay and cost, thereby achieving load balancing in the satellite network. <xref ref-type="bibr" rid="B8">Shokouhifar (2021)</xref> designed a fuzzy heuristic ACO algorithm that utilizes a multi-objective function incorporating power consumption, delay, and reliability, allowing for adaptive routing based on application-specific requirements. Ju et al. designed a dynamic load-balancing routing approach for LEO satellite networks was developed by integrating multiagent dueling double deep Q-network with spatiotemporal traffic prediction. By utilizing both local and predicted traffic data, the method efficiently optimizes routing, reduces peak and average bandwidth usage, and balances network load (<xref ref-type="bibr" rid="B14">Ju et al., 2025</xref>). While heuristic routing algorithms have demonstrated strong capability in solving complex network optimization problems and have been widely applied, they often require specific network assumptions and conditions. Variations in network topology and link status can introduce significant volatility and error, undermining algorithm scalability and stability. As a result, these methods tend to perform poorly in highly dynamic and unpredictable environments, such as tactical communication networks under battlefield conditions, where rapid changes in structure and link quality are common. Consequently, heuristic routing algorithms face challenges in maintaining robust QoS under such circumstances.</p>
</sec>
<sec id="s2-3">
<title>2.3 Deep reinforcement learning-based routing algorithms</title>
<p>DRL offers the capability to learn complex, nonlinear policies through interaction with the environment, exhibiting superior flexibility and adaptability in solving practical problems. These advantages have opened new avenues for addressing complex network optimization challenges (<xref ref-type="bibr" rid="B6">Dake et al., 2021</xref>). To address the limitations of traditional routing strategies in SDN, particularly their inability to meet diverse application-level QoS requirements and their lack of interaction with the network environment, <xref ref-type="bibr" rid="B17">Li et al. (2021)</xref> proposed DRNet&#x2014;an intelligent, QoS-aware routing algorithm. This approach utilizes traffic demand matrices and network state information as input to the DRL agent for training, enabling more flexible routing deployment. <xref ref-type="bibr" rid="B9">Fu et al. (2020)</xref> designed a deep Q network (DQN)-based routing strategy that distinguishes between mouse flows and elephant flows in data center networks, thereby improving throughput and reducing latency and packet loss. <xref ref-type="bibr" rid="B18">Liu et al. (2021)</xref> introduced a DRL-based routing algorithm that reconfigures network resources by assigning weighted importance to caching, bandwidth, and latency. Network states are represented as multidimensional inputs to the agent, enabling perceptual learning and optimized resource allocation, which enhances throughput and robustness. <xref ref-type="bibr" rid="B4">Casas-Velasco et al. (2021)</xref> proposed a DQN-based intelligent routing algorithm under the SDN architecture that learns path state information for dynamic routing decisions, addressing the challenge of traffic variability in traditional routing schemes. He et al. integrates graph neural networks with deep reinforcement learning to capture network topology information and guide routing decisions, which enables balanced traffic distribution and enhanced performance (<xref ref-type="bibr" rid="B10">He et al., 2023</xref>). <xref ref-type="bibr" rid="B11">Huang et al. (2022)</xref> developed a deep graph reinforcement learning (DGRL) routing algorithm that integrates graph convolutional network (GCN) with DDPG to achieve effective traffic control in wireless sensor networks. <xref ref-type="bibr" rid="B29">Zhou et al. (2022)</xref> proposed a PPO-based QoS-aware routing optimization mechanism, which adjusts the reward function dynamically according to different optimization goals, thus enabling a generic and customizable routing optimization framework.</p>
<p>These DRL-based intelligent routing algorithms demonstrate strong adaptability to complex and dynamic network conditions, enabling autonomous policy adjustments and improving overall service quality. However, several limitations persist. Current methods often suffer from slow convergence and unstable performance, and many focus solely on single or limited link-level features, neglecting the heterogeneous QoS demands across different traffic types. Moreover, the highly dynamic topology and rapidly changing link states of tactical communication networks challenge the fixed-architecture neural networks used in existing models, limiting their situational awareness and adaptability. Consequently, current intelligent routing algorithms are inadequate for optimal path decision-making in highly mobile, adversarial battlefield environments.</p>
</sec>
</sec>
<sec id="s3">
<title>3 Basic theory</title>
<sec id="s3-1">
<title>3.1 SDN network measurement mechanism</title>
<p>This study employs Ryu (<xref ref-type="bibr" rid="B24">Ryu SDN Framework Community, 2020</xref>) as the SDN controller to achieve network topology awareness and link status monitoring, implementing the described application requirements using Python programming.</p>
<sec id="s3-1-1">
<title>3.1.1 Network topology perception</title>
<p>The link layer discovery protocol (LLDP) (<xref ref-type="bibr" rid="B5">Chang et al., 2020</xref>) is used in the SDN network to obtain the network topology structure. The main principle is shown in <xref ref-type="fig" rid="F1">Figure 1</xref>, the controller periodically sends a Packet-Out message encapsulating LLDP data packet to the S<sub>1</sub> node of the data plane through the OpenFlow channel. Upon receiving the Packet-Out message, node S1 processes it and forwards it via the designated port to node S<sub>2</sub>. Since S<sub>2</sub> cannot find a matching flow table entry for the incoming packet, it responds by sending a Packet-In message to the controller. The controller then interprets this message to generate identifiers&#x2014;such as Classis ID and Port ID&#x2014;that help recognize neighboring nodes and their associated ports. Other adjacent nodes are identified in the same way, and finally the entire network topology structure is obtained.</p>
<fig id="F1" position="float">
<label>FIGURE 1</label>
<caption>
<p>Network topology awareness.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g001.tif">
<alt-text content-type="machine-generated">Diagram showing a network topology with an SDN controller and four nodes labeled S1 to S4. Nodes are connected by directional lines representing data flow. The controller interacts with nodes via &#x22;Packet-in&#x22; and &#x22;Packet-out&#x22; with LLDP. Each node has port numbers labeled Port1, Port2, and Port3, with corresponding Classis ID and Port ID for Node S1.</alt-text>
</graphic>
</fig>
<p>After achieving global network topology awareness, the NetworkX module in Python is used to store the topology as a graph <inline-formula id="inf1">
<mml:math id="m1">
<mml:mrow>
<mml:mi>G</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, where V represents the node set in the graph G, corresponding to the routing node set in the topology, and E represents the edge set in the graph G, corresponding to the link set between topological adjacent nodes.</p>
</sec>
<sec id="s3-1-2">
<title>3.1.2 Delay measurement SDN</title>
<p>The delay measurement method of the network link (<xref ref-type="bibr" rid="B21">Lu et al., 2023</xref>) is shown in <xref ref-type="fig" rid="F2">Figure 2</xref>. First, the SDN controller sends LLDP protocol to nodes S<sub>1</sub> and S<sub>2</sub> to obtain timestamps <inline-formula id="inf2">
<mml:math id="m2">
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf3">
<mml:math id="m3">
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> respectively. Then, by periodically sending Echo Request messages to nodes S<sub>1</sub> and S<sub>2</sub>, nodes S<sub>1</sub> and S<sub>2</sub> reply to the controller with Echo Reply messages after receiving the request, and obtain timestamps <inline-formula id="inf4">
<mml:math id="m4">
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>e</mml:mi>
<mml:mi>c</mml:mi>
<mml:mi>h</mml:mi>
<mml:mi>o</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf5">
<mml:math id="m5">
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>e</mml:mi>
<mml:mi>c</mml:mi>
<mml:mi>h</mml:mi>
<mml:mi>o</mml:mi>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> respectively. The delay <inline-formula id="inf6">
<mml:math id="m6">
<mml:mrow>
<mml:msub>
<mml:mi>d</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the link between nodes S<sub>1</sub> and S<sub>2</sub> is calculated according to <xref ref-type="disp-formula" rid="e1">Equation 1</xref>.<disp-formula id="e1">
<mml:math id="m7">
<mml:mrow>
<mml:msub>
<mml:mi>d</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>e</mml:mi>
<mml:mi>c</mml:mi>
<mml:mi>h</mml:mi>
<mml:mi>o</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mi>T</mml:mi>
<mml:mrow>
<mml:mi>e</mml:mi>
<mml:mi>c</mml:mi>
<mml:mi>h</mml:mi>
<mml:mi>o</mml:mi>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
<mml:mn>2</mml:mn>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(1)</label>
</disp-formula>
</p>
<fig id="F2" position="float">
<label>FIGURE 2</label>
<caption>
<p>Delay measurement.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g002.tif">
<alt-text content-type="machine-generated">Diagram of an SDN (Software-Defined Networking) controller connected to two network nodes, S&#x2081; and S&#x2082;. Arrows indicate data flow: blue for \( T_{ldp1} \), red for \( T_{ldp2} \), dashed purple for \( T_{echo1} \), and dashed green for \( T_{echo2} \).</alt-text>
</graphic>
</fig>
</sec>
<sec id="s3-1-3">
<title>3.1.3 Remaining bandwidth measurement</title>
<p>The remaining bandwidth measurement method of the SDN network link is shown in <xref ref-type="fig" rid="F3">Figure 3</xref>. The controller sends a port status request message to the node at t<sub>1</sub> and t<sub>2</sub> to query the node status. The node replies with a port status reply (Port Stats Reply) message. The controller parses the data packet of the reply message to obtain the sent byte count <inline-formula id="inf7">
<mml:math id="m8">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf8">
<mml:math id="m9">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, received byte count <inline-formula id="inf9">
<mml:math id="m10">
<mml:mrow>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf10">
<mml:math id="m11">
<mml:mrow>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and port lifetime <inline-formula id="inf11">
<mml:math id="m12">
<mml:mrow>
<mml:msub>
<mml:mi>t</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mi>u</mml:mi>
<mml:mi>r</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf12">
<mml:math id="m13">
<mml:mrow>
<mml:msub>
<mml:mi>t</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mi>u</mml:mi>
<mml:mi>r</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the node forwarding port statistics at t<sub>1</sub> and t<sub>2</sub>. The occupied bandwidth <inline-formula id="inf13">
<mml:math id="m14">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mi>s</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the link is calculated based on the statistical data at the two moments. The calculation method is shown in <xref ref-type="disp-formula" rid="e2">Equation 2</xref>. The remaining bandwidth <inline-formula id="inf14">
<mml:math id="m15">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>f</mml:mi>
<mml:mi>r</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>e</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the link is the difference between the total bandwidth <inline-formula id="inf15">
<mml:math id="m16">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>a</mml:mi>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the current link and the occupied bandwidth <inline-formula id="inf16">
<mml:math id="m17">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mi>s</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, as shown in <xref ref-type="disp-formula" rid="e3">Equation 3</xref>.<disp-formula id="e2">
<mml:math id="m18">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mi>s</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mi>t</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mi>u</mml:mi>
<mml:mi>r</mml:mi>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mi>t</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mi>u</mml:mi>
<mml:mi>r</mml:mi>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(2)</label>
</disp-formula>
<disp-formula id="e3">
<mml:math id="m19">
<mml:mrow>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>f</mml:mi>
<mml:mi>r</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>e</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>a</mml:mi>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mi>s</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
<label>(3)</label>
</disp-formula>
</p>
<fig id="F3" position="float">
<label>FIGURE 3</label>
<caption>
<p>Remaining bandwidth measurement.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g003.tif">
<alt-text content-type="machine-generated">Diagram illustrating a network with an SDN controller connected to two nodes, \( S_1 \) and \( S_2 \). Arrows indicate port statistics requests and replies between the controller and nodes. Ports 1 and 2 are labeled, with data boxes showing transmission (\( tx\_bytes \)), reception (\( rx\_bytes \)), and duration for each node interaction.</alt-text>
</graphic>
</fig>
</sec>
<sec id="s3-1-4">
<title>3.1.4 Packet loss rate measurement</title>
<p>The packet loss rate measurement method of the SDN link is shown in <xref ref-type="fig" rid="F4">Figure 4</xref>. The controller sends a Port Stats Request message to nodes S<sub>1</sub> and S<sub>2</sub> to query the node status. The node replies with a Port Stats Reply message. The controller parses the data packet of the reply message and obtains the number of sent data packets <inline-formula id="inf17">
<mml:math id="m20">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf18">
<mml:math id="m21">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, and the number of received data packets <inline-formula id="inf19">
<mml:math id="m22">
<mml:mrow>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf20">
<mml:math id="m23">
<mml:mrow>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of each node forwarding port statistics. The data packets are transmitted from S<sub>1</sub> and S<sub>2</sub>. The packet loss rate of the link <inline-formula id="inf21">
<mml:math id="m24">
<mml:mrow>
<mml:msub>
<mml:mi>l</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the ratio of the number of lost data packets (the difference between the number of data packets sent by node S<sub>1</sub> <inline-formula id="inf22">
<mml:math id="m25">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and the number of data packets received by node S<sub>2</sub> <inline-formula id="inf23">
<mml:math id="m26">
<mml:mrow>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>) to the number of data packets sent by node S<sub>1</sub> <inline-formula id="inf24">
<mml:math id="m27">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, as shown in <xref ref-type="disp-formula" rid="e4">Equation 4</xref>.<disp-formula id="e4">
<mml:math id="m28">
<mml:mrow>
<mml:msub>
<mml:mi>l</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>r</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(4)</label>
</disp-formula>
</p>
<fig id="F4" position="float">
<label>FIGURE 4</label>
<caption>
<p>Packet loss rate measurement.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g004.tif">
<alt-text content-type="machine-generated">Diagram showing an SDN (Software Defined Network) controller interacting with two nodes, Node S1 and Node S2. Dashed arrows indicate &#x22;Port Stats Request&#x22; and &#x22;Port Stats Reply&#x22; between the controller and nodes. Two nodes are connected via Port1 and Port2, respectively. Each node shows packet information: Node S1 has &#x22;tx_packets = tp1&#x22; and &#x22;rx_packets = rp1&#x22;; Node S2 has &#x22;tx_packets = tp2&#x22; and &#x22;rx_packets = rp2&#x22;.</alt-text>
</graphic>
</fig>
</sec>
</sec>
<sec id="s3-2">
<title>3.2 Proximal policy optimization algorithm</title>
<p>The PPO algorithm is an improvement on the trust region policy optimization (TRPO) algorithm (<xref ref-type="bibr" rid="B15">Li and He, 2023</xref>). Its main advantage is to limit the distance between the new policy <inline-formula id="inf25">
<mml:math id="m29">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and the old policy <inline-formula id="inf26">
<mml:math id="m30">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:msub>
<mml:mi>&#x3b8;</mml:mi>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in each update to ensure that the policy update amplitude is not too large, thereby improving the stability of the algorithm.</p>
<p>The basic framework of the PPO algorithm can be divided into two parts, the Actor network and the Critic network. The Actor network is a policy network that maps the state <inline-formula id="inf27">
<mml:math id="m31">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at time t to the probability distribution of the action <inline-formula id="inf28">
<mml:math id="m32">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>. While the Critic is a value network that evaluates the value of the current state <inline-formula id="inf29">
<mml:math id="m33">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>. In the traditional policy gradient update algorithm, after the agent collects experience samples according to the strategy <inline-formula id="inf30">
<mml:math id="m34">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, the parameter <inline-formula id="inf31">
<mml:math id="m35">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> will be updated. Then the agent will re-collect experience samples according to the new strategy, which will lead to a decrease in sample utilization. To solve this problem, the PPO algorithm uses the importance sampling method shown in <xref ref-type="disp-formula" rid="e5">Equation 5</xref> to limit the policy update range, thereby improving the utilization of experience samples, where <inline-formula id="inf32">
<mml:math id="m36">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf33">
<mml:math id="m37">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:msub>
<mml:mi>&#x3b8;</mml:mi>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represent the new and old policy probabilities of taking action <inline-formula id="inf34">
<mml:math id="m38">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> under state <inline-formula id="inf35">
<mml:math id="m39">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.<disp-formula id="e5">
<mml:math id="m40">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:msub>
<mml:mi>&#x3b8;</mml:mi>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(5)</label>
</disp-formula>
</p>
<p>The PPO algorithm can be refined into PPO-penalty with adaptive KL-divergence and PPO-clip with clip clipping according to the policy update method of the Actor network. The objective functions of PPO-Penalty and PPO-Clip are defined in <xref ref-type="disp-formula" rid="e6">Equations 6</xref>, <xref ref-type="disp-formula" rid="e7">7</xref>.<disp-formula id="e6">
<mml:math id="m41">
<mml:mrow>
<mml:msup>
<mml:mi>L</mml:mi>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>p</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3b2;</mml:mi>
<mml:mi>K</mml:mi>
<mml:mi>L</mml:mi>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:msub>
<mml:mi>&#x3b8;</mml:mi>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mo>&#xb7;</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mo>&#xb7;</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(6)</label>
</disp-formula>
<disp-formula id="e7">
<mml:math id="m42">
<mml:mrow>
<mml:msup>
<mml:mi>L</mml:mi>
<mml:mrow>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mi>min</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(7)</label>
</disp-formula>where <inline-formula id="inf36">
<mml:math id="m43">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the advantage function, which is used to measure the quality of the agent&#x2019;s actions in the policy network. <inline-formula id="inf37">
<mml:math id="m44">
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is the penalty term coefficient. <inline-formula id="inf38">
<mml:math id="m45">
<mml:mrow>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the clipping factor. Studies have shown that PPO-Clip has more efficient sample utilization and feasibility than PPO-Penalty.</p>
</sec>
<sec id="s3-3">
<title>3.3 System architecture and network model</title>
<sec id="s3-3-1">
<title>3.3.1 System architecture</title>
<p>The SDN architecture decouples network control from data forwarding, thereby improving resource utilization and enhancing network transmission performance. This separation offers new opportunities for enabling resilient and adaptive tactical communication networks with intelligent capabilities (<xref ref-type="bibr" rid="B3">Bale et al., 2023</xref>). Accordingly, this study designs a tactical communication network path optimization system architecture based on SDN, as illustrated in <xref ref-type="fig" rid="F5">Figure 5</xref>, which comprises three layers: the data layer, control layer, and application layer.</p>
<fig id="F5" position="float">
<label>FIGURE 5</label>
<caption>
<p>System architecture.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g005.tif">
<alt-text content-type="machine-generated">Flowchart illustrating a networked defense system. The top section is the &#x22;Application plane&#x22; with situational awareness and decision-making. Below is the &#x22;Control plane&#x22; containing a data processing module and an intelligent optimization module with agent-actor-critic mechanism. The &#x22;Data plane&#x22; at the bottom shows interconnected military vehicles and personnel, representing a tactical network. Arrows indicate data flows between modules and planes, including interfaces for information collection and flow rule distribution.</alt-text>
</graphic>
</fig>
<sec id="s3-3-1-1">
<title>3.3.1.1 Data layer</title>
<p>The data layer is responsible for deploying the tactical communication network, consisting of various routing nodes and communication links. It primarily handles the transmission of mission-critical traffic. Through the southbound interface, it reports real-time network information to the control layer and receives control policies from the control layer, executing corresponding operations such as data processing and forwarding.</p>
</sec>
<sec id="s3-3-1-2">
<title>3.3.1.2 Control layer</title>
<p>Serving as the core of the path optimization system, the control layer periodically issues requests to the data layer via the southbound interface to acquire global network information, including topology and link states. It then transmits optimized control strategies back to the data layer. A data processing module within the control layer maintains statistical records of global network metrics such as average throughput, average end-to-end delay, and average packet loss rate. These metrics are used as inputs to the reward function of the intelligent routing optimization module.</p>
<p>The control layer consists of five functional modules: network perception, network monitoring, data processing, intelligent decision-making, and path installation. The network perception module periodically issues feature request messages to the data layer to obtain the topology information of the tactical communication network. The network monitoring module periodically sends status request messages and asynchronously receives status reply messages to acquire the port status information of routing nodes. The data processing module utilizes the topology and port status data collected by the network perception and monitoring modules to compute link state and end-to-end path state metrics, including residual bandwidth, delay, and packet loss rate. These metrics serve as the network state inputs (<inline-formula id="inf39">
<mml:math id="m46">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>) and reward signals (<inline-formula id="inf40">
<mml:math id="m47">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>) for model training within the intelligent agent.</p>
<p>The intelligent decision-making module employs a learning agent that takes the network state as input and trains a model to generate forwarding actions for traffic in the data layer. The path installation module translates the agent&#x2019;s output actions into flow tables, which are then deployed to the data layer to guide service transmission.</p>
<p>The intelligent decision-making module in the control layer adopts a smart routing decision model based on the PPO algorithm within an actor-critic AC framework. To capture the spatiotemporal dependencies inherent in link state information, the proposed method integrates GRU and GAT in place of the traditional feedforward neural network employed in standard PPO architectures. This design enhances the agent&#x2019;s perception of link dynamics and improves the overall intelligence of routing decisions.</p>
<p>The control layer normalizes the collected network topology and link state information into a unified network model, which is subsequently fed into the deep reinforcement learning agent for training. The agent dynamically adjusts the optimal routing policy by continuously interacting with real-time state information, aiming to maximize the accumulated reward.</p>
</sec>
<sec id="s3-3-1-3">
<title>3.3.1.3 Application layer</title>
<p>The application layer encompasses a wide range of tactical services and applications, including situational awareness, command and control, and weapon engagement. It interacts with the control layer through a northbound interface to facilitate information exchange.</p>
</sec>
</sec>
<sec id="s3-3-2">
<title>3.3.2 Network model</title>
<p>The tactical communication network topology model is graph <inline-formula id="inf41">
<mml:math id="m48">
<mml:mrow>
<mml:mi>G</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, where <inline-formula id="inf42">
<mml:math id="m49">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>m</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the set of all routing nodes, <inline-formula id="inf43">
<mml:math id="m50">
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>V</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the number of routing nodes. <inline-formula id="inf44">
<mml:math id="m51">
<mml:mrow>
<mml:mi>E</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>23</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the set of all communication links, where <inline-formula id="inf45">
<mml:math id="m52">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the link between adjacent nodes <inline-formula id="inf46">
<mml:math id="m53">
<mml:mrow>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf47">
<mml:math id="m54">
<mml:mrow>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, and <inline-formula id="inf48">
<mml:math id="m55">
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the number of links. The path from the source node <inline-formula id="inf49">
<mml:math id="m56">
<mml:mrow>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>s</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> to the destination node <inline-formula id="inf50">
<mml:math id="m57">
<mml:mrow>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>d</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in the network is recorded as <inline-formula id="inf51">
<mml:math id="m58">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, where <inline-formula id="inf52">
<mml:math id="m59">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> consists of multiple communication links, namely, <inline-formula id="inf53">
<mml:math id="m60">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>b</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>c</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. <inline-formula id="inf54">
<mml:math id="m61">
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf55">
<mml:math id="m62">
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> represent the number of source nodes used to send services and the number of destination nodes used to receive services in the network, respectively. At the same time, the basic QoS indicators (delay, bandwidth, packet loss rate) used to describe the quality of service of communication links and paths are used, and the characteristics of their measurement functions can be defined as: additive parameters, concave parameters, and multiplicative parameters. The QoS parameter definitions are shown in <xref ref-type="table" rid="T1">Table 1</xref>.</p>
<table-wrap id="T1" position="float">
<label>TABLE 1</label>
<caption>
<p>QoS parameter definition.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="left">QoS parameter type</th>
<th align="left">Calculation formula</th>
<th align="left">Application parameter</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="left">Additive parameter</td>
<td align="center">
<inline-formula id="inf56">
<mml:math id="m63">
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mo>&#x2200;</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">Delay</td>
</tr>
<tr>
<td align="left">Concave parameter</td>
<td align="center">
<inline-formula id="inf57">
<mml:math id="m64">
<mml:mrow>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:munder>
<mml:mi>min</mml:mi>
<mml:mrow>
<mml:mo>&#x2200;</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:munder>
<mml:mi>b</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">Residual bandwidth</td>
</tr>
<tr>
<td align="left">Multiplicative parameter</td>
<td align="center">
<inline-formula id="inf58">
<mml:math id="m65">
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:munder>
<mml:mo>&#x220f;</mml:mo>
<mml:mrow>
<mml:mo>&#x2200;</mml:mo>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:munder>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>l</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">Packet loss rate</td>
</tr>
</tbody>
</table>
</table-wrap>
<p>The set of all end-to-end paths through which service flows exist in the global network is recorded as <inline-formula id="inf59">
<mml:math id="m66">
<mml:mrow>
<mml:msup>
<mml:mi>P</mml:mi>
<mml:mi>F</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
<mml:mi>f</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
<mml:mi>f</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>N</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, where <inline-formula id="inf60">
<mml:math id="m67">
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:msup>
<mml:mi>P</mml:mi>
<mml:mi>F</mml:mi>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>N</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> indicates that there are <italic>N</italic> service flows in the network. The performance indicators in the tactical communication network, such as average network throughput <inline-formula id="inf61">
<mml:math id="m68">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>t</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mi>h</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, average end-to-end delay <inline-formula id="inf62">
<mml:math id="m69">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>d</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula>, and average packet loss rate <inline-formula id="inf63">
<mml:math id="m70">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>l</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula>, are defined as follows:</p>
<p>
<statement content-type="definition" id="Definition_1">
<label>Definition 1</label>
<p>Average network throughput is presented in <xref ref-type="disp-formula" rid="e8">Equation 8</xref>. The average network throughput represents the average value of the total amount of data successfully transmitted through the end-to-end paths of all service flows per unit time.<disp-formula id="e8">
<mml:math id="m72">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>t</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mi>h</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>N</mml:mi>
</mml:mrow>
</mml:mfrac>
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>N</mml:mi>
</mml:munderover>
</mml:mstyle>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(8)</label>
</disp-formula>where <inline-formula id="inf65">
<mml:math id="m73">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the amount of data successfully transmitted through the path <inline-formula id="inf66">
<mml:math id="m74">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of service flow f per unit time.</p>
</statement>
</p>
<p>
<statement content-type="definition" id="Definition_2">
<label>Definition 2</label>
<p>Average end-to-end delay is presented in <xref ref-type="disp-formula" rid="e9">Equation 9</xref>. The average end-to-end delay represents the average value of the sum of the end-to-end delays of all service flows through the paths.<disp-formula id="e9">
<mml:math id="m76">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>d</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>N</mml:mi>
</mml:mrow>
</mml:mfrac>
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>N</mml:mi>
</mml:munderover>
</mml:mstyle>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(9)</label>
</disp-formula>where <inline-formula id="inf68">
<mml:math id="m77">
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the end-to-end delay of the path <inline-formula id="inf69">
<mml:math id="m78">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> through which service flow <italic>f</italic> passes.</p>
</statement>
</p>
<p>
<statement content-type="definition" id="Definition_3">
<label>Definition 3</label>
<p>Average packet loss rate is presented in <xref ref-type="disp-formula" rid="e10">Equation 10</xref>. The average packet loss rate represents the average value of the sum of the packet loss rates of all service flows through the paths.<disp-formula id="e10">
<mml:math id="m80">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>l</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>N</mml:mi>
</mml:mrow>
</mml:mfrac>
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>N</mml:mi>
</mml:munderover>
</mml:mstyle>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(10)</label>
</disp-formula>where <inline-formula id="inf71">
<mml:math id="m81">
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>f</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the packet loss rate of the path <inline-formula id="inf72">
<mml:math id="m82">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> through which service flow <italic>f</italic> passes.</p>
</statement>
</p>
</sec>
</sec>
</sec>
<sec id="s4">
<title>4 Design of intelligent path optimization algorithm</title>
<sec id="s4-1">
<title>4.1 PPO agent</title>
<p>The intelligent optimization algorithm proposed in this paper uses PPO to construct a deep reinforcement learning agent for model training. PPO is more versatile and stable than DDPG, actor-critic and other algorithms (<xref ref-type="bibr" rid="B16">Li and Wu, 2025</xref>). The PPO algorithm is implemented based on the AC network architecture. The policy network Actor is used to learn the mean of the network environment state output action, and the evaluation network Critic is used to output the state value. The agent interacts with the environment through state, action and reward information, and continuously optimizes the strategy through iterative training.</p>
<p>As a framework-based deep reinforcement learning algorithm, PPO requires the design of different state spaces, action spaces and reward functions for different problems and different application scenarios. In this paper, based on the PPO deep reinforcement learning framework, the state space, action space and reward function are designed for the tactical communication network scenario, as follows:<list list-type="simple">
<list-item>
<p>1. State space. The state space is the set of link state information obtained by the agent from the tactical communication network. The network state <inline-formula id="inf73">
<mml:math id="m83">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at any time <italic>t</italic> includes the global network service request <inline-formula id="inf74">
<mml:math id="m84">
<mml:mrow>
<mml:msubsup>
<mml:mi>Q</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula>, the link remaining bandwidth <inline-formula id="inf75">
<mml:math id="m85">
<mml:mrow>
<mml:msubsup>
<mml:mi>B</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula>, the delay <inline-formula id="inf76">
<mml:math id="m86">
<mml:mrow>
<mml:msubsup>
<mml:mi>D</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> and the packet loss rate <inline-formula id="inf77">
<mml:math id="m87">
<mml:mrow>
<mml:msubsup>
<mml:mi>L</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> information, as shown in <xref ref-type="disp-formula" rid="e11">Equation 11</xref>.</p>
</list-item>
</list>
<disp-formula id="e11">
<mml:math id="m88">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>Q</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>B</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>D</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>L</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(11)</label>
</disp-formula>where <inline-formula id="inf78">
<mml:math id="m89">
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>4</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> represents the above-mentioned state feature dimension, and n represents the total number of network links. <inline-formula id="inf79">
<mml:math id="m90">
<mml:mrow>
<mml:msubsup>
<mml:mi>Q</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> represents the service request information on the global network link at time t, and its specific representation is shown in <xref ref-type="disp-formula" rid="e12">Equation 12</xref>. This algorithm defines that when there is a service flow passing through the link, its element is assigned to 1, that is, <inline-formula id="inf80">
<mml:math id="m91">
<mml:mrow>
<mml:msubsup>
<mml:mi>q</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:msubsup>
<mml:mi>q</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>, and the link without service flow passing through and the link composed of non-adjacent nodes, its element is assigned to 0.<disp-formula id="e12">
<mml:math id="m92">
<mml:mrow>
<mml:msubsup>
<mml:mi>Q</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>q</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>q</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>23</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>q</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>i</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(12)</label>
</disp-formula>
</p>
<p>The specific representation of each network link state information set is shown in <xref ref-type="disp-formula" rid="e13">Equations 13</xref>&#x2013;<xref ref-type="disp-formula" rid="e15">15</xref>. Due to the large differences in the element values in the link state information, this difference will cause the intelligent path optimization algorithm to fluctuate greatly during the training process and it is difficult to reach a convergence state. Therefore, the Min-Max method is used to normalize the elements in the link state information.<disp-formula id="e13">
<mml:math id="m93">
<mml:mrow>
<mml:msubsup>
<mml:mi>B</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>b</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>b</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>23</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>b</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>i</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(13)</label>
</disp-formula>
<disp-formula id="e14">
<mml:math id="m94">
<mml:mrow>
<mml:msubsup>
<mml:mi>D</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>d</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>d</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>23</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>d</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>i</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(14)</label>
</disp-formula>
<disp-formula id="e15">
<mml:math id="m95">
<mml:mrow>
<mml:msubsup>
<mml:mi>L</mml:mi>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>l</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>l</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mn>23</mml:mn>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>l</mml:mi>
<mml:msub>
<mml:mi>e</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>t</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>i</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(15)</label>
</disp-formula>
</p>
<p>The state space of the agent is designed to be a state sequence formed by combining the state information containing time <italic>t</italic> and l&#x2212;1 time steps before the time, as shown in <xref ref-type="disp-formula" rid="e16">Equation 16</xref>.<disp-formula id="e16">
<mml:math id="m96">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>l</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>l</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(16)</label>
</disp-formula>
<list list-type="simple">
<list-item>
<p>2. Action space. The action space refers to the set of actions taken by the agent according to the state <inline-formula id="inf81">
<mml:math id="m97">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> during the interaction with the network environment, which is used to forward the business flow in the network. Assume that each source-destination node pair in the global network contains k feasible paths to form a feasible path set <inline-formula id="inf82">
<mml:math id="m98">
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mn>3</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mi>k</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, where each path corresponds to a weight coefficient, namely, <inline-formula id="inf83">
<mml:math id="m99">
<mml:mrow>
<mml:msup>
<mml:mi>W</mml:mi>
<mml:mi>P</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>w</mml:mi>
<mml:mn>1</mml:mn>
<mml:mi>p</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>w</mml:mi>
<mml:mn>2</mml:mn>
<mml:mi>p</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>w</mml:mi>
<mml:mi>k</mml:mi>
<mml:mi>p</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. The algorithm in this paper defines each action in the action space as the forwarding path weight between all source-destination node pairs in the global network, specifically expressed as <inline-formula id="inf84">
<mml:math id="m100">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. Among them, <inline-formula id="inf85">
<mml:math id="m101">
<mml:mrow>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi>W</mml:mi>
<mml:mi>P</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf86">
<mml:math id="m102">
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf87">
<mml:math id="m103">
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represent the weight of the forwarding path <italic>p</italic> selected between the source node <italic>i</italic> and the destination node <italic>j</italic>, and <inline-formula id="inf88">
<mml:math id="m104">
<mml:mrow>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>0</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>3. Reward function. The control layer feeds back the collected real-time network performance indicators as reward signals to the agent. The optimization goal of the algorithm in this paper is set to maximize network throughput and minimize end-to-end delay and packet loss. In order to make the optimization goal increase proportionally with the reward value, the reward function is defined as shown in <xref ref-type="disp-formula" rid="e17">Equation 17</xref>.</p>
</list-item>
</list>
<disp-formula id="e17">
<mml:math id="m105">
<mml:mrow>
<mml:mi>R</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3b1;</mml:mi>
<mml:mo>&#xb7;</mml:mo>
<mml:mover accent="true">
<mml:mi>t</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:msup>
<mml:mi>h</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b2;</mml:mi>
<mml:mo>&#xb7;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msup>
<mml:mover accent="true">
<mml:mi>d</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b3;</mml:mi>
<mml:mo>&#xb7;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msup>
<mml:mover accent="true">
<mml:mi>l</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(17)</label>
</disp-formula>where <inline-formula id="inf89">
<mml:math id="m106">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf90">
<mml:math id="m107">
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf91">
<mml:math id="m108">
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is defined as the reward weight, and the value range is <inline-formula id="inf92">
<mml:math id="m109">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>&#x3b2;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>&#x3b3;</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>0</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. These weights are used to define the importance of different network performance indicators. In order to avoid the impact of network performance indicator differences on the convergence performance of the algorithm, they are normalized, where <inline-formula id="inf93">
<mml:math id="m110">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>t</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:msup>
<mml:mi>h</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf94">
<mml:math id="m111">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>d</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf95">
<mml:math id="m112">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>l</mml:mi>
<mml:mo>&#xaf;</mml:mo>
</mml:mover>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> represents the performance indicator after normalization.</p>
<p>The PPO algorithm policy update stage uses the advantage function <inline-formula id="inf96">
<mml:math id="m113">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> to measure the quality of each action. The advantage function is defined as follows:<disp-formula id="e18">
<mml:math id="m114">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>R</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b3;</mml:mi>
<mml:mo>&#xb7;</mml:mo>
<mml:mi>V</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>V</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(18)</label>
</disp-formula>where <inline-formula id="inf97">
<mml:math id="m115">
<mml:mrow>
<mml:msub>
<mml:mi>R</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the reward for executing action <inline-formula id="inf98">
<mml:math id="m116">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in state <inline-formula id="inf99">
<mml:math id="m117">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf100">
<mml:math id="m118">
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the reward discount factor, <inline-formula id="inf101">
<mml:math id="m119">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf102">
<mml:math id="m120">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represent the evaluation values of the current state <inline-formula id="inf103">
<mml:math id="m121">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and the next state <inline-formula id="inf104">
<mml:math id="m122">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> respectively.</p>
<p>The traditional policy gradient update algorithm will cause serious deviations in the execution strategy of the agent due to the inaccurate estimation of the advantage function [33]. Therefore, this paper adopts the importance sampling method to adjust the strategy update amplitude and improve the efficiency of strategy update and sample utilization during training. The importance sampling is presented in <xref ref-type="disp-formula" rid="e19">Equation 19</xref>.<disp-formula id="e19">
<mml:math id="m123">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:msub>
<mml:mi>&#x3b8;</mml:mi>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:msub>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="|" close="" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(19)</label>
</disp-formula>where <inline-formula id="inf105">
<mml:math id="m124">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the ratio of the probability that the current strategy <inline-formula id="inf106">
<mml:math id="m125">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> takes action <inline-formula id="inf107">
<mml:math id="m126">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in state <inline-formula id="inf108">
<mml:math id="m127">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> to the probability that the old strategy takes action <inline-formula id="inf109">
<mml:math id="m128">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in state <inline-formula id="inf110">
<mml:math id="m129">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
<p>To better adapt to the dynamic changes of the network topology and the uneven distribution of samples, the algorithm in this paper adopts a gradient clipping method defined as <xref ref-type="disp-formula" rid="e20">Equation 20</xref>, which effectively limits the magnitude of policy updates. The policy update of the algorithm is shown in <xref ref-type="disp-formula" rid="e21">Equation 21</xref>. While restricting the extent of policy updates, the algorithm also aims to maximize the expected cumulative reward, thereby improving the convergence and stability of the algorithm.<disp-formula id="e20">
<mml:math id="m130">
<mml:mrow>
<mml:msup>
<mml:mi>L</mml:mi>
<mml:mrow>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mi>min</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(20)</label>
</disp-formula>
<disp-formula id="e21">
<mml:math id="m131">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>&#x3b8;</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>argmax</mml:mi>
<mml:msub>
<mml:mi>E</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mi>min</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(21)</label>
</disp-formula>where, <inline-formula id="inf111">
<mml:math id="m132">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>&#x3b8;</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula> represents the updated policy parameters. <inline-formula id="inf112">
<mml:math id="m133">
<mml:mrow>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> denotes the clipping factor, usually represented by a small positive number, which limits the extent of policy updates within the range of <inline-formula id="inf113">
<mml:math id="m134">
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</sec>
<sec id="s4-2">
<title>4.2 Link state feature extraction module</title>
<p>In order to better extract the temporal and spatial characteristics of the tactical communication network link state information, this paper introduces GRU, GAT, and a self-attention mechanism to construct a link state feature extraction module, replacing the feedforward neural network structure of PPO. This enables the perception of the spatiotemporal characteristics of link state information and realizes end-to-end optimal action policy output. As shown in <xref ref-type="fig" rid="F6">Figure 6</xref>, the link state feature extraction module mainly consists of a temporal feature extraction module, a self-attention mechanism module, a spatial feature extraction module, and a multilayer perceptron module.</p>
<fig id="F6" position="float">
<label>FIGURE 6</label>
<caption>
<p>Link state information feature extraction module.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g006.tif">
<alt-text content-type="machine-generated">Flowchart illustrating a model architecture. Input, labeled &#x22;Network state sequence,&#x22; is processed through a &#x22;Temporal feature extraction module&#x22; with GRUs. It then passes through &#x22;Self-attention mechanism&#x22; (SAM), &#x22;Spatial feature extraction&#x22; (GAT), and &#x22;Multi-layer perceptron&#x22; (MLP). The output is &#x22;Path weight.&#x22; Arrows indicate data flow direction.</alt-text>
</graphic>
</fig>
<sec id="s4-2-1">
<title>4.2.1 Time feature extraction module</title>
<p>The time correlation of link state refers to the mutual dependence between link state data within a certain time range, and is also the dynamic change of link state information over time. Therefore, when considering the time correlation of link state information, this paper needs to input a network state sequence with a time step of <italic>l</italic> for the agent. When the initial network state <inline-formula id="inf114">
<mml:math id="m135">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is obtained, the state of the previous <italic>l-1</italic> historical moments cannot be obtained, so its sequence element is assigned to 0. With the iterative update of reinforcement learning, the state sequence is also updated. The update gate <inline-formula id="inf115">
<mml:math id="m136">
<mml:mrow>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of GRU takes the hidden state <inline-formula id="inf116">
<mml:math id="m137">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the previous moment and the network state <inline-formula id="inf117">
<mml:math id="m138">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of the current moment as input, and calculates a value between [0,1] through the sigmoid function to determine the degree of information retention in <inline-formula id="inf118">
<mml:math id="m139">
<mml:mrow>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, as shown in <xref ref-type="disp-formula" rid="e22">Equation 22</xref>.<disp-formula id="e22">
<mml:math id="m140">
<mml:mrow>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>s</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>g</mml:mi>
<mml:mi>m</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>z</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mi>U</mml:mi>
<mml:mi>z</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(22)</label>
</disp-formula>where <inline-formula id="inf119">
<mml:math id="m141">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>z</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix, and <inline-formula id="inf120">
<mml:math id="m142">
<mml:mrow>
<mml:msub>
<mml:mi>U</mml:mi>
<mml:mi>z</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix of <inline-formula id="inf121">
<mml:math id="m143">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
<p>The reset gate <inline-formula id="inf122">
<mml:math id="m144">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> has a similar function to the update gate. The <inline-formula id="inf123">
<mml:math id="m145">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> input is the hidden state <inline-formula id="inf124">
<mml:math id="m146">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at the previous moment and the network state <inline-formula id="inf125">
<mml:math id="m147">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at the current moment. The sigmoid function calculates a value between [0,1] to determine the degree of information retention in <inline-formula id="inf126">
<mml:math id="m148">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, as shown in <xref ref-type="disp-formula" rid="e23">Equation 23</xref>.<disp-formula id="e23">
<mml:math id="m149">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>s</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>g</mml:mi>
<mml:mi>m</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>d</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>r</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mi>U</mml:mi>
<mml:mi>r</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(23)</label>
</disp-formula>where <inline-formula id="inf127">
<mml:math id="m150">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>r</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix, and <inline-formula id="inf128">
<mml:math id="m151">
<mml:mrow>
<mml:msub>
<mml:mi>U</mml:mi>
<mml:mi>r</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix of <inline-formula id="inf129">
<mml:math id="m152">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
<p>As shown in <xref ref-type="disp-formula" rid="e24">Equation 24</xref>, the output of the reset gate at the current time step (<inline-formula id="inf130">
<mml:math id="m153">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>) is multiplied elementwise with the previous hidden state (<inline-formula id="inf131">
<mml:math id="m154">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>). The resulting vector is then concatenated with the current network state (<inline-formula id="inf132">
<mml:math id="m155">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>) and passed through a tanh activation function to compute the candidate hidden state (<inline-formula id="inf133">
<mml:math id="m156">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>).<disp-formula id="e24">
<mml:math id="m157">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>tanh</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mi>U</mml:mi>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2299;</mml:mo>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(24)</label>
</disp-formula>where <inline-formula id="inf134">
<mml:math id="m158">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix. <inline-formula id="inf135">
<mml:math id="m159">
<mml:mrow>
<mml:mo>&#x2299;</mml:mo>
</mml:mrow>
</mml:math>
</inline-formula> represents the Hadamard product.</p>
<p>The output of the hidden state is shown in <xref ref-type="disp-formula" rid="e25">Equation 25</xref>. The hidden state <inline-formula id="inf136">
<mml:math id="m160">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at the previous moment and the candidate hidden state <inline-formula id="inf137">
<mml:math id="m161">
<mml:mrow>
<mml:msub>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at the current moment are combined through the update gate <inline-formula id="inf138">
<mml:math id="m162">
<mml:mrow>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at the current moment to obtain the next hidden state <inline-formula id="inf139">
<mml:math id="m163">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, which is passed to the neuron of the next time step, and the weight parameters of the GRU network model are continuously updated, thereby realizing the extraction of time features.<disp-formula id="e25">
<mml:math id="m164">
<mml:mrow>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2299;</mml:mo>
<mml:msub>
<mml:mi>h</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mi>z</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2299;</mml:mo>
<mml:msub>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
<label>(25)</label>
</disp-formula>
</p>
<p>Since the state sequence <inline-formula id="inf140">
<mml:math id="m165">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> input by the agent contains <italic>l</italic> time steps, the GRU contains <italic>l</italic> units, and the output of its hidden layer is <inline-formula id="inf141">
<mml:math id="m166">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>, as shown in <xref ref-type="disp-formula" rid="e26">Equation 26</xref>.<disp-formula id="e26">
<mml:math id="m167">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>1</mml:mn>
<mml:mi>R</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>2</mml:mn>
<mml:mi>R</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>R</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(26)</label>
</disp-formula>
</p>
</sec>
<sec id="s4-2-2">
<title>4.2.2 Self-attention mechanism module</title>
<p>The self-attention mechanism is used to capture the dependencies and importance among different samples in the state sequence. It determines the importance of each sample in model computation by calculating the correlations (i.e., attention weights) between different samples in the sequence, thereby enhancing the model&#x2019;s ability to perceive the critical parts of the input sequence. According to <xref ref-type="disp-formula" rid="e27">Equations 27</xref>&#x2013;<xref ref-type="disp-formula" rid="e29">29</xref>, the outputs of the GRU hidden layer are multiplied by three weight matrices <inline-formula id="inf142">
<mml:math id="m168">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>Q</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf143">
<mml:math id="m169">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>K</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf144">
<mml:math id="m170">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>V</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> to compute the query matrix (<italic>Q</italic>), key matrix (<italic>K</italic>), and value matrix (<italic>V</italic>).<disp-formula id="e27">
<mml:math id="m171">
<mml:mrow>
<mml:mi>Q</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>Q</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(27)</label>
</disp-formula>
<disp-formula id="e28">
<mml:math id="m172">
<mml:mrow>
<mml:mi>K</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>K</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(28)</label>
</disp-formula>
<disp-formula id="e29">
<mml:math id="m173">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>V</mml:mi>
</mml:msub>
<mml:mo>&#xb7;</mml:mo>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(29)</label>
</disp-formula>
</p>
<p>Then, the attention weight of each unit is computed, and the attention matrix <inline-formula id="inf145">
<mml:math id="m174">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> is generated, as shown in <xref ref-type="disp-formula" rid="e30">Equations 30</xref>, <xref ref-type="disp-formula" rid="e31">31</xref>.<disp-formula id="e30">
<mml:math id="m175">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>t</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>K</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>V</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:munderover>
</mml:mstyle>
<mml:mi>s</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>f</mml:mi>
<mml:mi>t</mml:mi>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>max</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:msubsup>
<mml:mi>k</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>T</mml:mi>
</mml:msubsup>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>q</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
<mml:msqrt>
<mml:msub>
<mml:mi>d</mml:mi>
<mml:mi>k</mml:mi>
</mml:msub>
</mml:msqrt>
</mml:mfrac>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#xb7;</mml:mo>
<mml:msub>
<mml:mi>v</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
<label>(30)</label>
</disp-formula>
<disp-formula id="e31">
<mml:math id="m176">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>1</mml:mn>
<mml:mi>&#x3b1;</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>2</mml:mn>
<mml:mi>&#x3b1;</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(31)</label>
</disp-formula>where, <italic>M</italic> represents the dimension of the weight matrix, and <inline-formula id="inf146">
<mml:math id="m177">
<mml:mrow>
<mml:msqrt>
<mml:msub>
<mml:mi>d</mml:mi>
<mml:mi>k</mml:mi>
</mml:msub>
</mml:msqrt>
</mml:mrow>
</mml:math>
</inline-formula> is the scaling factor.</p>
<p>Finally, the Hadamard product of <inline-formula id="inf147">
<mml:math id="m178">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf148">
<mml:math id="m179">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> is calculated, and a weighted average operation is performed along the l-dimension on the resulting matrix to obtain the output <inline-formula id="inf149">
<mml:math id="m180">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> of the self-attention module, as shown in <xref ref-type="disp-formula" rid="e32">Equation 32</xref>.<disp-formula id="e32">
<mml:math id="m181">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3bc;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>R</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>R</mml:mi>
</mml:msup>
<mml:mo>&#x2299;</mml:mo>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:mi>&#x3b1;</mml:mi>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>1</mml:mn>
<mml:mi>R</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mn>2</mml:mn>
<mml:mi>R</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>n</mml:mi>
<mml:mi>R</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(32)</label>
</disp-formula>where <inline-formula id="inf150">
<mml:math id="m182">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>R</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix, and <inline-formula id="inf151">
<mml:math id="m183">
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mo>&#xb7;</mml:mo>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> denotes the weighted average operation.</p>
</sec>
<sec id="s4-2-3">
<title>4.2.3 Spatial feature extraction module</title>
<p>GAT is capable of focusing on neighboring node information with different weights and demonstrates excellent spatial information capturing capabilities. Therefore, this paper employs GAT to extract the spatial correlation features of complex tactical communication network link state information. Since this study focuses on the link state characteristics in the tactical communication network, links are mapped to nodes in the graph structure for feature extraction. The data output from the temporal feature extraction module, after passing through the self-attention mechanism module, is denoted as <inline-formula id="inf152">
<mml:math id="m184">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>. The element <inline-formula id="inf153">
<mml:math id="m185">
<mml:mrow>
<mml:msup>
<mml:mi>H</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> in <inline-formula id="inf154">
<mml:math id="m186">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>i</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> is used as the node feature of the corresponding node in the graph structure. <inline-formula id="inf155">
<mml:math id="m187">
<mml:mrow>
<mml:msub>
<mml:mi>c</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is defined as the attention coefficient of node <italic>j</italic> to node <italic>i</italic>, as shown in <xref ref-type="disp-formula" rid="e33">Equation 33</xref>.<disp-formula id="e33">
<mml:math id="m188">
<mml:mrow>
<mml:msub>
<mml:mi>c</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="" separators="|">
<mml:mrow>
<mml:mtable columnalign="left">
<mml:mtr>
<mml:mtd>
<mml:mrow>
<mml:mi>L</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>a</mml:mi>
<mml:mi>k</mml:mi>
<mml:mi>y</mml:mi>
<mml:mtext>Re</mml:mtext>
<mml:mi>l</mml:mi>
<mml:mi>u</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>i</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
<mml:mrow>
<mml:mfenced open="&#x2016;" close="" separators="|">
<mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>j</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>A</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd>
<mml:mrow>
<mml:mn>0</mml:mn>
<mml:mo>,</mml:mo>
<mml:mtext>Others</mml:mtext>
</mml:mrow>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(33)</label>
</disp-formula>where, <inline-formula id="inf156">
<mml:math id="m189">
<mml:mrow>
<mml:mi>A</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="{" close="}" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>A</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> denotes the adjacency matrix, <inline-formula id="inf157">
<mml:math id="m190">
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mo>&#xb7;</mml:mo>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> is a transformation function, the symbol <inline-formula id="inf158">
<mml:math id="m191">
<mml:mrow>
<mml:mo>&#x2016;</mml:mo>
</mml:mrow>
</mml:math>
</inline-formula> indicates vector concatenation, and <inline-formula id="inf159">
<mml:math id="m192">
<mml:mrow>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the weight matrix.</p>
<p>To facilitate the calculation of attention coefficients, a softmax function is introduced to normalize the attention coefficients, as shown in <xref ref-type="disp-formula" rid="e34">Equation 34</xref>.<disp-formula id="e34">
<mml:math id="m193">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>s</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>f</mml:mi>
<mml:mi>t</mml:mi>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>max</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>c</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mi>exp</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>c</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>exp</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>c</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
<label>(34)</label>
</disp-formula>where, <inline-formula id="inf160">
<mml:math id="m194">
<mml:mrow>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the one-hop neighbor nodes of node i.</p>
<p>Then, the normalized attention coefficients are used to perform a linear aggregation of the neighborhood representations of the nodes to obtain the final output features of the node, which is illustrated in <xref ref-type="disp-formula" rid="e35">Equation 35</xref>.<disp-formula id="e35">
<mml:math id="m195">
<mml:mrow>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>A</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3c3;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>j</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(35)</label>
</disp-formula>where, <inline-formula id="inf161">
<mml:math id="m196">
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mo>&#xb7;</mml:mo>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> denotes a nonlinear activation function.</p>
<p>Since single-head attention may suffer from instability during the training process, multi-head attention is introduced to enhance the model&#x2019;s representation capability and improve the algorithm&#x2019;s stability. Specifically, <italic>K</italic> is used to represent the number of heads. The above operation is performed for each head, and the final results are concatenated. Then, an averaging operation is applied to the concatenated results, followed by a delayed application of a nonlinear function to obtain the final representation, as shown in <xref ref-type="disp-formula" rid="e36">Equation 36</xref>.<disp-formula id="e36">
<mml:math id="m197">
<mml:mrow>
<mml:msubsup>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>i</mml:mi>
<mml:mi>A</mml:mi>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mrow>
<mml:msubsup>
<mml:mi>&#x2016;</mml:mi>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>1</mml:mi>
</mml:mrow>
<mml:mi>k</mml:mi>
</mml:msubsup>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:msubsup>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
<mml:mi>k</mml:mi>
</mml:msubsup>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>j</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mrow>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3c3;</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:mfrac>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>K</mml:mi>
</mml:munderover>
</mml:mstyle>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mo>&#x2211;</mml:mo>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:msubsup>
<mml:mi>a</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
<mml:mi>k</mml:mi>
</mml:msubsup>
<mml:msub>
<mml:mi>W</mml:mi>
<mml:mi>e</mml:mi>
</mml:msub>
<mml:msubsup>
<mml:mi>h</mml:mi>
<mml:mi>j</mml:mi>
<mml:msup>
<mml:mi>R</mml:mi>
<mml:mo>&#x2032;</mml:mo>
</mml:msup>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(36)</label>
</disp-formula>
</p>
<p>The GAT module outputs all node characteristics to obtain the characteristic matrix <inline-formula id="inf162">
<mml:math id="m198">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>H</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>A</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>, which is depicted in <xref ref-type="disp-formula" rid="e37">Equation 37</xref>.<disp-formula id="e37">
<mml:math id="m199">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>H</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>A</mml:mi>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msubsup>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mn>1</mml:mn>
<mml:mi>A</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mn>2</mml:mn>
<mml:mi>A</mml:mi>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msubsup>
<mml:mover accent="true">
<mml:mi>h</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>n</mml:mi>
<mml:mi>A</mml:mi>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x2208;</mml:mo>
<mml:msup>
<mml:mi mathvariant="double-struck">R</mml:mi>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
<label>(37)</label>
</disp-formula>
</p>
</sec>
<sec id="s4-2-4">
<title>4.2.4 Multi-layer perceptron module</title>
<p>The multi-layer perceptron (MLP) module is used to implement the model&#x2019;s output function. The feature matrix <inline-formula id="inf163">
<mml:math id="m200">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>H</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>A</mml:mi>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> output by the GAT module is input into the MLP. Each layer in the MLP is composed of multiple neurons, with fully connected structures formed between layers through weight matrices and bias vectors. A softmax function is applied, and the action weight matrix <inline-formula id="inf164">
<mml:math id="m201">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at time t is finally output, as shown in <xref ref-type="disp-formula" rid="e38">Equation 38</xref>.<disp-formula id="e38">
<mml:math id="m202">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>s</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>f</mml:mi>
<mml:mi>t</mml:mi>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>max</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>M</mml:mi>
<mml:mi>L</mml:mi>
<mml:mi>P</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msup>
<mml:mover accent="true">
<mml:mi>H</mml:mi>
<mml:mo>&#x223c;</mml:mo>
</mml:mover>
<mml:mi>A</mml:mi>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
<label>(38)</label>
</disp-formula>
</p>
<p>The final output is the action weight matrix <inline-formula id="inf165">
<mml:math id="m203">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mn>12</mml:mn>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>w</mml:mi>
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, where <inline-formula id="inf166">
<mml:math id="m204">
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>j</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf167">
<mml:math id="m205">
<mml:mrow>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#xd7;</mml:mo>
<mml:mrow>
<mml:mfenced open="|" close="|" separators="|">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represent the number of neurons in the output layer.</p>
</sec>
<sec id="s4-2-5">
<title>4.2.5 DRL-SGA path optimization algorithm</title>
<p>The structure of the DRL-SGA path optimization algorithm is shown in <xref ref-type="fig" rid="F7">Figure 7</xref>.</p>
<fig id="F7" position="float">
<label>FIGURE 7</label>
<caption>
<p>DRL-SGA algorithm structure.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g007.tif">
<alt-text content-type="machine-generated">Flowchart illustrating a network management system. The upper part shows the acquisition of network state data, including service requests, residual bandwidth, latency, and packet loss rate. This data feeds into two modules: Actor and Critic, both with temporal feature extraction using GRU, self-attention mechanisms, spatial feature extraction using GAT, and multilayer perceptrons. The Actor module produces actions based on these inputs, leading to policy gradient updates. The lower part depicts a network environment with interconnected vehicles and nodes, representing service requests and updates.</alt-text>
</graphic>
</fig>
<p>The agent interacts with the network environment over T iterative cycles. At time t, the agent obtains the service request state and link state information of the tactical communication network, generates the current state sequence <inline-formula id="inf168">
<mml:math id="m206">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, and inputs it into the Actor policy network for state perception. It simultaneously obtains the current action <inline-formula id="inf169">
<mml:math id="m207">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>. After executing the action, the agent receives the reward <inline-formula id="inf170">
<mml:math id="m208">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> at time t and the next state sequence <inline-formula id="inf171">
<mml:math id="m209">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, storing the experience sample <inline-formula id="inf172">
<mml:math id="m210">
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> into the experience replay buffer. It then enters a new decision-making cycle until the Markov decision process (MDP) ends. During this process, once the number of stored experience samples meets the required amount, the agent trains by sampling the experience data to iteratively update the weight parameters of both networks, thereby optimizing the path decision model and enabling dynamic resolution of global routing. The detailed process of the DRL-SGA algorithm is shown in <xref ref-type="statement" rid="Algorithm_1">Algorithm 1</xref>.<statement content-type="algorithm" id="Algorithm_1">
<label>Algorithm 1</label>
<p>DRL-SGA algorithm.<list list-type="simple">
<list-item>
<p>Input: Reward discount factor <inline-formula id="inf173">
<mml:math id="m211">
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, actor learning rate <inline-formula id="inf174">
<mml:math id="m212">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3bb;</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, critic learning rate <inline-formula id="inf175">
<mml:math id="m213">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3bb;</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, total number of training rounds <italic>T</italic>, interaction frequency <inline-formula id="inf176">
<mml:math id="m214">
<mml:mrow>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, experience replay pool capacity <italic>M</italic>, number of experience samples <italic>D</italic>, network state sequence <inline-formula id="inf177">
<mml:math id="m215">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>Output: global network path weight</p>
</list-item>
<list-item>
<p>&#x2009;1: Initialize actor strategy network parameters <inline-formula id="inf178">
<mml:math id="m216">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> and Critic value network parameters <inline-formula id="inf179">
<mml:math id="m217">
<mml:mrow>
<mml:mi>&#x3c6;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>&#x2009;2: Initialize experience replay pool capacity <italic>M.</italic>
</p>
</list-item>
<list-item>
<p>&#x2009;3: fort <italic>episode</italic> &#x3d; 1 to <italic>T</italic> do:</p>
</list-item>
<list-item>
<p>4:&#x2003;The agent obtains the initial network state sequence <inline-formula id="inf180">
<mml:math id="m218">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mfenced open="[" close="]" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>l</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x22ef;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> at time t.</p>
</list-item>
<list-item>
<p>5:&#x2003;for <inline-formula id="inf181">
<mml:math id="m219">
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> to <inline-formula id="inf182">
<mml:math id="m220">
<mml:mrow>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>:</p>
</list-item>
<list-item>
<p>6:&#x2003;&#x2003;The Actor network generates the optimal path action <inline-formula id="inf183">
<mml:math id="m221">
<mml:mrow>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> according to the strategy <inline-formula id="inf184">
<mml:math id="m222">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3c0;</mml:mi>
<mml:mi>&#x3b8;</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and executes it.</p>
</list-item>
<list-item>
<p>7:&#x2003;&#x2003;The agent obtains the reward <inline-formula id="inf185">
<mml:math id="m223">
<mml:mrow>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and the new network state sequence <inline-formula id="inf186">
<mml:math id="m224">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>8:&#x2003;&#x2003;Store the experience sample <inline-formula id="inf187">
<mml:math id="m225">
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> in the experience replay pool.</p>
</list-item>
<list-item>
<p>9:&#x2003;&#x2003;Update the state sequence <inline-formula id="inf188">
<mml:math id="m226">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>&#x2190;</mml:mo>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> .</p>
</list-item>
<list-item>
<p>10:&#x2003;end for</p>
</list-item>
<list-item>
<p>11:&#x2003;if <inline-formula id="inf189">
<mml:math id="m227">
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>n</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mo>&#x3e;</mml:mo>
<mml:mi>D</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>:</p>
</list-item>
<list-item>
<p>12:&#x2003;for <inline-formula id="inf190">
<mml:math id="m228">
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> to D:</p>
</list-item>
<list-item>
<p>13:&#x2003;&#x2003;Collect experience samples <inline-formula id="inf191">
<mml:math id="m229">
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>r</mml:mi>
<mml:mi>t</mml:mi>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> and input them into the critic network to obtain all state values <inline-formula id="inf192">
<mml:math id="m230">
<mml:mrow>
<mml:mi>V</mml:mi>
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:msub>
<mml:mi>s</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>14:&#x2003;&#x2003;Calculate the advantage function <inline-formula id="inf193">
<mml:math id="m231">
<mml:mrow>
<mml:mover accent="true">
<mml:mi>A</mml:mi>
<mml:mo>&#x5e;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula> according to <xref ref-type="disp-formula" rid="e18">Equation 18</xref> and use back propagation to update the Critic network parameters <inline-formula id="inf194">
<mml:math id="m232">
<mml:mrow>
<mml:mi>&#x3c6;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>15:&#x2003;&#x2003;Calculate the objective function <inline-formula id="inf195">
<mml:math id="m233">
<mml:mrow>
<mml:msup>
<mml:mi>L</mml:mi>
<mml:mrow>
<mml:mi>c</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> according to <xref ref-type="disp-formula" rid="e20">Equation 20</xref> and use back propagation to update the Actor network parameters <inline-formula id="inf196">
<mml:math id="m234">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>16:&#x2003;&#x2003;end for</p>
</list-item>
<list-item>
<p>17:&#x2003;end if</p>
</list-item>
<list-item>
<p>18:end for</p>
</list-item>
</list>
</p>
</statement>
</p>
</sec>
</sec>
</sec>
<sec id="s5">
<title>5 Simulation scenario setup and result analysis</title>
<p>The experimental simulation platform in this paper is based on the Linux operating system Ubuntu 18.04. The hardware platform is equipped with an Intel(R) Core (TM) i7-10700 CPU, 16.0&#xa0;GB RAM, and a Tesla V100s GPU. The Mininet 2.3.0 network simulation software is used to build the network topology, with Ryu 4.34 serving as the network controller. The DRL-SGA algorithm is implemented based on PyTorch 1.11.0.</p>
<sec id="s5-1">
<title>5.1 Scenario setup</title>
<p>The experimental scenario constructs an integrated tactical communication network oriented toward joint operations. The network structure is shown in <xref ref-type="fig" rid="F8">Figure 8</xref>, consisting of 47 nodes and 61 links. Among them, nodes 18, 19, and 20 and nodes 32, 33, and 34 simulate sensor nodes. Nodes 42, 43, and 44 simulate command and control nodes. Nodes 11, 12, and 13, nodes 25, 26, and 27, and nodes 39, 40, and 41 simulate fire strike nodes. The traffic transmission path follows the principle of &#x201c;Sensor &#x2014; Command and Control &#x2014; Fire Strike.&#x201d; The tactical communication network contains more than 10 types of heterogeneous links. The types of heterogeneous links are shown in <xref ref-type="table" rid="T2">Table 2</xref>. To simulate the impact of a highly adversarial environment on link status, experiments are conducted by setting different link bandwidths in the Mininet simulation software.</p>
<fig id="F8" position="float">
<label>FIGURE 8</label>
<caption>
<p>Tactical communication network.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g008.tif">
<alt-text content-type="machine-generated">Network diagram illustrating military unit connections. Central backbone network nodes are linked to various units: Superior command post, air defense, artillery, and recon units, each represented by colored circles with red symbols. Dotted lines indicate the connection paths between nodes.</alt-text>
</graphic>
</fig>
<table-wrap id="T2" position="float">
<label>TABLE 2</label>
<caption>
<p>Types of heterogeneous links.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="left">Network type</th>
<th align="left">Link type</th>
<th align="left">Link bandwidth/Mbps</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="left">Backbone network</td>
<td align="left">Microwave, scattering, zone width, satellite, UHF, wired, data link</td>
<td align="left">1&#x223c;10</td>
</tr>
<tr>
<td rowspan="2" align="left">Superior command post</td>
<td align="left">Wired, VHF, UHF, zone width</td>
<td align="left">1&#x223c;10</td>
</tr>
<tr>
<td align="left">Fiber</td>
<td align="left">100&#x223c;600</td>
</tr>
<tr>
<td align="left">Reconnaissance unit</td>
<td align="left">UHF&#x3001;VHF</td>
<td align="left">1</td>
</tr>
<tr>
<td align="left">Artillery unit</td>
<td align="left">UHF&#x3001;VHF</td>
<td align="left">1</td>
</tr>
<tr>
<td align="left">Air defense unit</td>
<td align="left">UHF&#x3001;VHF</td>
<td align="left">1</td>
</tr>
<tr>
<td align="left">Synthetic unit</td>
<td align="left">UHF&#x3001;VHF</td>
<td align="left">1</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<fn>
<p>This paper takes the integrated tactical communication network oriented toward joint operations as the research background to verify the effectiveness of the DRL-SGA, algorithm. Four different traffic intensities are set for testing, including low intensity (with transmission rates of 25&#xa0;kbps and 50&#xa0;kbps) and high intensity (with transmission rates of 75&#xa0;kbps and 100&#xa0;kbps). For each traffic intensity, Iperf scripts are written to implement one-to-one or one-to-many traffic transmission from sensor nodes to fire strike nodes.</p>
</fn>
</table-wrap-foot>
</table-wrap>
<p>The experiment sets five different traffic intensities in units of kbps: {20, 40, 60, 80, 100} for testing. For each traffic intensity, an Iperf [34] script is written to implement one-to-one or one-to-many traffic transmission from sensor nodes to fire strike nodes, and the Ryu controller is used to measure network performance parameters.</p>
</sec>
<sec id="s5-2">
<title>5.2 Parameter settings</title>
<p>The parameter settings of the DRL-SGA algorithm are shown in <xref ref-type="table" rid="T3">Table 3</xref>.</p>
<table-wrap id="T3" position="float">
<label>TABLE 3</label>
<caption>
<p>DRL-SGA parameter settings.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="left">Parameters</th>
<th align="left">Value</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="left">Optimizer</td>
<td align="left">Adam</td>
</tr>
<tr>
<td align="left">Actor learning rate <inline-formula id="inf197">
<mml:math id="m235">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3bb;</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">0.001</td>
</tr>
<tr>
<td align="left">Critic learning rate <inline-formula id="inf198">
<mml:math id="m236">
<mml:mrow>
<mml:msub>
<mml:mi>&#x3bb;</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">0.001</td>
</tr>
<tr>
<td align="left">Reward discount factor <inline-formula id="inf199">
<mml:math id="m237">
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">0.9</td>
</tr>
<tr>
<td align="left">Crop factor <inline-formula id="inf200">
<mml:math id="m238">
<mml:mrow>
<mml:mi>&#x3b5;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">0.2</td>
</tr>
<tr>
<td align="left">Experience replay pool capacity <italic>M</italic>
</td>
<td align="left">5,000</td>
</tr>
<tr>
<td align="left">Experience sample number <italic>D</italic>
</td>
<td align="left">32</td>
</tr>
<tr>
<td align="left">Total number of training rounds <italic>T</italic>
</td>
<td align="left">200</td>
</tr>
<tr>
<td align="left">Interaction frequency <inline-formula id="inf201">
<mml:math id="m239">
<mml:mrow>
<mml:msub>
<mml:mi>N</mml:mi>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">30</td>
</tr>
<tr>
<td align="left">Number of GRU units</td>
<td align="left">3</td>
</tr>
<tr>
<td align="left">Number of GAT units</td>
<td align="left">1</td>
</tr>
<tr>
<td align="left">Number of MLP hidden layers</td>
<td align="left">3</td>
</tr>
<tr>
<td align="left">Number of MLP neurons</td>
<td align="left">[64,128,64]</td>
</tr>
<tr>
<td align="left">Number of feasible paths k</td>
<td align="left">10</td>
</tr>
<tr>
<td align="left">Reward weight <inline-formula id="inf202">
<mml:math id="m240">
<mml:mrow>
<mml:mfenced open="(" close=")" separators="|">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>&#x3b2;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="left">(1,1,1)</td>
</tr>
</tbody>
</table>
</table-wrap>
</sec>
<sec id="s5-3">
<title>5.3 Network performance comparison under different traffic intensities</title>
<p>To verify the effectiveness of the DRL-SGA algorithm in improving the quality of service under different traffic intensities, this paper compares the DRL-SGA algorithm with the following three routing algorithms:<list list-type="simple">
<list-item>
<p>1. OSPF: Open shortest path first algorithm. It obtains the weight information of each link in the network through an SDN measurement mechanism and calculates the path with the shortest link weight.</p>
</list-item>
<list-item>
<p>2. DQN [35]: A deep reinforcement learning routing algorithm based on the traditional DQN. The agent performs perception and training based on link state information. The reward function is set the same as in this paper.</p>
</list-item>
<list-item>
<p>3. DDPG [36]: A deep reinforcement learning routing algorithm based on the traditional DDPG. The agent adopts a fully connected feedforward neural network structure. By interacting with the network environment, it learns the routing strategy using link state information. The reward function is set the same as in this paper.</p>
</list-item>
<list-item>
<p>4. DRL-ST: Constructs an end-to-end transmission path decision model based on Dueling DQN and optimizes the sampling mechanism using SumTree. Routing decisions are made based on link state information, with the reward function set the same as in this paper.</p>
</list-item>
</list>
</p>
<p>The experiment uses average network throughput, average end-to-end delay, and average packet loss rate as evaluation metrics. For each traffic intensity, data transmission is tested five times, and the average of the measurement results is taken for comparison. The results are as follows:</p>
<p>The comparison results in <xref ref-type="fig" rid="F9">Figure 9</xref> indicate that under low traffic intensity scenarios, the throughput improvement of DRL-SGA over other algorithms is not significant. This is because the low service traffic intensity and sufficient link bandwidth resources do not lead to noticeable network congestion, resulting in relatively small differences in throughput across algorithms. However, as traffic intensity increases, DRL-SGA demonstrates a more pronounced advantage in throughput performance compared to the other algorithms. In high traffic intensity scenarios, DRL-SGA consistently maintains a relatively high level of network throughput. When the traffic intensity reaches 100&#xa0;kbps, the throughput improves by up to 23.48% compared with the OSPF algorithm, and by up to 2.59% compared with the better-performing DRL-ST algorithm. This indicates that DRL-SGA can formulate more optimal routing strategies based on network load conditions and dynamic link state variations.</p>
<fig id="F9" position="float">
<label>FIGURE 9</label>
<caption>
<p>Comparison of average network throughput.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g009.tif">
<alt-text content-type="machine-generated">Bar chart comparing the average network throughput at different traffic transmission rates: 25, 50, 75, and 100 kbps. Five protocols are shown: OSPF, DQN, DDPG, DRL-ST, and DRL-SGA, with throughput increasing as the transmission rate increases. DRL-SGA consistently shows the highest performance.</alt-text>
</graphic>
</fig>
<p>As shown in <xref ref-type="fig" rid="F10">Figure 10</xref>, DRL-SGA achieves lower average end-to-end delay than the comparison routing algorithms under all traffic intensity levels. Moreover, as the traffic intensity increases, DRL-SGA demonstrates more outstanding performance in guaranteeing end-to-end latency. Compared with the traditional OSPF algorithm, DRL-SGA reduces the average end-to-end delay by at least 14.42%, with a maximum reduction of 33.57%. Compared with the DQN algorithm, the delay is reduced by a minimum of 5.69% and up to 25.04%. When compared with the DDPG algorithm, the reduction ranges from 7.08% to 22.44%. Even compared to the better-performing DRL-ST algorithm, DRL-SGA achieves a delay reduction of at least 2.07% and up to 16.88%.</p>
<fig id="F10" position="float">
<label>FIGURE 10</label>
<caption>
<p>Comparison of average end-to-end delay.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g010.tif">
<alt-text content-type="machine-generated">Bar chart comparing average end-to-end delay in milliseconds across five routing protocols: OSPF, DQN, DDPG, DRL-ST, and DRL-SGA. Transmission rates are 25, 50, 75, and 100 kbps. OSPF generally shows higher delays, particularly at higher transmission rates.</alt-text>
</graphic>
</fig>
<p>The comparison results in <xref ref-type="fig" rid="F11">Figure 11</xref> show that DRL-SGA consistently achieves a lower average packet loss rate than the comparison routing algorithms across different traffic intensities. Moreover, as traffic intensity increases, DRL-SGA maintains more stable packet loss performance. Compared with the traditional OSPF algorithm, the average packet loss rate of DRL-SGA is reduced by at least 14.66%. When compared with intelligent algorithms DQN, DDPG, and DRL-ST, the average packet loss rate is reduced by at least 9.03%, 8.73%, and 1.65%, respectively.</p>
<fig id="F11" position="float">
<label>FIGURE 11</label>
<caption>
<p>Comparison of average packet loss rate.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g011.tif">
<alt-text content-type="machine-generated">Bar graph comparing average packet loss rates across various traffic transmission rates in kilobits per second. OSPF, DQN, DDPG, DRL-ST, and DRL-SGA are compared. Higher transmission rates generally show increased packet loss percentages for OSPF, while DRL-SGA consistently performs better across all rates.</alt-text>
</graphic>
</fig>
<p>The above results demonstrate that the DRL-SGA algorithm exhibits superior adaptability to network environments with varying traffic intensities. This advantage stems from the fact that OSPF is a routing algorithm based on fixed forwarding rules, which makes it difficult to dynamically adjust routing strategies in response to changes in network traffic, thereby increasing the likelihood of link congestion. In contrast, DQN, DDPG, and DRL-ST utilize deep reinforcement learning mechanisms to perceive network state changes in real time and dynamically adjust optimal routing strategies. This capability allows them to alleviate network congestion to a certain extent, thereby reducing latency and packet loss while improving throughput compared to the OSPF algorithm. However, due to the limitations of fully connected feedforward neural networks used in these traditional methods, their perception capabilities are constrained, which hinders optimal path decision-making.</p>
<p>DRL-SGA, by leveraging the collaborative functionality of GRU, GAT, and attention mechanisms, effectively captures the spatiotemporal characteristics of network link states. It fully exploits hidden information within the link state data and captures evolving trends in link status, thereby enhancing the model&#x2019;s decision-making and perception capabilities and enabling the generation of more optimal routing strategies. As a result, DRL-SGA demonstrates robust adaptability to varying traffic intensity conditions, effectively reducing end-to-end delay and packet loss while improving network throughput, thereby ensuring real-time and reliable transmission of service data.</p>
</sec>
<sec id="s5-4">
<title>5.4 Ablation study</title>
<p>To evaluate the effectiveness of individual components in the link-state feature extraction module for enhancing the perceptual capability of the DRL-SGA algorithm, an ablation study was conducted under the scenario where three links were broken and traffic intensity was set to 100&#xa0;kbps. The attention mechanism, the temporal feature extraction module (GRU), and the spatial feature extraction module (GAT) were removed individually. The performance of each ablated algorithm was compared with that of the full DRL-SGA model by analyzing the convergence behavior of their reward functions. The comparative results are illustrated in <xref ref-type="fig" rid="F12">Figure 12</xref>.</p>
<fig id="F12" position="float">
<label>FIGURE 12</label>
<caption>
<p>Comparison of reward convergence.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g012.tif">
<alt-text content-type="machine-generated">Line graph showing reward value versus iteration step, comparing four models: w/o GAT (blue), w/o GRU (green), w/o attention mechanism (yellow), and DRL-SGA (red). DRL-SGA consistently achieves higher reward values across iterations, peaking near 88.</alt-text>
</graphic>
</fig>
<p>As shown in <xref ref-type="fig" rid="F12">Figure 12</xref>, the removal of any component from the link-state feature extraction module resulted in a decrease in the final reward value at convergence, indicating a reduction in the perceptual and decision-making capabilities of the agent model. This led to suboptimal routing choices. Furthermore, the removal of each module increased the number of iterations required for convergence and slowed the convergence speed. The most significant degradation was observed when the GAT module was ablated, which yielded the lowest reward and slowest convergence. This is attributed to the role of GAT in capturing spatial dependencies in link states, which enhances the algorithm&#x2019;s adaptability to dynamic network changes and is crucial for achieving effective convergence. <xref ref-type="table" rid="T4">Table 4</xref> presents a quantitative comparison of the mean and variance of reward values obtained during training under each ablated condition.</p>
<table-wrap id="T4" position="float">
<label>TABLE 4</label>
<caption>
<p>Comparison of reward quantitative metrics.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="left">Method</th>
<th align="left">Reward mean</th>
<th align="left">Reward variance</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="left">w/o GAT</td>
<td align="left">85.08</td>
<td align="left">2.95</td>
</tr>
<tr>
<td align="left">w/o GRU</td>
<td align="left">85.79</td>
<td align="left">
<bold>2.37</bold>
</td>
</tr>
<tr>
<td align="left">w/o attention mechanism</td>
<td align="left">86.33</td>
<td align="left">2.49</td>
</tr>
<tr>
<td align="left">DRL-SGA</td>
<td align="left">
<bold>87.29</bold>
</td>
<td align="left">2.80</td>
</tr>
</tbody>
</table>
</table-wrap>
<p>The results indicate that, while the variance in rewards remains comparable across configurations, the full DRL-SGA algorithm achieved mean rewards that are 2.60%, 1.75%, and 1.11% higher than those of the models without GAT, GRU, and the attention mechanism, respectively. This demonstrates that each component in the link-state feature extraction module contributes positively to enhancing the perceptual ability of the agent, enabling the algorithm to adapt to dynamic structural changes in the network and thereby facilitating the output of an optimal routing policy.</p>
</sec>
<sec id="s5-5">
<title>5.5 Computational overhead and scalability analysis</title>
<p>To further assess the feasibility of the proposed DRL-SGA algorithm in practical applications, its computational overhead and scalability were analyzed. Evaluation metrics included model size, parameter count, per-epoch training time, inference latency, peak memory usage, throughput, and control overhead. The experimental environment was identical to that described earlier, and synthetic networks of different scales were constructed to simulate various deployment scenarios: small scale (16 nodes, 24 links), medium scale (47 nodes, 61 links), and large scale (100 nodes, 220 links). The results are presented in <xref ref-type="table" rid="T5">Table 5</xref>.</p>
<table-wrap id="T5" position="float">
<label>TABLE 5</label>
<caption>
<p>Computational overhead and scalability of DRL-SGA.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="left">Network scale</th>
<th align="left">Model size (MB)</th>
<th align="left">Parameters (M)</th>
<th align="left">Training time per epoch (s)</th>
<th align="left">Inference latency (ms, GPU)</th>
<th align="left">Peak memory (MB)</th>
<th align="left">Throughput (decisions/s)</th>
<th align="left">Control overhead (msgs/s)</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="left">Small</td>
<td align="left">12.4</td>
<td align="left">3.1</td>
<td align="left">0.38</td>
<td align="left">2.6</td>
<td align="left">940</td>
<td align="left">860</td>
<td align="left">12</td>
</tr>
<tr>
<td align="left">Medium</td>
<td align="left">13.1</td>
<td align="left">3.3</td>
<td align="left">1.12</td>
<td align="left">3.9</td>
<td align="left">1,350</td>
<td align="left">620</td>
<td align="left">28</td>
</tr>
<tr>
<td align="left">Large</td>
<td align="left">15.2</td>
<td align="left">3.8</td>
<td align="left">3.45</td>
<td align="left">7.5</td>
<td align="left">2,280</td>
<td align="left">370</td>
<td align="left">65</td>
</tr>
</tbody>
</table>
</table-wrap>
<p>Several conclusions can be drawn from <xref ref-type="table" rid="T5">Table 5</xref>. First, both model size and parameter count remain largely stable across different network scales, indicating that computational complexity is primarily determined by the architecture components (GRU, GAT, and MLP) rather than node count. Second, per-epoch training time grows approximately linearly with network size, consistent with the complexity of node-based feature extraction. Third, even in the large-scale scenario with 100 nodes and 220 links, inference latency remains below 10&#xa0;ms, meeting the real-time requirements of tactical communication environments. Peak memory consumption increases with network size but remains within the capacity of a V100 GPU. Throughput decreases as the network expands, yet continues to support several hundred routing decisions per second, sufficient for operational needs. Meanwhile, control-plane overhead rises nearly linearly with network size but remains negligible compared with conventional network telemetry traffic. In summary, the DRL-SGA algorithm demonstrates moderate computational overhead and strong scalability. Its complexity scales nearly linearly with network size, making it suitable for deployment in large-scale tactical communication networks. Furthermore, additional strategies such as batch processing, localized attention, and model compression (e.g., pruning or quantization) may further reduce computational costs in resource-constrained environments. Overall, DRL-SGA achieves a favorable balance between routing performance and computational efficiency, supporting its feasibility for real-world large-scale deployment.</p>
</sec>
<sec id="s5-6">
<title>5.6 Parameter sensitivity analysis</title>
<p>As shown in <xref ref-type="table" rid="T2">Table 2</xref>, the performance of DRL-SGA is influenced by several parameters, among which <italic>&#x3b5;</italic>, <italic>&#x3b1;</italic>, <italic>&#x3b2;</italic>, and <italic>&#x3b3;</italic> are the most critical. To validate the rationality of their selection, two sets of experiments were conducted. In the first, <italic>&#x3b2;</italic> and <italic>&#x3b3;</italic> were fixed while &#x3b5; and &#x3b1; were varied. In the second, &#x3b5; and &#x3b1; were fixed while <italic>&#x3b2;</italic> and <italic>&#x3b3;</italic> were adjusted. The results are illustrated in <xref ref-type="fig" rid="F13">Figure 13</xref>.</p>
<fig id="F13" position="float">
<label>FIGURE 13</label>
<caption>
<p>Parameter sensitivity analysis results. <bold>(a)</bold> Effect of <italic>&#x26;epsi;</italic> and <italic>&#x3b1;</italic> on DRL-SGA performance. <bold>(b)</bold> Effect of <italic>&#x3b2;</italic> and <italic>&#x3b3;</italic> on DRL-SGA performance.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g013.tif">
<alt-text content-type="machine-generated">Two adjacent 3D surface plots depict reward mean values. The left plot (a) uses shades of red and blue to indicate various &#x3B1; and &#x3B5; values, with highlighted points showing coordinates X, Y, and Z. The right plot (b) shows a surface in yellow and brown tones, representing rewards over &#x3B3; and &#x3B2; values, also with annotated points for X, Y, and Z. Both plots have axes labeled for combined parameter effects on the reward mean.</alt-text>
</graphic>
</fig>
<p>In the first set, <italic>&#x3b5;</italic> was varied within {0.10, 0.15, 0.20, 0.25, 0.30}, and <italic>&#x3b1;</italic> within {0.05, 0.10, 0.15, 0.20, 0.25}. The best performance was observed when <italic>&#x3b5;</italic> &#x3d; 0.20 and <italic>&#x3b1;</italic> &#x3d; 0.10, achieving a maximum average reward of 87.29. This indicates that a balanced exploration rate and reward weighting effectively enhance routing decisions. Excessively high or low exploration rates reduced performance, suggesting that both over-exploration and premature convergence degrade effectiveness. In the second set, <italic>&#x3b2;</italic> and <italic>&#x3b3;</italic> were varied within {0.05, 0.10, 0.15, 0.20, 0.25}. Optimal performance was obtained at <italic>&#x3b2;</italic> &#x3d; 0.10 and <italic>&#x3b3;</italic> &#x3d; 0.10 (maximum reward 87.29), with only marginal performance differences across values. This demonstrates robustness in weighting for delay and packet-loss optimization. However, further increases in <italic>&#x3b2;</italic> or <italic>&#x3b3;</italic> caused slight declines, indicating that overly emphasizing a single objective weakens overall multi-objective optimization. Overall, DRL-SGA exhibits a &#x201c;middle-value optimal&#x201d; pattern across <italic>&#x3b5;</italic>, <italic>&#x3b1;</italic>, <italic>&#x3b2;</italic>, and <italic>&#x3b3;</italic>, with stable and peak performance maintained under balanced parameter settings. The chosen configuration is therefore reasonable, providing an effective trade-off among exploration, delay optimization, and packet-loss control, ensuring both effectiveness and robustness across diverse network scenarios.</p>
</sec>
<sec id="s5-7">
<title>5.7 Network performance comparison under different topologies</title>
<p>To evaluate the adaptability of the DRL-SGA algorithm to dynamically changing network topologies, a new network structure was configured as shown in <xref ref-type="fig" rid="F14">Figure 14</xref>. Several links within the backbone network were sequentially disconnected to simulate link failure scenarios. Experiments were conducted under both low-intensity traffic (50&#xa0;kbps) and high-intensity traffic (100&#xa0;kbps), and the network performance metrics of each routing algorithm were recorded and compared accordingly.</p>
<fig id="F14" position="float">
<label>FIGURE 14</label>
<caption>
<p>Network topology with partial link failures.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g014.tif">
<alt-text content-type="machine-generated">Diagram of a network showing multiple units and connections, with labeled areas such as Air Defense Unit, Artillery Unit, Recon Unit, Combined Unit, and Superior Command Post. Units are connected by lines; four red crosses indicate link damage.</alt-text>
</graphic>
</fig>
<p>
<xref ref-type="fig" rid="F15">Figure 15</xref> presents the throughput comparison results of each algorithm under varying levels of link failures and different traffic intensities. As the number of failed backbone links increases, the overall network throughput of all algorithms gradually declines. However, the DRL-SGA consistently maintains the highest throughput performance. When the number of failed backbone links reaches three, under the low traffic intensity scenario (50&#xa0;kbps), the average throughput of DRL-SGA improves by 8.76% over the traditional routing algorithm OSPF, and by 5.34%, 4.25%, and 2.77% over the intelligent routing algorithms DQN, DDPG, and DRL-ST, respectively. Under the high traffic intensity scenario (100&#xa0;kbps), DRL-SGA outperforms these four algorithms with throughput improvements of 35.29%, 25.47%, 20.18%, and 4.39%, respectively.</p>
<fig id="F15" position="float">
<label>FIGURE 15</label>
<caption>
<p>Throughput comparison under link failure scenarios. <bold>(a)</bold> Traffic intensity: 50&#xa0;kbps. <bold>(b)</bold> Traffic intensity: 100&#xa0;kbps.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g015.tif">
<alt-text content-type="machine-generated">Two line graphs compare the average network throughput in kilobits per second against the number of damaged links, with labels (a) and (b). Both graphs display five methods: OSPF (blue triangles), DQN (green triangles), DDPG (orange circles), DRL-ST (purple stars), and DRL-SGA (red squares). Throughput declines as the number of damaged links increases, with DRL-SGA consistently performing the best and OSPF the worst across both graphs.</alt-text>
</graphic>
</fig>
<p>
<xref ref-type="fig" rid="F16">Figure 16</xref> shows the comparison of average end-to-end delay across algorithms under different levels of link failure and traffic intensities. As the number of failed links increases, all algorithms experience an increase in delay. Under 50&#xa0;kbps traffic intensity, the average end-to-end delay of DRL-SGA is at least 12.99%, 8.06%, 6.41%, and 3.15% lower than that of OSPF, DQN, DDPG, and DRL-ST, respectively. Under 100&#xa0;kbps traffic intensity, DRL-SGA achieves delay reductions of at least 24.25%, 16.95%, 14.43%, and 6.44% compared to the same respective algorithms.</p>
<fig id="F16" position="float">
<label>FIGURE 16</label>
<caption>
<p>Average end-to-end delay comparison under link failure scenarios. <bold>(a)</bold> Traffic intensity: 50&#xa0;kbps. <bold>(b)</bold> Traffic intensity: 100&#xa0;kbps.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g016.tif">
<alt-text content-type="machine-generated">Two line graphs depict the average end-to-end delay in milliseconds against the number of damaged links. Graph (a) ranges from 20 to 40 ms, and graph (b) from 40 to 80 ms. Both graphs compare five routing strategies: OSPF, DQN, DDPG, DRL-ST, and DRL-SGA, with DRL-SGA consistently showing the lowest delay increases as damage increases.</alt-text>
</graphic>
</fig>
<p>
<xref ref-type="fig" rid="F17">Figure 17</xref> illustrates the comparison of average packet loss rates for all algorithms under increasing link failure conditions and varying traffic intensities. As the number of failed links grows, all algorithms exhibit rising packet loss rates. When the traffic intensity is 50&#xa0;kbps, the average packet loss rate of DRL-SGA is reduced by at least 21.92%, 15.08%, 12.52%, and 1.65% relative to OSPF, DQN, DDPG, and DRL-ST, respectively. Under 100&#xa0;kbps traffic intensity, DRL-SGA achieves minimum reductions of 35.19%, 26.44%, 21.16%, and 9.63% compared to the same algorithms.</p>
<fig id="F17" position="float">
<label>FIGURE 17</label>
<caption>
<p>Average packet loss rate comparison under link failure scenarios. <bold>(a)</bold> Traffic intensity: 50&#xa0;kbps. <bold>(b)</bold> Traffic intensity: 100&#xa0;kbps.</p>
</caption>
<graphic xlink:href="frcmn-06-1635982-g017.tif">
<alt-text content-type="machine-generated">Line graphs (a) and (b) display average packet loss rate versus the number of damaged links. Graph (a) ranges from 0.20% to 0.40%, and graph (b) ranges from 0.25% to 0.55%. Both graphs compare five methods: OSPF, DQN, DDPG, DRL-ST, and DRL-SGA, with DRL-SGA showing the lowest packet loss rate.</alt-text>
</graphic>
</fig>
<p>In summary, compared with traditional routing algorithms and existing intelligent routing algorithms, DRL-SGA demonstrates superior adaptability to dynamic network changes and incurs smaller QoS performance degradation in scenarios involving link failures, regardless of whether the traffic intensity is low or high. This is primarily because the traditional OSPF algorithm is incapable of promptly adapting to changes in network topology and adjusting routing strategies in a timely manner, making it difficult to maintain optimal network performance. Although intelligent algorithms such as DQN, DDPG, and DRL-ST leverage deep reinforcement learning, they rely on conventional feedforward neural networks trained with link state information in fixed formats, making it difficult to adapt to complex network dynamics. Consequently, these algorithms exhibit lower training efficiency and suboptimal decision accuracy.</p>
<p>In contrast, the DRL-SGA algorithm effectively extracts the spatiotemporal characteristics of tactical communication network structures. Specifically, GRU captures the temporal evolution of historical link state information, enabling accurate prediction of future trends and enhancing the model&#x2019;s perception and reasoning capabilities. GAT enhances the algorithm&#x2019;s ability to represent non-Euclidean graph-structured data, thereby enabling a more comprehensive understanding and processing of complex network environments and significantly improving the model&#x2019;s structural awareness. As a result, DRL-SGA exhibits stronger adaptability to dynamic network topologies and superior capability in ensuring service quality.</p>
</sec>
</sec>
<sec sec-type="conclusion" id="s6">
<title>6 Conclusion</title>
<p>This study proposes a DRL-SGA for tactical communication networks, which enhances route selection through improved link state awareness. The algorithm constructs an intelligent path decision model using PPO and integrates a link state feature extraction module designed to capture the spatiotemporal dependencies within sequences composed of multiple link states. An optimal path forwarding strategy is then derived through a deep reinforcement learning training mechanism. Experimental results demonstrate that DRL-SGA exhibits superior adaptability to dynamic changes in network topology compared to traditional and existing intelligent routing algorithms.</p>
<p>It should be noted that all evaluations in this study are conducted in simulation. While the simulated scenarios are designed to resemble realistic tactical networks, the lack of real-world dataset or testbed validation remains a limitation. Future work will focus on validating the proposed approach in real operational environments to further assess its practical applicability and robustness. Moreover, tactical communication networks typically exhibit rapidly changing topologies and highly uncertain traffic patterns. Accurate and real-time traffic prediction is critical for ensuring timely delivery of high-priority services and effective traffic regulation, which in turn improves network resource utilization. In future work, we plan to integrate traffic prediction mechanisms with intelligent routing algorithms. By forecasting the future evolution of network traffic states, we aim to enhance the perception and decision-making capabilities of intelligent routing models, enabling more efficient and adaptive routing strategies in dynamic tactical communication scenarios.</p>
</sec>
</body>
<back>
<sec sec-type="data-availability" id="s7">
<title>Data availability statement</title>
<p>The datasets presented in this study can be found in online repositories. The names of the repository/repositories and accession number(s) can be found in the article/supplementary material.</p>
</sec>
<sec sec-type="author-contributions" id="s8">
<title>Author contributions</title>
<p>YS: Writing &#x2013; original draft, Conceptualization. LX: Writing &#x2013; review and editing, Investigation. ML: Writing &#x2013; review and editing, Conceptualization.</p>
</sec>
<sec sec-type="funding-information" id="s9">
<title>Funding</title>
<p>The author(s) declare that no financial support was received for the research and/or publication of this article.</p>
</sec>
<sec sec-type="COI-statement" id="s10">
<title>Conflict of interest</title>
<p>Authors YS, LX, and ML were employed by China Telecom Corporation Limited Zhejiang Branch.</p>
</sec>
<sec sec-type="ai-statement" id="s11">
<title>Generative AI statement</title>
<p>The author(s) declare that no Generative AI was used in the creation of this manuscript.</p>
<p>Any alternative text (alt text) provided alongside figures in this article has been generated by Frontiers with the support of artificial intelligence and reasonable efforts have been made to ensure accuracy, including review by the authors wherever possible. If you identify any issues, please contact us.</p>
</sec>
<sec sec-type="disclaimer" id="s12">
<title>Publisher&#x2019;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<ref-list>
<title>References</title>
<ref id="B1">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Alnajjar</surname>
<given-names>A. B.</given-names>
</name>
<name>
<surname>Kadim</surname>
<given-names>A. M.</given-names>
</name>
<name>
<surname>Jaber</surname>
<given-names>R. A.</given-names>
</name>
<name>
<surname>Hasan</surname>
<given-names>N. A.</given-names>
</name>
<name>
<surname>Ahmad</surname>
<given-names>E. Q.</given-names>
</name>
<name>
<surname>Altaei</surname>
<given-names>M. S.</given-names>
</name>
<etal/>
</person-group> (<year>2022</year>). <article-title>Wireless sensor network optimization using genetic algorithm</article-title>. <source>J. Robotics Control (JRC)</source> <volume>3</volume> (<issue>6</issue>), <fpage>827</fpage>&#x2013;<lpage>835</lpage>. <pub-id pub-id-type="doi">10.18196/jrc.v3i6.16526</pub-id>
</citation>
</ref>
<ref id="B2">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Bajracharya</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Shrestha</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Hassan</surname>
<given-names>S. A.</given-names>
</name>
<name>
<surname>Jung</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Shin</surname>
<given-names>H.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>5G and beyond private military communication: trend, requirements, challenges and enablers</article-title>. <source>IEEE Access</source> <volume>11</volume>, <fpage>83996</fpage>&#x2013;<lpage>84012</lpage>. <pub-id pub-id-type="doi">10.1109/access.2023.3303211</pub-id>
</citation>
</ref>
<ref id="B3">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Bale</surname>
<given-names>A. S.</given-names>
</name>
<name>
<surname>Hamsalekha</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Dash</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Baral</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Bajaj</surname>
<given-names>B.</given-names>
</name>
<name>
<surname>Arora</surname>
<given-names>S.</given-names>
</name>
<etal/>
</person-group> (<year>2023</year>). &#x201c;<article-title>Recent scientific achievements and developments in software defined networking: a survey</article-title>,&#x201d; in <conf-name>2023 1st International Conference on Circuits, Power and Intelligent Systems (CCPIS)</conf-name> (<publisher-name>IEEE</publisher-name>), <fpage>1</fpage>&#x2013;<lpage>6</lpage>.</citation>
</ref>
<ref id="B4">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Casas-Velasco</surname>
<given-names>D. M.</given-names>
</name>
<name>
<surname>Rendon</surname>
<given-names>O. M. C.</given-names>
</name>
<name>
<surname>da Fonseca</surname>
<given-names>N. L. S.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>DRSIR: a deep reinforcement learning approach for routing in software-defined networking</article-title>. <source>IEEE Trans. Netw. Serv. Manag.</source> <volume>19</volume> (<issue>4</issue>), <fpage>4807</fpage>&#x2013;<lpage>4820</lpage>. <pub-id pub-id-type="doi">10.1109/tnsm.2021.3132491</pub-id>
</citation>
</ref>
<ref id="B5">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Chang</surname>
<given-names>Y. C.</given-names>
</name>
<name>
<surname>Lin</surname>
<given-names>H. T.</given-names>
</name>
<name>
<surname>Chu</surname>
<given-names>H. M.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>P. C.</given-names>
</name>
</person-group> (<year>2020</year>). <article-title>Efficient topology discovery for software-defined networks</article-title>. <source>IEEE Trans. Netw. Serv. Manag.</source> <volume>18</volume> (<issue>2</issue>), <fpage>1375</fpage>&#x2013;<lpage>1388</lpage>. <pub-id pub-id-type="doi">10.1109/tnsm.2020.3047623</pub-id>
</citation>
</ref>
<ref id="B6">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Dake</surname>
<given-names>D. K.</given-names>
</name>
<name>
<surname>Gadze</surname>
<given-names>J. D.</given-names>
</name>
<name>
<surname>Klogo</surname>
<given-names>G. S.</given-names>
</name>
<name>
<surname>Nunoo-Mensah</surname>
<given-names>H.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Traffic engineering in software-defined networks using reinforcement learning: a review</article-title>. <source>Int. J. Adv. Comput. Sci. Appl.</source> <volume>12</volume> (<issue>5</issue>), <fpage>330</fpage>&#x2013;<lpage>345</lpage>. <pub-id pub-id-type="doi">10.14569/ijacsa.2021.0120541</pub-id>
</citation>
</ref>
<ref id="B7">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Deng</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Zeng</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Chang</surname>
<given-names>L.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Wu</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Liang</surname>
<given-names>J.</given-names>
</name>
<etal/>
</person-group> (<year>2022</year>). <article-title>An ant colony optimization-based routing algorithm for load balancing in leo satellite networks</article-title>. <source>Wirel. Commun. Mob. Comput.</source> <volume>2022</volume>, <fpage>3032997</fpage>. <pub-id pub-id-type="doi">10.1155/2022/3032997</pub-id>
</citation>
</ref>
<ref id="B8">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Shokouhifar</surname>
<given-names>M.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>FH-ACO: fuzzy heuristic-based ant colony optimization for joint virtual network function placement and routing</article-title>. <source>Appl. Soft Comput.</source> <volume>107</volume>, <fpage>107401</fpage>. <pub-id pub-id-type="doi">10.1016/j.asoc.2021.107401</pub-id>
</citation>
</ref>
<ref id="B9">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Fu</surname>
<given-names>Q.</given-names>
</name>
<name>
<surname>Sun</surname>
<given-names>E.</given-names>
</name>
<name>
<surname>Meng</surname>
<given-names>K.</given-names>
</name>
<name>
<surname>Li</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>Y.</given-names>
</name>
</person-group> (<year>2020</year>). <article-title>Deep Q-learning for routing schemes in SDN-based data center networks</article-title>. <source>IEEE Access</source> <volume>8</volume>, <fpage>103491</fpage>&#x2013;<lpage>103499</lpage>. <pub-id pub-id-type="doi">10.1109/access.2020.2995511</pub-id>
</citation>
</ref>
<ref id="B10">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>He</surname>
<given-names>Q.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Xu</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Li</surname>
<given-names>F.</given-names>
</name>
<name>
<surname>Yang</surname>
<given-names>K.</given-names>
</name>
<etal/>
</person-group> (<year>2023</year>). <article-title>Routing optimization with deep reinforcement learning in knowledge defined networking</article-title>. <source>IEEE Trans. Mob. Comput.</source> <volume>23</volume> (<issue>2</issue>), <fpage>1444</fpage>&#x2013;<lpage>1455</lpage>. <pub-id pub-id-type="doi">10.1109/tmc.2023.3235446</pub-id>
</citation>
</ref>
<ref id="B11">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Huang</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Guan</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Zhai</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>He</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Chu</surname>
<given-names>X.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Deep graph reinforcement learning based intelligent traffic routing control for software-defined wireless sensor networks</article-title>. <source>Appl. Sci.</source> <volume>12</volume> (<issue>4</issue>), <fpage>1951</fpage>. <pub-id pub-id-type="doi">10.3390/app12041951</pub-id>
</citation>
</ref>
<ref id="B12">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Hussain</surname>
<given-names>M. W.</given-names>
</name>
<name>
<surname>Sinha Roy</surname>
<given-names>D.</given-names>
</name>
</person-group> (<year>2021</year>). &#x201c;<article-title>Intelligent node placement for improving traffic engineering in hybrid SDN</article-title>,&#x201d; in <conf-name>Advances in Communication, Devices and Networking: Proceedings of ICCDN 2020</conf-name> (<publisher-loc>Singapore</publisher-loc>: <publisher-name>Springer Singapore</publisher-name>), <fpage>287</fpage>&#x2013;<lpage>296</lpage>.</citation>
</ref>
<ref id="B13">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Hussain</surname>
<given-names>O. F.</given-names>
</name>
<name>
<surname>Al-Kaseem</surname>
<given-names>B. R.</given-names>
</name>
<name>
<surname>Akif</surname>
<given-names>O. Z.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Smart flow steering agent for end-to-end delay improvement in software-defined networks</article-title>. <source>Baghdad Sci. J.</source> <volume>18</volume> (<issue>1</issue>), <fpage>0163</fpage>. <pub-id pub-id-type="doi">10.21123/bsj.2021.18.1.0163</pub-id>
</citation>
</ref>
<ref id="B14">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ju</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Song</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Li</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>He</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Dong</surname>
<given-names>F.</given-names>
</name>
<etal/>
</person-group> (<year>2025</year>). <article-title>Dynamic load-balancing routing strategy for LEO satellite networks based on spatio-temporal traffic prediction</article-title>. <source>IEEE Trans. Aerosp. Electron. Syst.</source>, <fpage>1</fpage>&#x2013;<lpage>16</lpage>. <pub-id pub-id-type="doi">10.1109/taes.2025.3571400</pub-id>
</citation>
</ref>
<ref id="B15">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Li</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>He</surname>
<given-names>H.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Multiagent trust region policy optimization</article-title>. <source>IEEE Trans. Neural Netw. Learn. Syst.</source> <volume>35</volume> (<issue>9</issue>), <fpage>12873</fpage>&#x2013;<lpage>12887</lpage>. <pub-id pub-id-type="doi">10.1109/tnnls.2023.3265358</pub-id>
<pub-id pub-id-type="pmid">37053062</pub-id>
</citation>
</ref>
<ref id="B16">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Li</surname>
<given-names>Q.</given-names>
</name>
<name>
<surname>Wu</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2025</year>). <article-title>Efficient network attack path optimization method based on prior knowledge-based PPO algorithm</article-title>. <source>Cybersecurity</source> <volume>8</volume> (<issue>1</issue>), <fpage>15</fpage>&#x2013;<lpage>14</lpage>. <pub-id pub-id-type="doi">10.1186/s42400-024-00288-8</pub-id>
</citation>
</ref>
<ref id="B17">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Li</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Jiang</surname>
<given-names>K.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>K.</given-names>
</name>
<name>
<surname>Shen</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Du</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>J.</given-names>
</name>
<etal/>
</person-group> (<year>2021</year>). &#x201c;<article-title>DRNet: QoS-aware routing for SDN using deep reinforcement learning</article-title>,&#x201d; in <conf-name>2021 IEEE 21st International Conference on Communication Technology (ICCT)</conf-name> (<publisher-name>IEEE</publisher-name>), <fpage>25</fpage>&#x2013;<lpage>30</lpage>.</citation>
</ref>
<ref id="B18">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Liu</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Cai</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Chen</surname>
<given-names>Q. C.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>Y.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>DRL-R: deep reinforcement learning approach for intelligent routing in software-defined data-center networks</article-title>. <source>J. Netw. Comput. Appl.</source> <volume>177</volume>, <fpage>102865</fpage>. <pub-id pub-id-type="doi">10.1016/j.jnca.2020.102865</pub-id>
</citation>
</ref>
<ref id="B19">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Liu</surname>
<given-names>L.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Yang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Liu</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Li</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>P.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Path planning techniques for mobile robots: review and prospect</article-title>. <source>Expert Syst. Appl.</source> <volume>227</volume>, <fpage>120254</fpage>. <pub-id pub-id-type="doi">10.1016/j.eswa.2023.120254</pub-id>
</citation>
</ref>
<ref id="B20">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Liu</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Xiao</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Albrethsen</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Goh</surname>
<given-names>L.</given-names>
</name>
<etal/>
</person-group> (<year>2024</year>). &#x201c;<article-title>Spatial-temporal graph representation learning for tactical networks future state prediction</article-title>,&#x201d; in <conf-name>2024 International Joint Conference on Neural Networks (IJCNN)</conf-name> (<publisher-name>IEEE</publisher-name>), <fpage>1</fpage>&#x2013;<lpage>8</lpage>.</citation>
</ref>
<ref id="B21">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Lu</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Zhao</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Liao</surname>
<given-names>L. X.</given-names>
</name>
</person-group> (<year>2023</year>). &#x201c;<article-title>C-LLDP-monitoring: latency monitoring across large-scale software defined networks</article-title>,&#x201d; in <conf-name>2023 5th International Conference on Frontiers Technology of Information and Computer (ICFTIC)</conf-name> (<publisher-name>IEEE</publisher-name>), <fpage>792</fpage>&#x2013;<lpage>799</lpage>.</citation>
</ref>
<ref id="B22">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Patel</surname>
<given-names>Z.</given-names>
</name>
<name>
<surname>Khanpara</surname>
<given-names>P.</given-names>
</name>
<name>
<surname>Valiveti</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Raval</surname>
<given-names>G.</given-names>
</name>
</person-group> (<year>2023</year>). &#x201c;<article-title>The evolution of Ad hoc networks for tactical military communications: trends, technologies, and case studies</article-title>,&#x201d; in <conf-name>Proceedings of Third International Conference on Sustainable Expert Systems: ICSES 2022</conf-name>, <conf-loc>Singapore</conf-loc> (<publisher-name>Springer Nature Singapore</publisher-name>), <fpage>331</fpage>&#x2013;<lpage>346</lpage>.</citation>
</ref>
<ref id="B23">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Pourranjbar</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Elleuch</surname>
<given-names>I.</given-names>
</name>
<name>
<surname>Landry-pellerin</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Kaddoum</surname>
<given-names>G.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Defense and offence strategies for tactical wireless networks using recurrent neural networks</article-title>. <source>IEEE Trans. Veh. Technol.</source> <volume>72</volume> (<issue>6</issue>), <fpage>8278</fpage>&#x2013;<lpage>8283</lpage>. <pub-id pub-id-type="doi">10.1109/tvt.2023.3243127</pub-id>
</citation>
</ref>
<ref id="B24">
<citation citation-type="book">
<collab>Ryu SDN Framework Community</collab> (<year>2020</year>). <source>Component-based software defined networking frame work</source>. <comment>Available online at: <ext-link ext-link-type="uri" xlink:href="http://osrg.github.io/ryu">http://osrg.github.io/ryu</ext-link>.</comment>
</citation>
</ref>
<ref id="B25">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Suomalainen</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Julku</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Vehkaper&#xe4;</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Posti</surname>
<given-names>H.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Securing public safety communications on commercial and tactical 5G networks: a survey and future research directions</article-title>. <source>IEEE Open J. Commun. Soc.</source> <volume>2</volume>, <fpage>1590</fpage>&#x2013;<lpage>1615</lpage>. <pub-id pub-id-type="doi">10.1109/ojcoms.2021.3093529</pub-id>
</citation>
</ref>
<ref id="B26">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Tabatabaei</surname>
<given-names>S.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Provide energy-aware routing protocol in wireless sensor networks using bacterial foraging optimization algorithm and Mobile sink</article-title>. <source>Plos one</source> <volume>17</volume> (<issue>3</issue>), <fpage>e0265113</fpage>. <pub-id pub-id-type="doi">10.1371/journal.pone.0265113</pub-id>
<pub-id pub-id-type="pmid">35320290</pub-id>
</citation>
</ref>
<ref id="B27">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Wang</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Shang</surname>
<given-names>F.</given-names>
</name>
<name>
<surname>Lei</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Energy-efficient and delay-guaranteed routing algorithm for software-defined wireless sensor networks: a cooperative deep reinforcement learning approach</article-title>. <source>J. Netw. Comput. Appl.</source> <volume>217</volume>, <fpage>103674</fpage>. <pub-id pub-id-type="doi">10.1016/j.jnca.2023.103674</pub-id>
</citation>
</ref>
<ref id="B28">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Yang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Yan</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>D.</given-names>
</name>
<name>
<surname>Xu</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Hua</surname>
<given-names>G.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>WOAD3QN-RP: an intelligent routing protocol in wireless sensor networks&#x2014;A swarm intelligence and deep reinforcement learning based approach</article-title>. <source>Expert Syst. Appl.</source> <volume>246</volume>, <fpage>123089</fpage>. <pub-id pub-id-type="doi">10.1016/j.eswa.2023.123089</pub-id>
</citation>
</ref>
<ref id="B29">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Zhou</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Jiang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Guo</surname>
<given-names>B.</given-names>
</name>
<name>
<surname>Meng</surname>
<given-names>L.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>PQROM: to optimize software defined network QoS aware routing with proximal policy optimization</article-title>. <source>J. Intelligent and Fuzzy Syst.</source> <volume>42</volume> (<issue>4</issue>), <fpage>3605</fpage>&#x2013;<lpage>3614</lpage>. <pub-id pub-id-type="doi">10.3233/jifs-211787</pub-id>
</citation>
</ref>
<ref id="B30">
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Zhou</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Yan</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Yan</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Mao</surname>
<given-names>K.</given-names>
</name>
<name>
<surname>Yang</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Liu</surname>
<given-names>W.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Path planning of rail-mounted logistics robots based on the improved dijkstra algorithm</article-title>. <source>Appl. Sci.</source> <volume>13</volume> (<issue>17</issue>), <fpage>9955</fpage>. <pub-id pub-id-type="doi">10.3390/app13179955</pub-id>
</citation>
</ref>
<ref id="B31">
<citation citation-type="confproc">
<person-group person-group-type="author">
<name>
<surname>Zhu</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Jin</surname>
<given-names>Z.</given-names>
</name>
</person-group> (<year>2023</year>). &#x201c;<article-title>Routing algorithm for AD hoc networks of LEO satellites based on OSPF</article-title>,&#x201d; in <conf-name>2023 3rd International Symposium on Computer Technology and Information Science (ISCTIS)</conf-name> (<publisher-name>IEEE</publisher-name>), <fpage>404</fpage>&#x2013;<lpage>408</lpage>.</citation>
</ref>
</ref-list>
</back>
</article>