<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.3 20210610//EN" "JATS-journalpublishing1-3-mathml3.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ali="http://www.niso.org/schemas/ali/1.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" article-type="research-article" dtd-version="1.3" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Commun. Netw.</journal-id>
<journal-title-group>
<journal-title>Frontiers in Communications and Networks</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Commun. Netw.</abbrev-journal-title>
</journal-title-group>
<issn pub-type="epub">2673-530X</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="publisher-id">1764320</article-id>
<article-id pub-id-type="doi">10.3389/frcmn.2025.1764320</article-id>
<article-version article-version-type="Version of Record" vocab="NISO-RP-8-2008"/>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Original Research</subject>
</subj-group>
</article-categories>
<title-group>
<article-title>DDPG-based energy efficiency optimization for ABS-assisted beyond-5G cellular networks with sleep mode management</article-title>
<alt-title alt-title-type="left-running-head">Saleh et al.</alt-title>
<alt-title alt-title-type="right-running-head">
<ext-link ext-link-type="uri" xlink:href="https://doi.org/10.3389/frcmn.2025.1764320">10.3389/frcmn.2025.1764320</ext-link>
</alt-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name>
<surname>Saleh</surname>
<given-names>Vala</given-names>
</name>
<xref ref-type="aff" rid="aff1">
<sup>1</sup>
</xref>
<uri xlink:href="https://loop.frontiersin.org/people/3306283"/>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; review &#x26; editing" vocab-term-identifier="https://credit.niso.org/contributor-roles/Writing - review &#x26; editing/">Writing - review and editing</role>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; original draft" vocab-term-identifier="https://credit.niso.org/contributor-roles/writing-original-draft/">Writing - original draft</role>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name>
<surname>Eslami</surname>
<given-names>Mohsen</given-names>
</name>
<xref ref-type="aff" rid="aff1">
<sup>1</sup>
</xref>
<xref ref-type="aff" rid="aff2">
<sup>2</sup>
</xref>
<xref ref-type="corresp" rid="c001">&#x2a;</xref>
<uri xlink:href="https://loop.frontiersin.org/people/3305799"/>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; original draft" vocab-term-identifier="https://credit.niso.org/contributor-roles/writing-original-draft/">Writing - original draft</role>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; review &#x26; editing" vocab-term-identifier="https://credit.niso.org/contributor-roles/Writing - review &#x26; editing/">Writing - review and editing</role>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Kazemi</surname>
<given-names>Kamran</given-names>
</name>
<xref ref-type="aff" rid="aff1">
<sup>1</sup>
</xref>
<uri xlink:href="https://loop.frontiersin.org/people/3290829"/>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; review &#x26; editing" vocab-term-identifier="https://credit.niso.org/contributor-roles/Writing - review &#x26; editing/">Writing - review and editing</role>
<role vocab="credit" vocab-identifier="https://credit.niso.org/" vocab-term="Writing &#x2013; original draft" vocab-term-identifier="https://credit.niso.org/contributor-roles/writing-original-draft/">Writing - original draft</role>
</contrib>
</contrib-group>
<aff id="aff1">
<label>1</label>
<institution>Department of Electrical Engineering, Shiraz University of Technology</institution>, <city>Shiraz</city>, <country country="IR">Iran</country>
</aff>
<aff id="aff2">
<label>2</label>
<institution>Electrical and Computer Engineering Department, University of Alberta</institution>, <city>Edmonton</city>, <state>AB</state>, <country country="CA">Canada</country>
</aff>
<author-notes>
<corresp id="c001">
<label>&#x2a;</label>Correspondence: Mohsen Eslami, <email xlink:href="mailto:meslami1@ualberta.ca">meslami1@ualberta.ca</email>
</corresp>
</author-notes>
<pub-date publication-format="electronic" date-type="pub" iso-8601-date="2026-01-26">
<day>26</day>
<month>01</month>
<year>2026</year>
</pub-date>
<pub-date publication-format="electronic" date-type="collection">
<year>2025</year>
</pub-date>
<volume>6</volume>
<elocation-id>1764320</elocation-id>
<history>
<date date-type="received">
<day>09</day>
<month>12</month>
<year>2025</year>
</date>
<date date-type="rev-recd">
<day>20</day>
<month>12</month>
<year>2025</year>
</date>
<date date-type="accepted">
<day>29</day>
<month>12</month>
<year>2025</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#xa9; 2026 Saleh, Eslami and Kazemi.</copyright-statement>
<copyright-year>2026</copyright-year>
<copyright-holder>Saleh, Eslami and Kazemi</copyright-holder>
<license>
<ali:license_ref start_date="2026-01-26">https://creativecommons.org/licenses/by/4.0/</ali:license_ref>
<license-p>This is an open-access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution License (CC BY)</ext-link>. The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</license-p>
</license>
</permissions>
<abstract>
<sec>
<title>Introduction</title>
<p>Energy efficiency is a critical challenge in Beyond-5G (B5G) cellular networks, where ground base stations (GBSs) are responsible for a substantial portion of network energy consumption. Reducing this consumption while maintaining minimum user data rate requirements remains a key research problem.</p>
</sec>
<sec>
<title>Methods</title>
<p>This paper proposes an Aerial Base Station (ABS)-assisted energy optimization framework that integrates ABS deployment with low-power sleep states of GBSs. Traffic is selectively offloaded from lightly loaded GBSs to ABSs, enabling energy savings without violating user quality-of-service constraints. A Deep Deterministic Policy Gradient (DDPG) algorithm is employed to jointly optimize ABS positioning, GBS sleep mode scheduling, and resource allocation under dynamic traffic conditions.</p>
</sec>
<sec>
<title>Results</title>
<p>Simulation results demonstrate that the proposed DDPG-based framework significantly reduces network energy consumption while improving achievable user data rates compared to baseline schemes without ABS assistance or learning-based optimization.</p>
</sec>
<sec>
<title>Discussion</title>
<p>The results highlight the effectiveness of integrating ABSs with GBS low-power sleep states using reinforcement learning. By enforcing minimum data rate constraints and dynamically adapting to traffic variations, the proposed approach provides a scalable and energy-efficient solution for sustainable operation.</p>
</sec>
</abstract>
<kwd-group>
<kwd>deep-double Q-learning (DDQL)</kwd>
<kwd>deterministic-policy gradient (DDPG)</kwd>
<kwd>energy efficiency</kwd>
<kwd>sleeping ground BS</kwd>
<kwd>ABS-assisted beyond-5G network</kwd>
</kwd-group>
<funding-group>
<funding-statement>The author(s) declared that financial support was not received for this work and/or its publication.</funding-statement>
</funding-group>
<counts>
<fig-count count="5"/>
<table-count count="1"/>
<equation-count count="46"/>
<ref-count count="42"/>
<page-count count="11"/>
</counts>
<custom-meta-group>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>IoT and Sensor Networks</meta-value>
</custom-meta>
</custom-meta-group>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="s1">
<label>1</label>
<title>Introduction</title>
<p>The evolution of <italic>Beyond 5G (B5G)</italic> technology builds upon the worldwide deployment of <italic>fifth-generation (5G)</italic> networks, aiming to meet growing connectivity demands through more advanced standards and innovative communication solutions. B5G is expected to deliver enhanced mobility, superior reliability, ultra-high data rates, intelligent network management, and improved energy efficiency. With increasing dependency on high-speed and ubiquitous communication, B5G networks are anticipated to progress toward greater resilience and maturity. To enable this transition, key technologies such as <italic>Artificial Intelligence (AI)</italic>, <italic>Edge Computing</italic>, <italic>Reconfigurable Intelligent Surfaces (RIS)</italic>, <italic>Terahertz (THz)</italic> communication, <italic>Quantum Computing</italic>, and <italic>Unmanned Aerial Vehicles (UAVs)</italic> are being actively explored (<xref ref-type="bibr" rid="B35">Sufyan et al., 2023</xref>; <xref ref-type="bibr" rid="B29">Puspitasari et al., 2023</xref>; <xref ref-type="bibr" rid="B12">Dogra et al., 2020</xref>; <xref ref-type="bibr" rid="B2">Alsharif et al., 2018</xref>).</p>
<p>The potential of UAVs in cellular and wireless networks has gained substantial attention in recent literature. <italic>UAV-assisted communication systems</italic> have shown promise in extending network coverage beyond the limitations of terrestrial infrastructure, enhancing link reliability, and offering flexible, resilient, and sustainable connectivity in diverse deployment scenarios (<xref ref-type="bibr" rid="B17">Gryech et al., 2024</xref>). A prominent outcome of this integration is the development of <italic>Aerial Access Networks (AANs)</italic>, where UAVs are utilized to provide communication services from the air. Unlike <italic>Terrestrial Access Networks (TANs)</italic>, AANs overcome geographic and infrastructural constraints, offering wide-area coverage, improved communication quality, and high mobility support&#x2014;particularly in remote or hard-to-reach environments where TANs may be infeasible (<xref ref-type="bibr" rid="B9">Behjati et al., 2025</xref>). In such systems, UAVs often operate as <italic>Aerial Base Stations (ABSs)</italic> or airborne relays, enabling direct access to users and thereby improving service availability and network adaptability.</p>
<p>Significant research efforts have been directed toward the integration of UAVs into B5G networks, addressing challenges such as trajectory optimization, user association mechanisms, transmission power management, and the cooperative deployment of UAVs with Intelligent Reflecting Surfaces (IRS) for improved signal propagation (<xref ref-type="bibr" rid="B7">Banafaa et al., 2024</xref>; <xref ref-type="bibr" rid="B30">Qazzaz et al., 2024</xref>; <xref ref-type="bibr" rid="B32">Shahzadi et al., 2021</xref>; <xref ref-type="bibr" rid="B15">Geraci et al., 2022</xref>; <xref ref-type="bibr" rid="B18">Gu and Zhang, 2023</xref>; <xref ref-type="bibr" rid="B31">Sarkar and Gul, 2023</xref>; <xref ref-type="bibr" rid="B20">Jangsher et al., 2022</xref>). These advancements improve performance, reliability, and energy efficiency in future wireless communication systems (<xref ref-type="bibr" rid="B4">Amponis et al., 2022</xref>).</p>
<p>A diverse set of techniques has been proposed to effectively reduce energy consumption in drone networks (<xref ref-type="bibr" rid="B1">Abubakar et al., 2023</xref>). These approaches encompass resource management (<xref ref-type="bibr" rid="B27">Masroor et al., 2021</xref>; <xref ref-type="bibr" rid="B8">Basharat et al., 2022</xref>), flight and transmission scheduling (<xref ref-type="bibr" rid="B39">Wu et al., 2022</xref>), path planning (<xref ref-type="bibr" rid="B5">Azadur et al., 2024</xref>), and optimal placement and trajectory design (<xref ref-type="bibr" rid="B13">Elnabty et al., 2022</xref>; <xref ref-type="bibr" rid="B38">Won et al., 2023</xref>; <xref ref-type="bibr" rid="B6">Azarhava et al., 2024</xref>; <xref ref-type="bibr" rid="B36">Tung et al., 2022</xref>).</p>
<p>Beyond UAV-assisted solutions, a key energy challenge in B5G networks is the high power consumption of <italic>Ground Base Stations (GBSs)</italic>. To mitigate this issue, researchers are actively developing optimization strategies to improve GBS energy efficiency. One widely adopted approach, applicable to both terrestrial networks and ABS-assisted frameworks, is the <italic>GBS sleep strategy</italic>, which dynamically deactivates underutilized GBSs with a small number of associated users while ensuring that user data rate requirements are satisfied.</p>
<p>GBS sleep strategies are designed to identify optimal opportunities for base stations to enter sleep mode without compromising network coverage or service quality (<xref ref-type="bibr" rid="B26">L&#xf3;pez-P&#xe9;rez et al., 2022</xref>). These strategies can be broadly classified into <italic>binary on/off schemes</italic> and <italic>multi-level sleep modes</italic>, each offering distinct mechanisms for reducing energy consumption while maintaining network performance.</p>
<p>The binary scheme conserves energy by deactivating underutilized GBSs; however, this approach may negatively impact data transmission rates (<xref ref-type="bibr" rid="B23">Kim et al., 2015</xref>; <xref ref-type="bibr" rid="B25">Kooshki et al., 2023</xref>). To optimize energy efficiency in ultra-dense networks, researchers in (<xref ref-type="bibr" rid="B3">Amine et al., 2022</xref>) and (<xref ref-type="bibr" rid="B21">Ju et al., 2022</xref>) proposed <italic>reinforcement learning (RL)</italic>-based cell switching algorithms for managing small cells. Specifically, the work in (<xref ref-type="bibr" rid="B21">Ju et al., 2022</xref>) introduces a <italic>Decision Selection Network (DSN)</italic> to streamline the action space within a <italic>Deep Reinforcement Learning (DRL)</italic> framework, demonstrating effective management of active and sleep modes while maintaining essential data rate requirements.</p>
<p>In contrast, multi-level sleep modes leverage mobile traffic prediction to dynamically transition idle small cells into different sleep states, further optimizing energy efficiency while ensuring network performance (<xref ref-type="bibr" rid="B24">Kim et al., 2023</xref>).</p>
<p>An often-overlooked application of drones is their use as <italic>Aerial Base Stations (ABSs)</italic> to facilitate GBS sleep strategies. In <xref ref-type="bibr" rid="B11">Chowdary et al. (2021)</xref>, the authors propose a resource allocation algorithm that leverages active GBSs to serve users in areas where some GBSs have entered sleep mode. While this approach ensures user connectivity in sleeping areas, it introduces challenges such as service instability for users in non-sleeping areas and increased algorithmic complexity due to resource reallocation after GBS deactivation. Moreover, this work does not explore the potential of drones operating explicitly as ABSs to enable GBS sleep modes.</p>
<p>Motivated by these limitations, we propose an <italic>ABS-assisted GBS sleep strategy</italic> that selectively allows lightly loaded GBSs to enter sleep mode during periods of reduced traffic demand. This approach minimizes overall network energy consumption while ensuring that each ABS satisfies the minimum data rate required to maintain <italic>Quality of Service (QoS)</italic> during GBS downtime. Our analysis demonstrates that effective ABS deployment not only enhances overall network transmission rates but also significantly reduces GBS power consumption.</p>
<p>To further amplify the energy-saving benefits, we introduce a <italic>joint optimization framework</italic> that integrates GBS sleep scheduling, resource allocation, and ABS position optimization to minimize network-wide energy consumption. The resulting decision-making process forms a complex <italic>binary integer programming</italic> problem, motivating the need for efficient learning-based optimization techniques.</p>
<p>To address this challenge, we propose a <italic>hybrid Deep Reinforcement Learning (DRL)</italic> framework that combines <italic>Deep Deterministic Policy Gradient (DDPG)</italic> and <italic>Deep Double Q-Learning (DDQL)</italic> algorithms. The rationale for this integration is that the considered optimization includes both <italic>continuous</italic> variables (e.g., ABS horizontal positioning and transmit power allocation) and <italic>discrete</italic> decisions (e.g., GBS sleep mode control and association selection). DDPG is well-suited to continuous control, whereas DDQL is effective in discrete action spaces and reduces Q-value overestimation. By combining them, the proposed framework can efficiently handle the mixed discrete&#x2013;continuous action space in a unified learning process.</p>
<p>Unless otherwise stated, users are assumed <italic>quasi-static</italic> during each optimization interval, i.e., user locations remain fixed while decisions on association, sleep scheduling, and resource allocation are optimized.</p>
<p>The remainder of this paper is organized as follows: <xref ref-type="sec" rid="s2">Section 2</xref> introduces the system model, including the sleep model, terrestrial and aerial channel models, and the power consumption model, followed by the optimization problem formulation. <xref ref-type="sec" rid="s3">Section 3</xref> presents the theoretical preliminaries of the DRL algorithms and details the proposed hybrid framework. <xref ref-type="sec" rid="s4">Section 4</xref> discusses the simulation results and provides an in-depth analysis. Finally, <xref ref-type="sec" rid="s5">Section 5</xref> concludes the paper with key findings.</p>
</sec>
<sec id="s2">
<label>2</label>
<title>System model</title>
<p>This section provides a structured overview of the key components considered in this study, including the network architecture, user/ABS location assumptions, channel models, energy consumption model, and problem formulation.</p>
<p>As illustrated in <xref ref-type="fig" rid="F1">Figure 1</xref>, this study focuses on an ABS-assisted downlink wireless network. The analysis is conducted within a <inline-formula id="inf1">
<mml:math id="m1">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#xd7;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> km area, where <inline-formula id="inf2">
<mml:math id="m2">
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> users are served by <inline-formula id="inf3">
<mml:math id="m3">
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> strategically positioned Ground Base Stations (GBSs). User distribution follows a Poisson point process, modeling realistic spatial randomness. In line with many related works and to focus on network-level optimization, users are assumed static during each optimization interval (i.e., user locations do not change during a decision epoch).</p>
<fig id="F1" position="float">
<label>FIGURE 1</label>
<caption>
<p>A wireless network consisting of multiple Ground Base Stations (GBSs) and Aerial Base Stations (ABSs), where the GBSs are equipped with sleep mode capability to optimize energy efficiency.</p>
</caption>
<graphic xlink:href="frcmn-06-1764320-g001.tif">
<alt-text content-type="machine-generated">Illustration of a city grid with roads and buildings. Cell towers emit red signal waves, surrounded by blue rectangles representing mobile devices. Lightning symbols indicate connections. Two drones, one active and one labeled &#x22;Sleep,&#x22; are shown, illustrating a network or communication concept.</alt-text>
</graphic>
</fig>
<p>Additionally, the network includes <inline-formula id="inf4">
<mml:math id="m4">
<mml:mrow>
<mml:mi>U</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> Aerial Base Stations (ABSs), each hovering at an altitude <inline-formula id="inf5">
<mml:math id="m5">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, where <inline-formula id="inf6">
<mml:math id="m6">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2264;</mml:mo>
<mml:mi>u</mml:mi>
<mml:mo>&#x2264;</mml:mo>
<mml:mi>U</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>. The x-y coordinates of the ABSs will be optimized as part of the network design to enhance coverage, ensure seamless connectivity, and improve overall network performance.</p>
<p>The network employs Orthogonal Frequency Division Multiplexing (OFDM) to serve users across <inline-formula id="inf7">
<mml:math id="m7">
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> subcarriers, where <inline-formula id="inf8">
<mml:math id="m8">
<mml:mrow>
<mml:mi>L</mml:mi>
<mml:mo>&#x226b;</mml:mo>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>. Each GBS <inline-formula id="inf9">
<mml:math id="m9">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> transmits a signal with power <inline-formula id="inf10">
<mml:math id="m10">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> on the <inline-formula id="inf11">
<mml:math id="m11">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th subcarrier, subject to the total power constraint (<xref ref-type="disp-formula" rid="e1">Equation 1</xref>):<disp-formula id="e1">
<mml:math id="m12">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(1)</label>
</disp-formula>where <inline-formula id="inf12">
<mml:math id="m13">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the maximum transmit power of GBS <inline-formula id="inf13">
<mml:math id="m14">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>. Similarly, the <inline-formula id="inf14">
<mml:math id="m15">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th ABS transmits with power <inline-formula id="inf15">
<mml:math id="m16">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> on the <inline-formula id="inf16">
<mml:math id="m17">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th subcarrier, constrained by (<xref ref-type="disp-formula" rid="e2">Equation 2</xref>):<disp-formula id="e2">
<mml:math id="m18">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(2)</label>
</disp-formula>where <inline-formula id="inf17">
<mml:math id="m19">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the maximum transmit power of the ABS.</p>
<p>To model user association, we define a binary variable <inline-formula id="inf18">
<mml:math id="m20">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">{</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">}</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> that indicates whether the <inline-formula id="inf19">
<mml:math id="m21">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user is associated with GBS <inline-formula id="inf20">
<mml:math id="m22">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> on subcarrier <inline-formula id="inf21">
<mml:math id="m23">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>. Specifically, <inline-formula id="inf22">
<mml:math id="m24">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> implies that <inline-formula id="inf23">
<mml:math id="m25">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3e;</mml:mo>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>, and the <inline-formula id="inf24">
<mml:math id="m26">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user is receiving a signal from GBS <inline-formula id="inf25">
<mml:math id="m27">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>. The set of users served by GBS <inline-formula id="inf26">
<mml:math id="m28">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is denoted as (<xref ref-type="disp-formula" rid="e3">Equation 3</xref>):<disp-formula id="e3">
<mml:math id="m29">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="script">D</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:mn>1,2</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>N</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(3)</label>
</disp-formula>where the total number of users in the network satisfies (<xref ref-type="disp-formula" rid="e4">Equation 4</xref>):<disp-formula id="e4">
<mml:math id="m30">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>N</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>K</mml:mi>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(4)</label>
</disp-formula>
</p>
<p>To represent the operational status of each GBS, we introduce a binary sleep indicator <inline-formula id="inf27">
<mml:math id="m31">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, defined as (<xref ref-type="disp-formula" rid="e5">Equation 5</xref>):<disp-formula id="e5">
<mml:math id="m32">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="{" close="">
<mml:mrow>
<mml:mtable class="cases">
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mtext>if&#x2002;GBS&#x2002;</mml:mtext>
<mml:mi>m</mml:mi>
<mml:mtext>&#x2002;is&#x2002;active</mml:mtext>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mn>0</mml:mn>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mtext>if&#x2002;GBS&#x2002;</mml:mtext>
<mml:mi>m</mml:mi>
<mml:mtext>&#x2002;is&#x2002;in&#x2002;sleep&#x2002;mode</mml:mtext>
<mml:mo>.</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
<label>(5)</label>
</disp-formula>
</p>
<p>The <inline-formula id="inf28">
<mml:math id="m33">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th UAV&#x2019;s location is denoted by (<xref ref-type="disp-formula" rid="e6">Equation 6</xref>):<disp-formula id="e6">
<mml:math id="m34">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>z</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(6)</label>
</disp-formula>while each GBS is positioned at the center of its respective cell. The distance between the <inline-formula id="inf29">
<mml:math id="m35">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th UAV and the <inline-formula id="inf30">
<mml:math id="m36">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user at time slot <inline-formula id="inf31">
<mml:math id="m37">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is given by (<xref ref-type="disp-formula" rid="e7">Equation 7</xref>)<disp-formula id="e7">
<mml:math id="m38">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:msqrt>
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>z</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mfenced open="&#x2016;" close="&#x2016;">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">w</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">w</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:msqrt>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(7)</label>
</disp-formula>where <inline-formula id="inf32">
<mml:math id="m39">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">w</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf33">
<mml:math id="m40">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">w</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> denote the horizontal locations of the <inline-formula id="inf34">
<mml:math id="m41">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th UAV and the <inline-formula id="inf35">
<mml:math id="m42">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user, respectively. Furthermore, <inline-formula id="inf36">
<mml:math id="m43">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>z</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the altitude of the UAV, which, without loss of generality, is assumed to be constant in this work.</p>
<sec id="s2-1">
<label>2.1</label>
<title>Air-to-ground (A2G) channel model</title>
<p>The Line-of-Sight (LoS) channel model is commonly employed in UAV-assisted networks to facilitate communication between Aerial Base Stations (ABSs) and Cellular Users (CUs) (<xref ref-type="bibr" rid="B23">Kim et al., 2015</xref>; <xref ref-type="bibr" rid="B22">Khawaja et al., 2019</xref>). The expected channel power gain from the <inline-formula id="inf37">
<mml:math id="m44">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user to the UAV on the <inline-formula id="inf38">
<mml:math id="m45">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th subchannel can be expressed as<disp-formula id="e8">
<mml:math id="m46">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>LoS</mml:mtext>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2b;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>LoS</mml:mtext>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3ba;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:mrow>
</mml:mfenced>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:mrow>
</mml:msup>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(8)</label>
</disp-formula>where <inline-formula id="inf39">
<mml:math id="m47">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:mi>&#x3bb;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>4</mml:mn>
<mml:mi>&#x3c0;</mml:mi>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> represents the channel power gain under LoS conditions at a reference distance of 1&#xa0;m, <inline-formula id="inf40">
<mml:math id="m48">
<mml:mrow>
<mml:mi>&#x3bb;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> denotes the carrier wavelength, <inline-formula id="inf41">
<mml:math id="m49">
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula> is the path-loss exponent constrained within <inline-formula id="inf42">
<mml:math id="m50">
<mml:mrow>
<mml:mn>2</mml:mn>
<mml:mo>&#x3c;</mml:mo>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x3c;</mml:mo>
<mml:mn>6</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>, and <inline-formula id="inf43">
<mml:math id="m51">
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3ba;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x3e;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> is the additional attenuation factor due to Non-Line-of-Sight (NLoS) propagation.</p>
<p>Furthermore, the probability of establishing an LoS link, <inline-formula id="inf44">
<mml:math id="m52">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>LoS</mml:mtext>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, is modeled as (<xref ref-type="disp-formula" rid="e9">Equation 9</xref>):<disp-formula id="e9">
<mml:math id="m53">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>LoS</mml:mtext>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>exp</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2212;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfrac>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(9)</label>
</disp-formula>where <inline-formula id="inf45">
<mml:math id="m54">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf46">
<mml:math id="m55">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> are environment-dependent parameters, and (<xref ref-type="disp-formula" rid="e10">Equation 10</xref>)<disp-formula id="e10">
<mml:math id="m56">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>sin</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msup>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>z</mml:mi>
</mml:mrow>
<mml:mo>&#x304;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
<label>(10)</label>
</disp-formula>denotes the elevation angle between the <inline-formula id="inf47">
<mml:math id="m57">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th UAV and the <inline-formula id="inf48">
<mml:math id="m58">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user at time slot <inline-formula id="inf49">
<mml:math id="m59">
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</sec>
<sec id="s2-2">
<label>2.2</label>
<title>Ground-to-ground (G2G) channel model</title>
<p>For the terrestrial part of the network, which refers to the channel between Ground Base Stations (GBSs) and users, we adopt a fading channel model. The small-scale fading coefficient, <inline-formula id="inf50">
<mml:math id="m60">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>H</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x223c;</mml:mo>
<mml:mi mathvariant="script">C</mml:mi>
<mml:mi mathvariant="script">N</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, follows a complex Gaussian distribution, while the large-scale fading coefficient, <inline-formula id="inf51">
<mml:math id="m61">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>g</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, is modeled using the Hata-COST231 model (<xref ref-type="bibr" rid="B34">Singh, 2012</xref>). The frequency domain downlink channel gain <inline-formula id="inf52">
<mml:math id="m62">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> between the <inline-formula id="inf53">
<mml:math id="m63">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS and the <inline-formula id="inf54">
<mml:math id="m64">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th Cellular User (CU) on the <inline-formula id="inf55">
<mml:math id="m65">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th subchannel consists of small-scale fading, path loss and shadowing components. Pathloss is given by (<xref ref-type="disp-formula" rid="e11">Equation 11</xref>):<disp-formula id="e11">
<mml:math id="m66">
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:msubsup>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2a;</mml:mo>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="{" close="">
<mml:mrow>
<mml:mtable class="cases">
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mo>&#x2212;</mml:mo>
<mml:mi>L</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>35</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3e;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mo>&#x2212;</mml:mo>
<mml:mi>L</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>15</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>20</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mo>&#x2212;</mml:mo>
<mml:mi>L</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>15</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>20</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>.</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
<label>(11)</label>
</disp-formula>where <inline-formula id="inf56">
<mml:math id="m67">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the distance between the <inline-formula id="inf57">
<mml:math id="m68">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user and the <inline-formula id="inf58">
<mml:math id="m69">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS. The parameter <inline-formula id="inf59">
<mml:math id="m70">
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is obtained as (<xref ref-type="disp-formula" rid="e12">Equation 12</xref>):<disp-formula id="e12">
<mml:math id="m71">
<mml:mrow>
<mml:mtable class="align" columnalign="left">
<mml:mtr>
<mml:mtd columnalign="right">
<mml:mi>L</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>46.3</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>33.9</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mi>f</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>13.82</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>GBS</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mspace width="1em"/>
<mml:mo>&#x2212;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1.1</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mi>f</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>0.7</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>user</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="right"/>
<mml:mtd columnalign="left">
<mml:mo>&#x2212;</mml:mo>
<mml:mspace width=".1em"/>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1.56</mml:mn>
<mml:mo>&#x2061;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mi>f</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>0.8</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:math>
<label>(12)</label>
</disp-formula>where <inline-formula id="inf60">
<mml:math id="m72">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>GBS</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf61">
<mml:math id="m73">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>user</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> denote the heights of the GBSs and users, respectively, and <inline-formula id="inf62">
<mml:math id="m74">
<mml:mrow>
<mml:mi>f</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the carrier frequency.</p>
<p>Adding shadowing to the pasthloss, we have (<xref ref-type="disp-formula" rid="e13">Equation 13</xref>):<disp-formula id="e13">
<mml:math id="m75">
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>P</mml:mi>
<mml:msubsup>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2a;</mml:mo>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#xd7;</mml:mo>
<mml:mn>1</mml:mn>
<mml:msup>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>th</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:msup>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(13)</label>
</disp-formula>where <inline-formula id="inf63">
<mml:math id="m76">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:msup>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>th</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:mn>10</mml:mn>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> represents the shadowing fading. Here, <inline-formula id="inf64">
<mml:math id="m77">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x223c;</mml:mo>
<mml:mi mathvariant="script">N</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> is a Gaussian random variable modeling the shadowing effect, and <inline-formula id="inf65">
<mml:math id="m78">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mtext>th</mml:mtext>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is the standard deviation that determines the scale of the shadowing effect.</p>
<p>Hence, the channel gain is expressed as (<xref ref-type="disp-formula" rid="e14">Equation 14</xref>):<disp-formula id="e14">
<mml:math id="m79">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msqrt>
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:msqrt>
<mml:mo>&#xd7;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>H</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(14)</label>
</disp-formula>
</p>
<p>The Signal-to-Interference-plus-Noise Ratio (SINR) of the <inline-formula id="inf66">
<mml:math id="m80">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user served by the <inline-formula id="inf67">
<mml:math id="m81">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS is given by (<xref ref-type="disp-formula" rid="e15">Equation 15</xref>):<disp-formula id="e15">
<mml:math id="m82">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msup>
<mml:mrow>
<mml:mo stretchy="false">&#x7c;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>:</mml:mo>
<mml:mi>M</mml:mi>
<mml:mo>,</mml:mo>
<mml:mtext>&#x2009;</mml:mtext>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x2260;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msup>
<mml:mrow>
<mml:mo stretchy="false">&#x7c;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfrac>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(15)</label>
</disp-formula>
</p>
<p>Similarly, for the <inline-formula id="inf68">
<mml:math id="m83">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th user served by the <inline-formula id="inf69">
<mml:math id="m84">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th ABS, the SINR is (<xref ref-type="disp-formula" rid="e16">Equation 16</xref>):<disp-formula id="e16">
<mml:math id="m85">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msup>
<mml:mrow>
<mml:mo stretchy="false">&#x7c;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>:</mml:mo>
<mml:mi>U</mml:mi>
<mml:mo>,</mml:mo>
<mml:mtext>&#x2009;</mml:mtext>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>&#x2260;</mml:mo>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mo>&#x303;</mml:mo>
</mml:mover>
</mml:mrow>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msup>
<mml:mrow>
<mml:mo stretchy="false">&#x7c;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3c3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfrac>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(16)</label>
</disp-formula>
</p>
<p>The achievable data rate for user <inline-formula id="inf70">
<mml:math id="m86">
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is then expressed as (<xref ref-type="disp-formula" rid="e17">Equation 17</xref>):<disp-formula id="e17">
<mml:math id="m87">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>log</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>q</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mi>q</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(17)</label>
</disp-formula>
</p>
<p>If no user is served on a given subcarrier <inline-formula id="inf71">
<mml:math id="m88">
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula>, then the corresponding transmit power is set to zero, i.e., <inline-formula id="inf72">
<mml:math id="m89">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>q</mml:mi>
<mml:mo>,</mml:mo>
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>. To represent the operational state of each GBS, we introduce a binary parameter <inline-formula id="inf73">
<mml:math id="m90">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">{</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">}</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>, which determines whether the <inline-formula id="inf74">
<mml:math id="m91">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS is active or in sleep mode. If a GBS is forced into sleep mode, all users associated with it will be disconnected and must connect to neighboring GBSs. Consequently, the transmit power of a sleeping GBS is forced to zero, i.e., <inline-formula id="inf75">
<mml:math id="m92">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</sec>
<sec id="s2-3">
<label>2.3</label>
<title>Power consumption model</title>
<p>The total power consumption in UAV-assisted networks comprises two primary components: the power consumption of the ground network and the power consumption of UAVs.</p>
<sec id="s2-3-1">
<label>2.3.1</label>
<title>Power consumption of the ground network</title>
<p>The power consumption of the ground network includes the following components:<list list-type="order">
<list-item>
<p>Transmission Power: <inline-formula id="inf76">
<mml:math id="m93">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula>, which represents the transmission power of the <inline-formula id="inf77">
<mml:math id="m94">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS.</p>
</list-item>
<list-item>
<p>Circuit Power: <inline-formula id="inf78">
<mml:math id="m95">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">cte</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula>, the power consumed by the circuits and other electronic components of the <inline-formula id="inf79">
<mml:math id="m96">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS, which is considered constant.</p>
</list-item>
<list-item>
<p>Mode-dependent Power Consumption: The power consumed by a GBS due to its active or sleep mode operation, including power supply and air conditioning, given by (<xref ref-type="disp-formula" rid="e18">Equation 18</xref>):</p>
</list-item>
</list>
<disp-formula id="e18">
<mml:math id="m97">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">mod</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">off</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(18)</label>
</disp-formula>where <inline-formula id="inf80">
<mml:math id="m98">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>o</mml:mi>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf81">
<mml:math id="m99">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">off</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> represent the power consumption of the GBS in active mode and sleep mode, respectively.<list list-type="order">
<list-item>
<p>Mode Transition Power: The power associated with transitioning the base station <inline-formula id="inf82">
<mml:math id="m100">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> between operational modes is given by (<xref ref-type="disp-formula" rid="e19">Equation 19</xref>):</p>
</list-item>
</list>
<disp-formula id="e19">
<mml:math id="m101">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tran</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">prev</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tran</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
<label>(19)</label>
</disp-formula>where <inline-formula id="inf83">
<mml:math id="m102">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">prev</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> denotes the operational state (active or sleep) of the <inline-formula id="inf84">
<mml:math id="m103">
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>th GBS during the previous time slot, and <inline-formula id="inf85">
<mml:math id="m104">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tran</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> represents the energy consumed for transitioning between modes.</p>
</sec>
<sec id="s2-3-2">
<label>2.3.2</label>
<title>Power consumption of UAVs</title>
<p>The power consumption of a UAV consists of three main components (<xref ref-type="disp-formula" rid="e20">Equation 20</xref>):<disp-formula id="e20">
<mml:math id="m105">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">com</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">hov</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">cte</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
<label>(20)</label>
</disp-formula>where <inline-formula id="inf86">
<mml:math id="m106">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">com</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> is the power consumed by the UAV&#x2019;s communication systems, including the radio transmitter, receiver, antennas, and other related components, expressed as (<xref ref-type="disp-formula" rid="e21">Equation 21</xref>):<disp-formula id="e21">
<mml:math id="m107">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">com</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(21)</label>
</disp-formula>
<inline-formula id="inf87">
<mml:math id="m108">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">cte</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> is the power consumed by the UAV&#x2019;s internal circuits and other onboard electronic devices, and <inline-formula id="inf88">
<mml:math id="m109">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">hov</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:math>
</inline-formula> represents the power consumed when the UAV is hovering or stationary in the air. According to <xref ref-type="bibr" rid="B16">Ghorbel et al. (2019)</xref>, the hovering power is given by (<xref ref-type="disp-formula" rid="e22">Equation 22</xref>):<disp-formula id="e22">
<mml:math id="m110">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">hov</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
<mml:msqrt>
<mml:mrow>
<mml:mfrac>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tot</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mo>&#x2061;</mml:mo>
<mml:mi>g</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mn>3</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
<mml:mi>&#x3c0;</mml:mi>
<mml:msubsup>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msubsup>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>&#x3c1;</mml:mi>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:msqrt>
</mml:mrow>
</mml:math>
<label>(22)</label>
</disp-formula>where <inline-formula id="inf89">
<mml:math id="m111">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tot</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> is the total mass of the UAV, <inline-formula id="inf90">
<mml:math id="m112">
<mml:mrow>
<mml:mi>g</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is the gravitational acceleration, <inline-formula id="inf91">
<mml:math id="m113">
<mml:mrow>
<mml:mi>&#x3c1;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is the air density, and <inline-formula id="inf92">
<mml:math id="m114">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf93">
<mml:math id="m115">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> are the radius and number of propellers, respectively.</p>
</sec>
<sec id="s2-3-3">
<label>2.3.3</label>
<title>Total weighted power consumption</title>
<p>In UAV-based networks, the energy consumed by UAVs for hovering and flying is typically much higher than the power required for communication. To address this imbalance, a weighting factor <inline-formula id="inf94">
<mml:math id="m116">
<mml:mrow>
<mml:mi>w</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is introduced to balance the trade-off between hovering/flying power and communication-related power consumption (<xref ref-type="bibr" rid="B42">Zhu et al., 2021</xref>). This approach allows for an optimized allocation of energy resources across the network.</p>
<p>The total weighted power consumption of the UAV-assisted network is given by (<xref ref-type="disp-formula" rid="e23">Equation 23</xref>):<disp-formula id="e23">
<mml:math id="m117">
<mml:mrow>
<mml:mtable class="aligned">
<mml:mtr>
<mml:mtd columnalign="right">
<mml:msup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tot</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mo>&#x3d;</mml:mo>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mi>w</mml:mi>
<mml:mfenced open="(" close="">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">mod</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tran</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">cte</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="right"/>
<mml:mtd columnalign="left">
<mml:mfenced open="" close=")">
<mml:mrow>
<mml:mo>&#x2b;</mml:mo>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>U</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">com</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">cte</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2b;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>w</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>U</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">hov</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>.</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:math>
<label>(23)</label>
</disp-formula>In the simulations, the weighting factor is set to <inline-formula id="inf95">
<mml:math id="m118">
<mml:mrow>
<mml:mi>w</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0.5</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>, providing a balanced trade-off between GBS-related power consumption and ABS hovering energy, consistent with <xref ref-type="bibr" rid="B14">F&#xe4;hrmann et al. (2022)</xref>.</p>
</sec>
</sec>
<sec id="s2-4">
<label>2.4</label>
<title>Energy efficiency</title>
<p>The Energy Efficiency (EE) criteria serve as a critical framework for evaluating the effectiveness of resource allocation within the network, particularly when GBSs are in sleep mode. By computing the Energy Efficiency metric, we ensure that the power consumption of GBSs does not compromise users&#x2019; quality of service. The Energy Efficiency (EE) criterion is defined as (<xref ref-type="disp-formula" rid="e24">Equation 24</xref>):<disp-formula id="e24">
<mml:math id="m119">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>E</mml:mi>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c4;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tot</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfrac>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(24)</label>
</disp-formula>where <inline-formula id="inf96">
<mml:math id="m120">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3c4;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> represents the total achievable data rate of all <inline-formula id="inf97">
<mml:math id="m121">
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> users, and <inline-formula id="inf98">
<mml:math id="m122">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">tot</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> denotes the total power consumption of the network.</p>
</sec>
<sec id="s2-5">
<label>2.5</label>
<title>Optimization problem formulation</title>
<p>The optimization problem for maximizing the energy efficiency of the cellular network is formulated as (<xref ref-type="disp-formula" rid="e25">Equation 25</xref>):<disp-formula id="e25">
<mml:math id="m123">
<mml:mrow>
<mml:munder>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:munder>
<mml:mtext>&#x2003;&#x2003;</mml:mtext>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>E</mml:mi>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
<label>(25)</label>
</disp-formula>subject to the following constraints:<disp-formula id="e26a">
<mml:math id="m124">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">1</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mn>0</mml:mn>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>G</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mn>0</mml:mn>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>U</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
<label>(26a)</label>
</disp-formula>
<disp-formula id="e26b">
<mml:math id="m125">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">2</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:mfenced open="{" close="}">
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(26b)</label>
</disp-formula>
<disp-formula id="e26c">
<mml:math id="m126">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">3</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>M</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2b;</mml:mo>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>U</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:math>
<label>(26c)</label>
</disp-formula>
<disp-formula id="e26d">
<mml:math id="m127">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">4</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:math>
<label>(26d)</label>
</disp-formula>
<disp-formula id="e26e">
<mml:math id="m128">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">5</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>l</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
<label>(26e)</label>
</disp-formula>
<disp-formula id="e26f">
<mml:math id="m129">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">6</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mn>0</mml:mn>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>x</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
<label>(26f)</label>
</disp-formula>
<disp-formula id="e26g">
<mml:math id="m130">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">7</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:mn>0</mml:mn>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2264;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>L</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>y</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>u</mml:mi>
</mml:mrow>
</mml:math>
<label>(26g)</label>
</disp-formula>
<disp-formula id="e26h">
<mml:math id="m131">
<mml:mrow>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold">C</mml:mi>
<mml:mn mathvariant="bold">8</mml:mn>
</mml:mrow>
</mml:mfenced>
<mml:mspace width="1em"/>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2265;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:math>
<label>(26h)</label>
</disp-formula>
</p>
<p>Constraint <bold>(C1)</bold> ensures that the transmission power per subcarrier remains within the maximum allowable thresholds for both GBSs and ABSs, as defined in (<xref ref-type="disp-formula" rid="e26a">Equation 26a</xref>).</p>
<p>Constraint <bold>(C2)</bold> determines the operational status of each GBS, where <inline-formula id="inf99">
<mml:math id="m132">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> indicates that the GBS is active and <inline-formula id="inf100">
<mml:math id="m133">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> signifies that it is switched to sleep mode, as given in (<xref ref-type="disp-formula" rid="e26b">Equation 26b</xref>).</p>
<p>Constraint <bold>(C3)</bold> guarantees exclusive user association by ensuring that each user is connected to exactly one serving node&#x2014;either a single GBS or a single ABS&#x2014;on each subcarrier. The inclusion of the activity indicator <inline-formula id="inf101">
<mml:math id="m134">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> prevents user association with sleeping GBSs, thereby avoiding multiple or invalid connections, as stated in (<xref ref-type="disp-formula" rid="e26c">Equation 26c</xref>).</p>
<p>Constraint <bold>(C4)</bold> enforces that the total transmit power of each GBS does not exceed its maximum permissible value, accounting for both active and sleep states, as given in (<xref ref-type="disp-formula" rid="e26d">Equation 26d</xref>). Constraint <bold>(C5)</bold> ensures that ABSs operate within their defined power limitations, as specified in (<xref ref-type="disp-formula" rid="e26e">Equation 26e</xref>).</p>
<p>Constraints <bold>(C6)</bold> and <bold>(C7)</bold> confine ABSs within the designated region of interest, ensuring that they remain within operational limits, as enforced in (<xref ref-type="disp-formula" rid="e26f">Equation 26f</xref>) and (<xref ref-type="disp-formula" rid="e26g">Equation 26g</xref>), respectively. Finally, Constraint <bold>(C8)</bold> guarantees that each user achieves the minimum required data rate, thereby maintaining the network&#x2019;s quality of service (QoS), as defined in (<xref ref-type="disp-formula" rid="e26h">Equation 26h</xref>).</p>
<p>The optimization problem formulated in (<xref ref-type="disp-formula" rid="e25">Equation 25</xref>) is a mixed-integer non-convex problem involving both discrete and continuous variables, rendering it intractable for conventional mathematical optimization techniques. To address this complexity, a learning-based approach is introduced in the following section.</p>
</sec>
</sec>
<sec id="s3">
<label>3</label>
<title>DRL-based framework for optimizing complex problems</title>
<p>This section presents the hybrid DDPG&#x2013;DDQL framework developed to address the energy efficiency optimization problem in ABS-assisted B5G networks incorporating a sleep strategy, as formulated in (<xref ref-type="disp-formula" rid="e25">Equation 25</xref>).</p>
<sec id="s3-1">
<label>3.1</label>
<title>Basics of deep reinforcement learning (DRL)</title>
<p>Reinforcement Learning (RL) has significantly advanced Artificial Intelligence (AI) by enabling agents to make decisions, observe outcomes, and iteratively refine their strategies to determine an optimal policy (<xref ref-type="bibr" rid="B28">Morocho-Cayamcela et al., 2019</xref>; <xref ref-type="bibr" rid="B19">Huang et al., 2019</xref>). However, due to its reliance on extensive exploration, traditional RL can be slow and computationally expensive, limiting its applicability in large-scale networks.</p>
<p>Deep Reinforcement Learning (DRL) integrates Deep Neural Networks (DNNs) into RL, significantly enhancing learning speed and efficiency. In applications such as IoT and UAV-assisted networks, devices often need to make independent decisions to optimize network performance. These scenarios are frequently modeled as Markov Decision Processes (MDPs), which are formally defined as a quintuple <inline-formula id="inf102">
<mml:math id="m135">
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi mathvariant="script">S</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi mathvariant="script">A</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi mathvariant="script">P</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi mathvariant="script">R</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>&#x3b6;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:math>
</inline-formula>:<list list-type="bullet">
<list-item>
<p>
<inline-formula id="inf103">
<mml:math id="m136">
<mml:mrow>
<mml:mi mathvariant="script">S</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents a K-dimensional state space, with each state at time <inline-formula id="inf104">
<mml:math id="m137">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> denoted as <inline-formula id="inf105">
<mml:math id="m138">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>
<inline-formula id="inf106">
<mml:math id="m139">
<mml:mrow>
<mml:mi mathvariant="script">A</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> defines the finite action space available to the agent.</p>
</list-item>
<list-item>
<p>
<inline-formula id="inf107">
<mml:math id="m140">
<mml:mrow>
<mml:mi mathvariant="script">P</mml:mi>
<mml:mo>:</mml:mo>
<mml:mi mathvariant="script">S</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi mathvariant="script">A</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi mathvariant="script">S</mml:mi>
<mml:mo>&#x2192;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> is the transition probability function, specifying the likelihood of transitioning from state <inline-formula id="inf108">
<mml:math id="m141">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> to state <inline-formula id="inf109">
<mml:math id="m142">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> after taking action <inline-formula id="inf110">
<mml:math id="m143">
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, expressed as <inline-formula id="inf111">
<mml:math id="m144">
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo>,</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>
<inline-formula id="inf112">
<mml:math id="m145">
<mml:mrow>
<mml:mi mathvariant="script">R</mml:mi>
<mml:mo>:</mml:mo>
<mml:mi mathvariant="script">S</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mi mathvariant="script">A</mml:mi>
<mml:mo>&#x2192;</mml:mo>
<mml:mi mathvariant="double-struck">R</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> is the expected reward function, quantifying the anticipated reward upon executing action <inline-formula id="inf113">
<mml:math id="m146">
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> in state <inline-formula id="inf114">
<mml:math id="m147">
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
<list-item>
<p>
<inline-formula id="inf115">
<mml:math id="m148">
<mml:mrow>
<mml:mi>&#x3b6;</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mfenced open="" close=")">
<mml:mrow>
<mml:mfenced open="[" close="">
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
</inline-formula> is the discount factor that determines the importance of future rewards.</p>
</list-item>
</list>
</p>
<p>Although traditional methods such as dynamic programming and value iteration can solve MDPs, they become computationally impractical for large-scale and complex networks. DRL techniques, particularly Deep Q-Learning (DQL), provide scalable solutions by approximating value functions using deep neural networks.</p>
</sec>
<sec id="s3-2">
<label>3.2</label>
<title>Deep Q-learning (DQL) and its limitations</title>
<p>Deep Q-Learning (DQL) is a fundamental DRL algorithm that estimates Q-values for state-action pairs using neural networks (<xref ref-type="bibr" rid="B10">Braga et al., 2020</xref>). For an agent parameterized by <inline-formula id="inf116">
<mml:math id="m149">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> at time <inline-formula id="inf117">
<mml:math id="m150">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, the DQL update equation after taking action <inline-formula id="inf118">
<mml:math id="m151">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> in state <inline-formula id="inf119">
<mml:math id="m152">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, receiving immediate reward <inline-formula id="inf120">
<mml:math id="m153">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, and transitioning to the next state <inline-formula id="inf121">
<mml:math id="m154">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is (<xref ref-type="disp-formula" rid="e27">Equation 27</xref>):<disp-formula id="e27">
<mml:math id="m155">
<mml:mrow>
<mml:mtable class="aligned">
<mml:mtr>
<mml:mtd columnalign="right">
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mo>&#x3d;</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3bd;</mml:mi>
<mml:mfenced open="[" close="">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3b6;</mml:mi>
<mml:munder>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:munder>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="right"/>
<mml:mtd columnalign="left">
<mml:mfenced open="" close="]">
<mml:mrow>
<mml:mspace width="1em"/>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:math>
<label>(27)</label>
</disp-formula>Here, <inline-formula id="inf122">
<mml:math id="m156">
<mml:mrow>
<mml:mi>&#x3bd;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> represents the learning rate. However, the Q-learning update consistently overestimates Q-values due to the use of bootstrapping, where estimates are derived from other estimates. This bias is exacerbated by using the same Q-network for both action selection and evaluation.</p>
</sec>
<sec id="s3-3">
<label>3.3</label>
<title>Double deep Q-learning (DDQL)</title>
<p>To mitigate the overestimation bias in DQL, the authors in (<xref ref-type="bibr" rid="B14">F&#xe4;hrmann et al., 2022</xref>; <xref ref-type="bibr" rid="B33">Shokrnezhad et al., 2024</xref>) introduced Double Deep Q-Learning (DDQL), which decouples action selection and evaluation employing two distinct Q networks:<list list-type="bullet">
<list-item>
<p>The primary Q-network: <inline-formula id="inf123">
<mml:math id="m157">
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> selects actions.</p>
</list-item>
<list-item>
<p>The target network: <inline-formula id="inf124">
<mml:math id="m158">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> evaluates actions, using separate parameters <inline-formula id="inf125">
<mml:math id="m159">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf126">
<mml:math id="m160">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</list-item>
</list>
</p>
<p>The DDQL update equation is (<xref ref-type="disp-formula" rid="e28">Equation 28</xref>):<disp-formula id="e28">
<mml:math id="m161">
<mml:mrow>
<mml:mtable class="aligned">
<mml:mtr>
<mml:mtd columnalign="right">
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mo>&#x3d;</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3bd;</mml:mi>
<mml:mo>[</mml:mo>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2b;</mml:mo>
</mml:mrow>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="left">
<mml:mfenced open="" close="]">
<mml:mrow>
<mml:mi>&#x3b6;</mml:mi>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mi>arg</mml:mi>
<mml:munder>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:munder>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:math>
<label>(28)</label>
</disp-formula>
</p>
<p>The target network <inline-formula id="inf127">
<mml:math id="m162">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> is periodically updated using Polyak avenging (<xref ref-type="disp-formula" rid="e29">Equation 29</xref>):<disp-formula id="e29">
<mml:math id="m163">
<mml:mrow>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
<mml:mo>&#x2b;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3c4;</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:msubsup>
<mml:mo>&#x2b;</mml:mo>
<mml:mi>&#x3c4;</mml:mi>
<mml:msubsup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msubsup>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(29)</label>
</disp-formula>where <inline-formula id="inf128">
<mml:math id="m164">
<mml:mrow>
<mml:mi>&#x3c4;</mml:mi>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mn>0,1</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> controls the update rate.</p>
<p>Although DDQL effectively reduces overestimation and improves convergence in discrete action spaces (such as GBS sleep mode decisions), it struggles with continuous action spaces, such as power allocation and ABS positioning.</p>
</sec>
<sec id="s3-4">
<label>3.4</label>
<title>Deep deterministic policy gradient (DDPG)</title>
<p>For continuous action spaces, Deep Deterministic Policy Gradient (DDPG) algorithm (<xref ref-type="bibr" rid="B40">Yu et al., 2021</xref>) is a more suitable approach. DDPG is an actor-critic algorithm that efficiently handles sequential decision making. It optimizes a policy function <inline-formula id="inf129">
<mml:math id="m165">
<mml:mrow>
<mml:mi>&#x3c0;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, mapping states to actions, by maximizing the objective function (<xref ref-type="disp-formula" rid="e30">Equation 30</xref>):<disp-formula id="e30">
<mml:math id="m166">
<mml:mrow>
<mml:mi>J</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:mi mathvariant="double-struck">E</mml:mi>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:mi>S</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3c0;</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(30)</label>
</disp-formula>
</p>
<p>Unlike DQL, where policies output a probability distribution over discrete actions, DDPG directly maps states to actions through a policy network <inline-formula id="inf130">
<mml:math id="m167">
<mml:mrow>
<mml:mi>&#x3c0;</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. The policy network parameters <inline-formula id="inf131">
<mml:math id="m168">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula> are updated using the gradient (<xref ref-type="disp-formula" rid="e31">Equation 31</xref>):<disp-formula id="e31">
<mml:math id="m169">
<mml:mrow>
<mml:mi>&#x2207;</mml:mi>
<mml:msub>
<mml:mrow>
<mml:mi>J</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
<mml:mi>&#x3bc;</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2248;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x2207;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mi>&#x2207;</mml:mi>
<mml:mi>&#x3bc;</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(31)</label>
</disp-formula>where <inline-formula id="inf132">
<mml:math id="m170">
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> is the policy network (the actor), and <inline-formula id="inf133">
<mml:math id="m171">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x2207;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>Q</mml:mi>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> represents the gradient derived from a Q-network (the critic).</p>
<p>In large-scale environments with numerous actions, the actor-critic framework efficiently approximates Q-values using (<xref ref-type="disp-formula" rid="e32">Equation 32</xref>):<disp-formula id="e32">
<mml:math id="m172">
<mml:mrow>
<mml:munder>
<mml:mrow>
<mml:mi>max</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:munder>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2248;</mml:mo>
<mml:mi>Q</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>a</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>Q</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfenced>
<mml:msub>
<mml:mrow>
<mml:mo stretchy="false">&#x7c;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mi>&#x3bc;</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>s</mml:mi>
<mml:mo stretchy="false">&#x7c;</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b8;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>&#x3bc;</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:msub>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(32)</label>
</disp-formula>
</p>
<p>Similarly to DDQL, DDPG enhances stability by using:<list list-type="bullet">
<list-item>
<p>Experience replay to train the critic network.</p>
</list-item>
<list-item>
<p>Target networks for both actors and critics, updated using Polyak averaging.</p>
</list-item>
</list>
</p>
</sec>
<sec id="s3-5">
<label>3.5</label>
<title>Hybrid DDPG-DDQL for UAV-assisted B5G networks</title>
<p>Given the nature of our optimization problem, which involves both discrete decisions (e.g., GBS sleep mode and discrete association choices) and continuous variables (e.g., power allocation and ABS positioning), we propose a Hybrid DDPG&#x2013;DDQL framework. In this hybrid design, DDQL handles the discrete decision component and mitigates overestimation bias, while DDPG learns a deterministic continuous-control policy through actor&#x2013;critic training. This explicit separation helps stabilize learning and reduces the overall search complexity compared to using a single algorithm to handle a mixed action space.</p>
</sec>
<sec id="s3-6">
<label>3.6</label>
<title>Proposed hybrid DDPG-DDQL framework with ABS-assisted sleep strategy</title>
<p>The objective of this research is to develop a DRL-based framework that optimizes the sleep scheduling of GBSs, the power allocation vector, and the ABS positioning vector, based on a given Channel State Information (CSI) matrix, defined as (<xref ref-type="disp-formula" rid="e33">Equation 33</xref>):<disp-formula id="e33">
<mml:math id="m173">
<mml:mrow>
<mml:mi mathvariant="bold">H</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(33)</label>
</disp-formula>where without loss of generality, a single UAV in the network has been assumed and hence, the UAV index of the channel matrix discarded. Furthermore, <inline-formula id="inf134">
<mml:math id="m174">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2026;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>G</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>L</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>. To achieve this, we propose a hybrid DRL system that integrates DDPG and DDQL to jointly learn optimal sleeping configurations and continuous control actions in relation to the CSI matrix <inline-formula id="inf135">
<mml:math id="m175">
<mml:mrow>
<mml:mi mathvariant="bold">H</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> (or any related network metric). A schematic representation of the proposed hybrid DDPG&#x2013;DDQL architecture is illustrated in <xref ref-type="fig" rid="F2">Figure 2</xref>.</p>
<fig id="F2" position="float">
<label>FIGURE 2</label>
<caption>
<p>Hybrid DDPG&#x2013;DDQL framework for optimizing ABS-assisted networks.</p>
</caption>
<graphic xlink:href="frcmn-06-1764320-g002.tif">
<alt-text content-type="machine-generated">A diagram illustrating a reinforcement learning model architecture combining DDQL and DDPG. It includes neural networks for DDQL Q-N, and DDPG Critic and Actor, connected to components such as observed state \(S\), action outputs \(a^c\) and \(a^b\), and a Replay Buffer. A visual of telecom towers interacting with mobile devices is also present.</alt-text>
</graphic>
</fig>
<p>For a given sleep configuration, the actor&#x2013;critic DDPG algorithm (<xref ref-type="bibr" rid="B41">Zhou et al., 2022</xref>) is used to optimize the power allocation and ABS horizontal positioning. DDPG continuously outputs the continuous control vector, denoted by <inline-formula id="inf136">
<mml:math id="m176">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi mathvariant="bold">p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>L</mml:mi>
<mml:mi>K</mml:mi>
<mml:mo>&#xd7;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, and the ABS location vector <inline-formula id="inf137">
<mml:math id="m177">
<mml:mrow>
<mml:mi mathvariant="bold">w</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">(</mml:mo>
<mml:mrow>
<mml:mi>x</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>y</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
<p>To determine the optimal sleep configuration, the DDQL algorithm (<xref ref-type="bibr" rid="B37">Van Hasselt et al., 2016</xref>) is used, since the number of possible sleep configurations is finite and each configuration index is discrete.</p>
</sec>
<sec id="s3-7">
<label>3.7</label>
<title>Optimization problem formulation</title>
<p>As illustrated in <xref ref-type="fig" rid="F2">Figure 2</xref>, both algorithms interact with a simulated ABS-assisted network environment to address the optimization problem formulated in <xref ref-type="disp-formula" rid="e24">Equation 24</xref>.</p>
<p>The network environment state is represented as (<xref ref-type="disp-formula" rid="e34">Equation 34</xref>):<disp-formula id="e34">
<mml:math id="m178">
<mml:mrow>
<mml:mi mathvariant="bold">S</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
</mml:mrow>
</mml:math>
<label>(34)</label>
</disp-formula>where each state <inline-formula id="inf138">
<mml:math id="m179">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> is defined by the Signal-to-Interference-plus-Noise Ratio (SINR) values of users, represented as (<xref ref-type="disp-formula" rid="e35">Equation 35</xref>):<disp-formula id="e35">
<mml:math id="m180">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>s</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b3;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>u</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>l</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(35)</label>
</disp-formula>
</p>
</sec>
<sec id="s3-8">
<label>3.8</label>
<title>Reward function and action space</title>
<p>The immediate reward function <inline-formula id="inf139">
<mml:math id="m181">
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> plays a crucial role in estimating optimal policies and Q-values. While energy efficiency is the primary objective, the reward must also discourage QoS violations. Therefore, we define the reward as the energy efficiency when all users satisfy the minimum rate requirement, and apply a penalty otherwise (<xref ref-type="disp-formula" rid="e36">Equation 36</xref>):<disp-formula id="e36">
<mml:math id="m182">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="{" close="">
<mml:mrow>
<mml:mtable class="cases">
<mml:mtr>
<mml:mtd columnalign="left">
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>E</mml:mi>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mtext>if&#x2009;</mml:mtext>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2265;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:mspace width="0.3333em"/>
<mml:mo>&#x2200;</mml:mo>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd columnalign="left">
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b7;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>E</mml:mi>
<mml:mi>E</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x3bb;</mml:mi>
<mml:mstyle displaystyle="true">
<mml:munderover>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mi>K</mml:mi>
</mml:mrow>
</mml:munderover>
</mml:mstyle>
<mml:mi>max</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>0</mml:mn>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>k</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>t</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mo>,</mml:mo>
<mml:mspace width="1em"/>
</mml:mtd>
<mml:mtd columnalign="left">
<mml:mtext>otherwise</mml:mtext>
<mml:mo>,</mml:mo>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:math>
<label>(36)</label>
</disp-formula>where <inline-formula id="inf140">
<mml:math id="m183">
<mml:mrow>
<mml:mi>&#x3bb;</mml:mi>
<mml:mo>&#x3e;</mml:mo>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> controls the penalty severity. This reward structure encourages the agent to improve energy efficiency while maintaining feasibility with respect to QoS constraints. (If the original <xref ref-type="disp-formula" rid="e27">Equation 27</xref> corresponds to the reward, <xref ref-type="disp-formula" rid="e36">Equation 36</xref> replaces it and is referenced accordingly in the revised manuscript).</p>
<p>In this framework, as shown in <xref ref-type="fig" rid="F2">Figure 2</xref>, the action space <inline-formula id="inf141">
<mml:math id="m184">
<mml:mrow>
<mml:mi mathvariant="script">A</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> consists of an action pair (<xref ref-type="disp-formula" rid="e37">Equation 37</xref>):<disp-formula id="e37">
<mml:math id="m185">
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>b</mml:mi>
</mml:mrow>
</mml:msup>
<mml:mo>,</mml:mo>
<mml:msup>
<mml:mrow>
<mml:mi>a</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>c</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:mfenced>
<mml:mo>&#x3d;</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="bold-italic">&#x3c9;</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi mathvariant="bold">c</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(37)</label>
</disp-formula>Here,<list list-type="bullet">
<list-item>
<p>The continuous action vector <inline-formula id="inf142">
<mml:math id="m186">
<mml:mrow>
<mml:mi mathvariant="bold-italic">&#x3c9;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, which includes power allocation and ABS position, is generated using the DDPG algorithm.</p>
</list-item>
<list-item>
<p>tThe discrete action vector <inline-formula id="inf143">
<mml:math id="m187">
<mml:mrow>
<mml:mi mathvariant="bold">c</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, which includes the GBS sleep configuration (and any discrete association selection if applicable), is produced by the DDQL algorithm.</p>
</list-item>
</list>
</p>
</sec>
<sec id="s3-9">
<label>3.9</label>
<title>Computational complexity analysis</title>
<p>The computational complexity of the proposed method mainly arises during the offline training stage due to iterative neural network updates in both DDPG and DDQL. For a fully connected network layer <inline-formula id="inf144">
<mml:math id="m188">
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, let <inline-formula id="inf145">
<mml:math id="m189">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf146">
<mml:math id="m190">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> denote the input and output dimensions, and let <inline-formula id="inf147">
<mml:math id="m191">
<mml:mrow>
<mml:mi>b</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> be the batch size. The dominant cost per training update is due to matrix multiplications, leading to (<xref ref-type="disp-formula" rid="e38">Equation 38</xref>):<disp-formula id="e38">
<mml:math id="m192">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:mi mathvariant="script">O</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mi>b</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(38)</label>
</disp-formula>
</p>
<p>After training, the online inference stage requires only forward passes through the trained networks. With <inline-formula id="inf148">
<mml:math id="m193">
<mml:mrow>
<mml:mi>b</mml:mi>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> (as in online RL decision-making), the complexity becomes (<xref ref-type="disp-formula" rid="e39">Equation 39</xref>):<disp-formula id="e39">
<mml:math id="m194">
<mml:mrow>
<mml:mstyle displaystyle="true">
<mml:munder>
<mml:mrow>
<mml:mo>&#x2211;</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:munder>
</mml:mstyle>
<mml:mi mathvariant="script">O</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
<mml:mo>.</mml:mo>
</mml:mrow>
</mml:math>
<label>(39)</label>
</disp-formula>
</p>
<p>In addition, the DDQL component evaluates a discrete action among a finite set of sleep configurations; thus, the per-step selection overhead scales with the number of discrete actions (sleep configurations) considered by the DDQL output layer. Overall, the proposed approach is computationally intensive during offline training but has low online computational overhead, making it suitable for real-time operation once trained.</p>
</sec>
</sec>
<sec sec-type="results|discussion" id="s4">
<label>4</label>
<title>Results and discussions</title>
<p>In this section, we present the results of the proposed hybrid-DRL algorithm to optimize energy efficiency in ABS-assisted cellular networks. The simulation parameters are summarized in <xref ref-type="table" rid="T1">Table 1</xref>.</p>
<table-wrap id="T1" position="float">
<label>TABLE 1</label>
<caption>
<p>Simulation parameters.</p>
</caption>
<table>
<thead valign="top">
<tr>
<th align="center">Parameter</th>
<th align="center">Value</th>
</tr>
</thead>
<tbody valign="top">
<tr>
<td align="center">
<inline-formula id="inf149">
<mml:math id="m195">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>f</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>c</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">2&#xa0;GHz</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf150">
<mml:math id="m196">
<mml:mrow>
<mml:mi>B</mml:mi>
<mml:mi>W</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">20&#xa0;MHz</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf151">
<mml:math id="m197">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>r</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">0.2</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf152">
<mml:math id="m198">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>n</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">4</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf153">
<mml:math id="m199">
<mml:mrow>
<mml:mi>&#x3c1;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">102&#xa0;kg/s<sup>3</sup>
</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf154">
<mml:math id="m200">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>m</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">ToT</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">750 gr</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf155">
<mml:math id="m201">
<mml:mrow>
<mml:mover accent="true">
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mo stretchy="false">&#x302;</mml:mo>
</mml:mover>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">3</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf156">
<mml:math id="m202">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">GBS</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>h</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">user</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">15&#xa0;m, 1.65&#xa0;m</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf157">
<mml:math id="m203">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>o</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mrow>
<mml:mi>d</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, <inline-formula id="inf158">
<mml:math id="m204">
<mml:mrow>
<mml:mi>D</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">10&#xa0;m, 50&#xa0;m</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf159">
<mml:math id="m205">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b1;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">9.53</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf160">
<mml:math id="m206">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2032;</mml:mo>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">0.41</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf161">
<mml:math id="m207">
<mml:mrow>
<mml:msup>
<mml:mrow>
<mml:mi>P</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi mathvariant="italic">max</mml:mi>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">5&#xa0;W</td>
</tr>
<tr>
<td align="center">Number of GBSs</td>
<td align="center">12</td>
</tr>
<tr>
<td align="center">Number of users</td>
<td align="center">20</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf162">
<mml:math id="m208">
<mml:mrow>
<mml:mi>&#x3b6;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">0.99</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf163">
<mml:math id="m209">
<mml:mrow>
<mml:mi>&#x3bd;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">
<inline-formula id="inf164">
<mml:math id="m210">
<mml:mrow>
<mml:mn>5</mml:mn>
<mml:mo>&#xd7;</mml:mo>
<mml:mn>1</mml:mn>
<mml:msup>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>5</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
</tr>
<tr>
<td align="center">Polyak avaraging parameter <inline-formula id="inf165">
<mml:math id="m211">
<mml:mrow>
<mml:mi>&#x3c4;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">
<inline-formula id="inf166">
<mml:math id="m212">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:msup>
<mml:mrow>
<mml:mn>0</mml:mn>
</mml:mrow>
<mml:mrow>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>3</mml:mn>
</mml:mrow>
</mml:msup>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
</tr>
<tr>
<td align="center">R</td>
<td align="center">2000</td>
</tr>
<tr>
<td align="center">w</td>
<td align="center">0.5</td>
</tr>
<tr>
<td align="center">
<inline-formula id="inf167">
<mml:math id="m213">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mi>&#x26;</mml:mi>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>
</td>
<td align="center">0.9 <inline-formula id="inf168">
<mml:math id="m214">
<mml:mrow>
<mml:mi>&#x26;</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> 0.999</td>
</tr>
</tbody>
</table>
</table-wrap>
<p>The actor and critic networks, along with their respective target networks, are designed with two hidden layers comprising 256 and 128 neurons, respectively. In contrast, the DDQN architecture includes two fully connected layers with 64 neurons each, followed by ReLU activation functions, and terminates with a linear output layer. To enhance convergence and ensure training stability, the Adam optimizer is employed for the critic network, using its default hyperparameters <inline-formula id="inf169">
<mml:math id="m215">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf170">
<mml:math id="m216">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>&#x3b2;</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>2</mml:mn>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>, which control the exponential moving averages of the gradients and their squared values, respectively (<xref ref-type="bibr" rid="B37">Van Hasselt et al., 2016</xref>).</p>
<p>
<xref ref-type="fig" rid="F3">Figure 3</xref> illustrates the relationship between the number of sleeping GBSs and the rate requirement across three optimization scenarios: (i) optimizing achievable rate without considering EE, (ii) prioritizing EE while ignoring rate constraints, and (iii) jointly optimizing EE under the minimum-rate constraint. The results show that <inline-formula id="inf171">
<mml:math id="m217">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> strongly limits the number of GBSs that can enter sleep mode. In the first scenario, no GBS enters sleep mode due to strict rate-driven operation. In the second scenario, more GBSs can be deactivated, improving EE but violating user rate requirements. The third scenario balances both objectives and is effective in the range <inline-formula id="inf172">
<mml:math id="m218">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2208;</mml:mo>
<mml:mrow>
<mml:mo stretchy="false">[</mml:mo>
<mml:mrow>
<mml:mn>0.1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>0.6</mml:mn>
</mml:mrow>
<mml:mo stretchy="false">]</mml:mo>
</mml:mrow>
</mml:mrow>
</mml:math>
</inline-formula> bps/Hz. When <inline-formula id="inf173">
<mml:math id="m219">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0.6</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> bps/Hz, up to seven GBSs can be switched off, while at <inline-formula id="inf174">
<mml:math id="m220">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0.1</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> bps/Hz, only one GBS is switched off. For <inline-formula id="inf175">
<mml:math id="m221">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3e;</mml:mo>
<mml:mn>0.6</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> bps/Hz, the method cannot place GBSs into sleep mode without violating the QoS constraint.</p>
<fig id="F3" position="float">
<label>FIGURE 3</label>
<caption>
<p>The number of sleeping GBSs as a function of rate requirement <inline-formula id="inf176">
<mml:math id="m222">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</caption>
<graphic xlink:href="frcmn-06-1764320-g003.tif">
<alt-text content-type="machine-generated">Line graph showing the number of Base Stations (BSs) in sleep mode versus minimum rate (Mbps) from 0.1 to 0.8. Two data series are plotted: &#x22;Energy Minimization Criterion&#x22; with red squares, declining sharply, and &#x22;Energy Efficiency Criterion&#x22; with blue circles, following a similar path. A dashed line represents the &#x22;Rate Criterion&#x22; at the bottom, indicating zero BSs in sleep mode for all rates.</alt-text>
</graphic>
</fig>
<p>It can be observed from <xref ref-type="fig" rid="F3">Figure 3</xref> that when the minimum rate requirement exceeds <inline-formula id="inf177">
<mml:math id="m223">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0.6</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula> bps/Hz, the number of GBSs entering sleep mode drops to zero. This outcome highlights the trade-off between stringent QoS constraints and achievable energy savings. Under high QoS requirements, maintaining user data rates necessitates keeping all GBSs active, which limits the applicability of sleep strategies. Consequently, the proposed ABS-assisted framework is most effective in low-to-medium QoS regimes, such as off-peak or lightly loaded traffic conditions, where energy efficiency gains can be achieved without compromising user performance.</p>
<p>As illustrated in <xref ref-type="fig" rid="F4">Figure 4</xref>, the energy efficiency criterion is expressed in terms of training episodes. To evaluate the role of the ABS in the proposed system, three scenarios were considered: one with the ABS at a higher altitude, another at a lower altitude, and a scenario without an ABS. In this context, <inline-formula id="inf178">
<mml:math id="m224">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> was set to 0.2 bps/Hz based on the results in <xref ref-type="fig" rid="F3">Figure 3</xref>. The results indicate that the learning process improves energy efficiency over training as the agent refines sleep scheduling, power allocation, and ABS positioning decisions. <xref ref-type="fig" rid="F5">Figure 5</xref> presents the energy efficiency criterion as a function of the minimum rate requirement. The observations indicate that, with an average <inline-formula id="inf179">
<mml:math id="m225">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula> of 0.4, the proposed methods employing low-altitude and high-altitude ABSs can improve the energy efficiency criterion by <inline-formula id="inf180">
<mml:math id="m226">
<mml:mrow>
<mml:mn>30</mml:mn>
<mml:mi>%</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula> and <inline-formula id="inf181">
<mml:math id="m227">
<mml:mrow>
<mml:mn>25</mml:mn>
<mml:mi>%</mml:mi>
</mml:mrow>
</mml:math>
</inline-formula>, respectively, compared to the scenario without an ABS.</p>
<fig id="F4" position="float">
<label>FIGURE 4</label>
<caption>
<p>Energy efficiency criterion as a function of the number of training episodes for <inline-formula id="inf182">
<mml:math id="m228">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x3d;</mml:mo>
<mml:mn>0.2</mml:mn>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</caption>
<graphic xlink:href="frcmn-06-1764320-g004.tif">
<alt-text content-type="machine-generated">Line graph showing energy efficiency (Mbps/J) over episodes. Three lines represent different scenarios: UAV at low height (black squares), UAV at high height (blue circles), and without UAV (green diamonds). UAV at low height shows the highest efficiency, followed by UAV at high height, with a gradual increase. Without UAV has the lowest efficiency.</alt-text>
</graphic>
</fig>
<fig id="F5" position="float">
<label>FIGURE 5</label>
<caption>
<p>Energy efficiency criterion as a function of rate requirement <inline-formula id="inf183">
<mml:math id="m229">
<mml:mrow>
<mml:msub>
<mml:mrow>
<mml:mi>R</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>min</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:math>
</inline-formula>.</p>
</caption>
<graphic xlink:href="frcmn-06-1764320-g005.tif">
<alt-text content-type="machine-generated">Line graph showing energy efficiency versus data rate for three scenarios: UAV at low height (red squares), UAV at high height (blue circles), and without UAV (green diamonds). Energy efficiency decreases as data rate increases from 0.1 to 0.6 megabits per second.</alt-text>
</graphic>
</fig>
</sec>
<sec sec-type="conclusion" id="s5">
<label>5</label>
<title>Conclusion</title>
<p>This paper presents an ABS-assisted energy optimization framework for beyond-5G (B5G) cellular networks, utilizing selective ground base station (GBS) sleep modes and traffic offloading through aerial base stations (ABSs). To address the dynamic and non-convex nature of the problem, a hybrid reinforcement learning algorithm combining Deep Deterministic Policy Gradient (DDPG) and Double Deep Q-Learning (DDQL) is developed. This algorithm jointly optimizes ABS positioning, GBS sleep scheduling, and resource allocation. Simulation results demonstrate that the proposed framework significantly reduces overall network energy consumption while maintaining service quality.</p>
<p>These findings underscore the potential of hybrid deep reinforcement learning techniques in enabling intelligent and energy-efficient wireless communication systems. Future research directions include incorporating renewable-powered ABSs, implementing cooperative multi-ABS coordination, and developing real-time adaptive mechanisms to further enhance system scalability and performance.</p>
</sec>
</body>
<back>
<sec sec-type="data-availability" id="s6">
<title>Data availability statement</title>
<p>The original contributions presented in the study are included in the article/supplementary material, further inquiries can be directed to the corresponding author.</p>
</sec>
<sec sec-type="author-contributions" id="s7">
<title>Author contributions</title>
<p>VS: Writing &#x2013; review and editing, Writing &#x2013; original draft. ME: Writing &#x2013; original draft, Writing &#x2013; review and editing. KK: Writing &#x2013; review and editing, Writing &#x2013; original draft.</p>
</sec>
<sec sec-type="COI-statement" id="s9">
<title>Conflict of interest</title>
<p>The author(s) declared that this work was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec sec-type="ai-statement" id="s10">
<title>Generative AI statement</title>
<p>The author(s) declared that generative AI was used in the creation of this manuscript. Generative AI was used exclusively to edit and improve the clarity of the manuscript&#x2019;s text.</p>
<p>Any alternative text (alt text) provided alongside figures in this article has been generated by Frontiers with the support of artificial intelligence and reasonable efforts have been made to ensure accuracy, including review by the authors wherever possible. If you identify any issues, please contact us.</p>
</sec>
<sec sec-type="disclaimer" id="s11">
<title>Publisher&#x2019;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<fn-group>
<fn fn-type="custom" custom-type="edited-by">
<p>
<bold>Edited by:</bold> <ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/1320355/overview">Mehran Behjati</ext-link>, Sunway University, Malaysia</p>
</fn>
<fn fn-type="custom" custom-type="reviewed-by">
<p>
<bold>Reviewed by:</bold> <ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/3312159/overview">Mohammed Sani Adam</ext-link>, National University of Malaysia, Malaysia</p>
<p>
<ext-link ext-link-type="uri" xlink:href="https://loop.frontiersin.org/people/3314444/overview">Javad Haghighat</ext-link>, TED University, T&#xfc;rkiye</p>
</fn>
</fn-group>
<ref-list>
<title>References</title>
<ref id="B1">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Abubakar</surname>
<given-names>A. I.</given-names>
</name>
<name>
<surname>Ahmad</surname>
<given-names>I.</given-names>
</name>
<name>
<surname>Omeke</surname>
<given-names>K.</given-names>
</name>
<name>
<surname>Ozturk</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Ozturk</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Abdel-Salam</surname>
<given-names>A.</given-names>
</name>
<etal/>
</person-group> (<year>2023</year>). <article-title>A survey on energy optimization techniques in UAV-based cellular networks: from conventional to machine learning approaches</article-title>. <source>Drones</source> <volume>7</volume> (<issue>3</issue>), <fpage>214</fpage>. <pub-id pub-id-type="doi">10.3390/drones7030214</pub-id>
</mixed-citation>
</ref>
<ref id="B2">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Alsharif</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Nordin</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Abdullah</surname>
<given-names>N. F.</given-names>
</name>
<name>
<surname>Kelechi</surname>
<given-names>A. H.</given-names>
</name>
</person-group> (<year>2018</year>). <article-title>How to make key 5G wireless technologies environmental friendly: a review</article-title>. <source>Trans. Emerg. Telecommun. Technol.</source> <volume>29</volume> (<issue>1</issue>), <fpage>e3254</fpage>. <pub-id pub-id-type="doi">10.1002/ett.3254</pub-id>
</mixed-citation>
</ref>
<ref id="B3">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Amine</surname>
<given-names>A. E.</given-names>
</name>
<name>
<surname>Chaiban</surname>
<given-names>J. P.</given-names>
</name>
<name>
<surname>Hassan</surname>
<given-names>H. A. H.</given-names>
</name>
<name>
<surname>Dini</surname>
<given-names>P.</given-names>
</name>
<name>
<surname>Nuaymi</surname>
<given-names>L.</given-names>
</name>
<name>
<surname>Achkar</surname>
<given-names>R.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Energy optimization with multi-sleeping control in 5G heterogeneous networks using reinforcement learning</article-title>. <source>IEEE Trans. Netw. Serv. Manag.</source> <volume>19</volume> (<issue>4</issue>), <fpage>4310</fpage>&#x2013;<lpage>4322</lpage>. <pub-id pub-id-type="doi">10.1109/tnsm.2022.3157650</pub-id>
</mixed-citation>
</ref>
<ref id="B4">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Amponis</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>Lagkas</surname>
<given-names>T.</given-names>
</name>
<name>
<surname>Zevgara</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Katsikas</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>Xirofotos</surname>
<given-names>T.</given-names>
</name>
<name>
<surname>Moscholios</surname>
<given-names>I.</given-names>
</name>
<etal/>
</person-group> (<year>2022</year>). <article-title>Drones in B5G/6G networks as flying base stations</article-title>. <source>Drones</source> <volume>6</volume> (<issue>2</issue>), <fpage>39</fpage>. <pub-id pub-id-type="doi">10.3390/drones6020039</pub-id>
</mixed-citation>
</ref>
<ref id="B5">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Azadur</surname>
<given-names>R.Md.</given-names>
</name>
<name>
<surname>Pawase</surname>
<given-names>C. J.</given-names>
</name>
<name>
<surname>Chang</surname>
<given-names>K.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>Multi-UAV path planning utilizing the PGA algorithm for terrestrial IoT sensor network under ISAC framework</article-title>. <source>Trans. Emerg. Telecommun. Technol.</source> <volume>35</volume> (<issue>1</issue>), <fpage>e4916</fpage>. <pub-id pub-id-type="doi">10.1002/ett.4916</pub-id>
</mixed-citation>
</ref>
<ref id="B6">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Azarhava</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Abdollahi</surname>
<given-names>M. P.</given-names>
</name>
<name>
<surname>Musevi Niya</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>Placement and power assignment for hierarchical UAV networks under hovering fluctuations in mmWave communications</article-title>. <source>Trans. Emerg. Telecommun. Technol.</source> <volume>35</volume> (<issue>11</issue>), <fpage>e70002</fpage>. <pub-id pub-id-type="doi">10.1002/ett.70002</pub-id>
</mixed-citation>
</ref>
<ref id="B7">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Banafaa</surname>
<given-names>M. K.</given-names>
</name>
<name>
<surname>Pepeo&#x11f;lu</surname>
<given-names>&#xd6;.</given-names>
</name>
<name>
<surname>Shayea</surname>
<given-names>I.</given-names>
</name>
<name>
<surname>Alhammadi</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Shamsan</surname>
<given-names>Z. A.</given-names>
</name>
<name>
<surname>Razaz</surname>
<given-names>M. A.</given-names>
</name>
<etal/>
</person-group> (<year>2024</year>). <article-title>A comprehensive survey on 5G-and-Beyond networks with UAVs: applications, emerging technologies, regulatory aspects, research trends and challenges</article-title>. <source>IEEE Access</source> <volume>12</volume>, <fpage>7786</fpage>&#x2013;<lpage>7826</lpage>. <pub-id pub-id-type="doi">10.1109/access.2023.3349208</pub-id>
</mixed-citation>
</ref>
<ref id="B8">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Basharat</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Naeem</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Qadir</surname>
<given-names>Z.</given-names>
</name>
<name>
<surname>Anpalagan</surname>
<given-names>A.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Resource optimization in UAV-Assisted wireless networks&#x2014;A comprehensive survey</article-title>. <source>Trans. Emerg. Telecommun. Technol.</source> <volume>33</volume> (<issue>7</issue>), <fpage>e4464</fpage>. <pub-id pub-id-type="doi">10.1002/ett.4464</pub-id>
</mixed-citation>
</ref>
<ref id="B9">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Behjati</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Alobaidy</surname>
<given-names>H. A. H.</given-names>
</name>
<name>
<surname>Nordin</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Abdullah</surname>
<given-names>N. F.</given-names>
</name>
</person-group> (<year>2025</year>). <article-title>UAV-assisted federated learning with hybrid LoRa P2P/LoRaWAN for sustainable biosphere</article-title>. <source>Front. Commun. Netw.</source> <volume>6</volume>, <fpage>1529453</fpage>. <pub-id pub-id-type="doi">10.3389/frcmn.2025.1529453</pub-id>
</mixed-citation>
</ref>
<ref id="B10">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Braga</surname>
<given-names>I. M.</given-names>
</name>
<name>
<surname>Cavalcante</surname>
<given-names>E. d. O.</given-names>
</name>
<name>
<surname>Fodor</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>Silva</surname>
<given-names>Y. C. B.</given-names>
</name>
<name>
<surname>e Silva</surname>
<given-names>C. F. M.</given-names>
</name>
<name>
<surname>Freitas</surname>
<given-names>W. C.</given-names>
</name>
</person-group> (<year>2020</year>). <article-title>User scheduling based on multi-agent deep Q-learning for robust beamforming in multicell MISO systems</article-title>. <source>IEEE Commun. Lett.</source> <volume>24</volume> (<issue>12</issue>), <fpage>2809</fpage>&#x2013;<lpage>2813</lpage>. <pub-id pub-id-type="doi">10.1109/lcomm.2020.3015462</pub-id>
</mixed-citation>
</ref>
<ref id="B11">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Chowdary</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Ramamoorthi</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Kumar</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Cenkeramaddi</surname>
<given-names>L. R.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Joint resource allocation and UAV scheduling with ground radio station sleeping</article-title>. <source>IEEE Access</source> <volume>9</volume>, <fpage>124505</fpage>&#x2013;<lpage>124518</lpage>. <pub-id pub-id-type="doi">10.1109/access.2021.3111087</pub-id>
</mixed-citation>
</ref>
<ref id="B12">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Dogra</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Jha</surname>
<given-names>R. K.</given-names>
</name>
<name>
<surname>Jain</surname>
<given-names>S.</given-names>
</name>
</person-group> (<year>2020</year>). <article-title>A survey on beyond 5G network with the advent of 6G: architecture and emerging technologies</article-title>. <source>IEEE Access</source> <volume>9</volume>, <fpage>67512</fpage>&#x2013;<lpage>67547</lpage>. <pub-id pub-id-type="doi">10.1109/access.2020.3031234</pub-id>
</mixed-citation>
</ref>
<ref id="B13">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Elnabty</surname>
<given-names>I. A.</given-names>
</name>
<name>
<surname>Fahmy</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Kafafy</surname>
<given-names>M.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>A survey on UAV placement optimization for UAV-Assisted communication in 5G and beyond networks</article-title>. <source>Phys. Commun.</source> <volume>51</volume>, <fpage>101564</fpage>. <pub-id pub-id-type="doi">10.1016/j.phycom.2021.101564</pub-id>
</mixed-citation>
</ref>
<ref id="B14">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>F&#xe4;hrmann</surname>
<given-names>D.</given-names>
</name>
<name>
<surname>Jorek</surname>
<given-names>N.</given-names>
</name>
<name>
<surname>Damer</surname>
<given-names>N.</given-names>
</name>
<name>
<surname>Kirchbuchner</surname>
<given-names>F.</given-names>
</name>
<name>
<surname>Kuijper</surname>
<given-names>A.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Double deep Q-Learning with prioritized experience replay for anomaly detection in smart environments</article-title>. <source>IEEE Access</source> <volume>10</volume>, <fpage>60836</fpage>&#x2013;<lpage>60848</lpage>. <pub-id pub-id-type="doi">10.1109/access.2022.3179720</pub-id>
</mixed-citation>
</ref>
<ref id="B15">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Geraci</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>Garcia-Rodriguez</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Azari</surname>
<given-names>M. M.</given-names>
</name>
<name>
<surname>Lozano</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Mezzavilla</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Chatzinotas</surname>
<given-names>S.</given-names>
</name>
<etal/>
</person-group> (<year>2022</year>). <article-title>What will the future of UAV cellular communications be? A flight from 5G to 6G</article-title>. <source>IEEE Commun. Surv. Tutor.</source> <volume>24</volume> (<issue>3</issue>), <fpage>1304</fpage>&#x2013;<lpage>1335</lpage>. <pub-id pub-id-type="doi">10.1109/comst.2022.3171135</pub-id>
</mixed-citation>
</ref>
<ref id="B16">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ghorbel</surname>
<given-names>M. B.</given-names>
</name>
<name>
<surname>Rodriguez-Duarte</surname>
<given-names>D.</given-names>
</name>
<name>
<surname>Ghazzai</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Hossain</surname>
<given-names>M. J.</given-names>
</name>
<name>
<surname>Menouar</surname>
<given-names>H.</given-names>
</name>
</person-group> (<year>2019</year>). <article-title>Joint position and travel path optimization for energy efficient wireless data gathering using unmanned aerial vehicles</article-title>. <source>IEEE Trans. Veh. Technol.</source> <volume>68</volume> (<issue>3</issue>), <fpage>2165</fpage>&#x2013;<lpage>2175</lpage>. <pub-id pub-id-type="doi">10.1109/tvt.2019.2893374</pub-id>
</mixed-citation>
</ref>
<ref id="B17">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Gryech</surname>
<given-names>I.</given-names>
</name>
<name>
<surname>Vinogradov</surname>
<given-names>E.</given-names>
</name>
<name>
<surname>Saboor</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Bithas</surname>
<given-names>P. S.</given-names>
</name>
<name>
<surname>Mathiopoulos</surname>
<given-names>P. T.</given-names>
</name>
<name>
<surname>Pollin</surname>
<given-names>S.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>A systematic literature review on the role of UAV-enabled communications in advancing the UN&#x2019;s sustainable development goals</article-title>. <source>Front. Commun. Netw.</source> <volume>5</volume>, <fpage>1286073</fpage>. <pub-id pub-id-type="doi">10.3389/frcmn.2024.1286073</pub-id>
</mixed-citation>
</ref>
<ref id="B18">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Gu</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>G.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>A survey on UAV-assisted wireless communications: recent advances and future trends</article-title>. <source>Comput. Commun.</source> <volume>208</volume>, <fpage>44</fpage>&#x2013;<lpage>78</lpage>. <pub-id pub-id-type="doi">10.1016/j.comcom.2023.05.013</pub-id>
</mixed-citation>
</ref>
<ref id="B19">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Huang</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Xu</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Hua</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>Z.</given-names>
</name>
</person-group> (<year>2019</year>). <article-title>An overview of intelligent wireless communications using deep reinforcement learning</article-title>. <source>J. Commun. Inf. Netw.</source> <volume>4</volume> (<issue>2</issue>), <fpage>15</fpage>&#x2013;<lpage>29</lpage>. <pub-id pub-id-type="doi">10.23919/jcin.2019.8917869</pub-id>
</mixed-citation>
</ref>
<ref id="B20">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Jangsher</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Al-Jarrah</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Al-Dweik</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Alsusa</surname>
<given-names>E.</given-names>
</name>
<name>
<surname>Kong</surname>
<given-names>P. Y.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Energy constrained sum-rate maximization in IRS-assisted UAV networks with imperfect channel information</article-title>. <source>IEEE Trans. Aerosp. Electron. Syst.</source> <volume>59</volume> (<issue>3</issue>), <fpage>2898</fpage>&#x2013;<lpage>2908</lpage>. <pub-id pub-id-type="doi">10.1109/taes.2022.3220493</pub-id>
</mixed-citation>
</ref>
<ref id="B21">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ju</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Kim</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Kim</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Shim</surname>
<given-names>B.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Energy-efficient ultra-dense network with deep reinforcement learning</article-title>. <source>IEEE Trans. Wirel. Commun.</source> <volume>21</volume> (<issue>8</issue>), <fpage>6539</fpage>&#x2013;<lpage>6552</lpage>. <pub-id pub-id-type="doi">10.1109/twc.2022.3150425</pub-id>
</mixed-citation>
</ref>
<ref id="B22">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Khawaja</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Guvenc</surname>
<given-names>I.</given-names>
</name>
<name>
<surname>Matolak</surname>
<given-names>D. W.</given-names>
</name>
<name>
<surname>Fiebig</surname>
<given-names>U. C.</given-names>
</name>
<name>
<surname>Schneckenburger</surname>
<given-names>N.</given-names>
</name>
</person-group> (<year>2019</year>). <article-title>A survey of air-to-ground propagation channel modeling for unmanned aerial vehicles</article-title>. <source>IEEE Commun. Surv. Tutor.</source> <volume>21</volume> (<issue>3</issue>), <fpage>2361</fpage>&#x2013;<lpage>2391</lpage>. <pub-id pub-id-type="doi">10.1109/comst.2019.2915069</pub-id>
</mixed-citation>
</ref>
<ref id="B23">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kim</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Jeon</surname>
<given-names>W. S.</given-names>
</name>
<name>
<surname>Jeong</surname>
<given-names>D. G.</given-names>
</name>
</person-group> (<year>2015</year>). <article-title>Base-station sleep management in open-access femtocell networks</article-title>. <source>IEEE Trans. Veh. Technol.</source> <volume>65</volume> (<issue>5</issue>), <fpage>3786</fpage>&#x2013;<lpage>3791</lpage>. <pub-id pub-id-type="doi">10.1109/tvt.2015.2445922</pub-id>
</mixed-citation>
</ref>
<ref id="B24">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kim</surname>
<given-names>T.</given-names>
</name>
<name>
<surname>Lee</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Choi</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Park</surname>
<given-names>H. S.</given-names>
</name>
<name>
<surname>Choi</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>An energy-efficient multi-level sleep strategy for periodic uplink transmission in industrial private 5G networks</article-title>. <source>Sensors</source> <volume>23</volume>, <fpage>9070</fpage>. <pub-id pub-id-type="doi">10.3390/s23229070</pub-id>
<pub-id pub-id-type="pmid">38005465</pub-id>
</mixed-citation>
</ref>
<ref id="B25">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kooshki</surname>
<given-names>F.</given-names>
</name>
<name>
<surname>Armada</surname>
<given-names>A. G.</given-names>
</name>
<name>
<surname>Mowla</surname>
<given-names>M. M.</given-names>
</name>
<name>
<surname>Flizikowski</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Pietrzyk</surname>
<given-names>S.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Energy-efficient sleep mode schemes for cell-less RAN in 5G and beyond 5G networks</article-title>. <source>IEEE Access</source> <volume>11</volume>, <fpage>1432</fpage>&#x2013;<lpage>1444</lpage>. <pub-id pub-id-type="doi">10.1109/access.2022.3233430</pub-id>
</mixed-citation>
</ref>
<ref id="B26">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>L&#xf3;pez-P&#xe9;rez</surname>
<given-names>D.</given-names>
</name>
<name>
<surname>De Domenico</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Piovesan</surname>
<given-names>N.</given-names>
</name>
<name>
<surname>Xinli</surname>
<given-names>G.</given-names>
</name>
<name>
<surname>Bao</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Qitao</surname>
<given-names>S.</given-names>
</name>
<etal/>
</person-group> (<year>2022</year>). <article-title>A survey on 5G radio access network energy efficiency: massive MIMO, lean carrier design, sleep modes, and machine learning</article-title>. <source>IEEE Commun. Surv. Tutor.</source> <volume>24</volume> (<issue>1</issue>), <fpage>653</fpage>&#x2013;<lpage>697</lpage>. <pub-id pub-id-type="doi">10.1109/comst.2022.3142532</pub-id>
</mixed-citation>
</ref>
<ref id="B27">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Masroor</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Naeem</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Ejaz</surname>
<given-names>W.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Resource management in UAV-assisted wireless networks: an optimization perspective</article-title>. <source>Ad Hoc Netw.</source> <volume>121</volume>, <fpage>102596</fpage>. <pub-id pub-id-type="doi">10.1016/j.adhoc.2021.102596</pub-id>
</mixed-citation>
</ref>
<ref id="B28">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Morocho-Cayamcela</surname>
<given-names>M. E.</given-names>
</name>
<name>
<surname>Lee</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Lim</surname>
<given-names>W.</given-names>
</name>
</person-group> (<year>2019</year>). <article-title>Machine learning for 5G/B5G Mobile and wireless communications: potential, limitations, and future directions</article-title>. <source>IEEE Access</source> <volume>7</volume>, <fpage>137184</fpage>&#x2013;<lpage>137206</lpage>. <pub-id pub-id-type="doi">10.1109/access.2019.2942390</pub-id>
</mixed-citation>
</ref>
<ref id="B29">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Puspitasari</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>An</surname>
<given-names>T. T.</given-names>
</name>
<name>
<surname>Alsharif</surname>
<given-names>M. H.</given-names>
</name>
<name>
<surname>Lee</surname>
<given-names>B. M.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Emerging technologies for 6G communication networks: machine learning approaches</article-title>. <source>Sensors</source> <volume>23</volume>, <fpage>7709</fpage>. <pub-id pub-id-type="doi">10.3390/s23187709</pub-id>
<pub-id pub-id-type="pmid">37765765</pub-id>
</mixed-citation>
</ref>
<ref id="B30">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Qazzaz</surname>
<given-names>M. M. H.</given-names>
</name>
<name>
<surname>Zaidi</surname>
<given-names>S. A.</given-names>
</name>
<name>
<surname>McLernon</surname>
<given-names>D. C.</given-names>
</name>
<name>
<surname>Hayajneh</surname>
<given-names>A. M.</given-names>
</name>
<name>
<surname>Salama</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Aldalahmeh</surname>
<given-names>S. A.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>Non-terrestrial UAV clients for beyond 5G networks: a comprehensive survey</article-title>. <source>Ad Hoc Netw.</source> <volume>157</volume>, <fpage>103440</fpage>. <pub-id pub-id-type="doi">10.1016/j.adhoc.2024.103440</pub-id>
</mixed-citation>
</ref>
<ref id="B31">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Sarkar</surname>
<given-names>N. I.</given-names>
</name>
<name>
<surname>Gul</surname>
<given-names>S.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>Artificial intelligence-based autonomous UAV networks: a survey</article-title>. <source>Drones</source> <volume>7</volume> (<issue>5</issue>), <fpage>322</fpage>. <pub-id pub-id-type="doi">10.3390/drones7050322</pub-id>
</mixed-citation>
</ref>
<ref id="B32">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Shahzadi</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Ali</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Khan</surname>
<given-names>H. Z.</given-names>
</name>
<name>
<surname>Naeem</surname>
<given-names>M.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>UAV assisted 5G and beyond wireless networks: a survey</article-title>. <source>J. Netw. Comput. Appl.</source> <volume>189</volume>, <fpage>103114</fpage>. <pub-id pub-id-type="doi">10.1016/j.jnca.2021.103114</pub-id>
</mixed-citation>
</ref>
<ref id="B33">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Shokrnezhad</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Taleb</surname>
<given-names>T.</given-names>
</name>
<name>
<surname>Dazzi</surname>
<given-names>P.</given-names>
</name>
</person-group> (<year>2024</year>). <article-title>Double deep Q-Learning-Based path selection and service placement for latency-sensitive beyond 5G applications</article-title>. <source>IEEE Trans. Mob. Comput.</source> <volume>23</volume> (<issue>5</issue>), <fpage>5097</fpage>&#x2013;<lpage>5110</lpage>. <pub-id pub-id-type="doi">10.1109/tmc.2023.3301506</pub-id>
</mixed-citation>
</ref>
<ref id="B34">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Singh</surname>
<given-names>Y.</given-names>
</name>
</person-group> (<year>2012</year>). <article-title>Comparison of okumura, hata and COST-231 models on the basis of path loss and signal strength</article-title>. <source>Int. J. Comput. Appl.</source> <volume>59</volume> (<issue>11</issue>), <fpage>37</fpage>&#x2013;<lpage>41</lpage>. <pub-id pub-id-type="doi">10.5120/9594-4216</pub-id>
</mixed-citation>
</ref>
<ref id="B35">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Sufyan</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Khan</surname>
<given-names>K. B.</given-names>
</name>
<name>
<surname>Khashan</surname>
<given-names>O. A.</given-names>
</name>
<name>
<surname>Mir</surname>
<given-names>T.</given-names>
</name>
<name>
<surname>Mir</surname>
<given-names>U.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>From 5G to beyond 5G: a comprehensive survey of wireless network evolution, challenges, and promising technologies</article-title>. <source>Electronics</source> <volume>12</volume>, <fpage>2200</fpage>. <pub-id pub-id-type="doi">10.3390/electronics12102200</pub-id>
</mixed-citation>
</ref>
<ref id="B36">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Tung</surname>
<given-names>T. V.</given-names>
</name>
<name>
<surname>An</surname>
<given-names>T. T.</given-names>
</name>
<name>
<surname>Lee</surname>
<given-names>B. M.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Joint resource and trajectory optimization for energy efficiency maximization in UAV-based networks</article-title>. <source>Mathematics</source> <volume>10</volume> (<issue>20</issue>), <fpage>3840</fpage>. <pub-id pub-id-type="doi">10.3390/math10203840</pub-id>
</mixed-citation>
</ref>
<ref id="B37">
<mixed-citation publication-type="book">
<person-group person-group-type="author">
<name>
<surname>Van Hasselt</surname>
<given-names>H.</given-names>
</name>
<name>
<surname>Guez</surname>
<given-names>A.</given-names>
</name>
<name>
<surname>Silver</surname>
<given-names>D.</given-names>
</name>
</person-group> (<year>2016</year>). &#x201c;<article-title>Deep reinforcement learning with double Q-learning</article-title>,&#x201d; in <source>Proc. AAAI Conf. Artif. Intell</source>.</mixed-citation>
</ref>
<ref id="B38">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Won</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Kim</surname>
<given-names>D. Y.</given-names>
</name>
<name>
<surname>Park</surname>
<given-names>Y. I.</given-names>
</name>
<name>
<surname>Lee</surname>
<given-names>J. W.</given-names>
</name>
</person-group> (<year>2023</year>). <article-title>A survey on UAV placement and trajectory optimization in communication networks: from the perspective of air-to-ground channel models</article-title>. <source>ICT Express</source> <volume>9</volume> (<issue>3</issue>), <fpage>385</fpage>&#x2013;<lpage>397</lpage>. <pub-id pub-id-type="doi">10.1016/j.icte.2022.01.015</pub-id>
</mixed-citation>
</ref>
<ref id="B39">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Wu</surname>
<given-names>W.</given-names>
</name>
<name>
<surname>Sun</surname>
<given-names>S.</given-names>
</name>
<name>
<surname>Shan</surname>
<given-names>F.</given-names>
</name>
<name>
<surname>Yang</surname>
<given-names>M.</given-names>
</name>
<name>
<surname>Luo</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Energy-constrained UAV flight scheduling for IoT data collection with 60 GHz communication</article-title>. <source>IEEE Trans. Veh. Technol.</source> <volume>71</volume> (<issue>10</issue>), <fpage>10991</fpage>&#x2013;<lpage>11005</lpage>. <pub-id pub-id-type="doi">10.1109/tvt.2022.3184869</pub-id>
</mixed-citation>
</ref>
<ref id="B40">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Yu</surname>
<given-names>Y.</given-names>
</name>
<name>
<surname>Tang</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Huang</surname>
<given-names>J.</given-names>
</name>
<name>
<surname>Zhang</surname>
<given-names>X.</given-names>
</name>
<name>
<surname>So</surname>
<given-names>D. K. C.</given-names>
</name>
<name>
<surname>Wong</surname>
<given-names>K. K.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>Multi-objective optimization for UAV-assisted wireless powered IoT networks based on extended DDPG algorithm</article-title>. <source>IEEE Trans. Commun.</source> <volume>69</volume> (<issue>9</issue>), <fpage>6361</fpage>&#x2013;<lpage>6374</lpage>. <pub-id pub-id-type="doi">10.1109/tcomm.2021.3089476</pub-id>
</mixed-citation>
</ref>
<ref id="B41">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Zhou</surname>
<given-names>Q.</given-names>
</name>
<name>
<surname>Guo</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Wang</surname>
<given-names>C.</given-names>
</name>
<name>
<surname>Cui</surname>
<given-names>L.</given-names>
</name>
</person-group> (<year>2022</year>). <article-title>Radio resource management for C-V2X using graph matching and actor&#x2013;critic learning</article-title>. <source>IEEE Wirel. Commun. Lett.</source> <volume>11</volume> (<issue>12</issue>), <fpage>2645</fpage>&#x2013;<lpage>2649</lpage>. <pub-id pub-id-type="doi">10.1109/lwc.2022.3213176</pub-id>
</mixed-citation>
</ref>
<ref id="B42">
<mixed-citation publication-type="journal">
<person-group person-group-type="author">
<name>
<surname>Zhu</surname>
<given-names>B.</given-names>
</name>
<name>
<surname>Bedeer</surname>
<given-names>E.</given-names>
</name>
<name>
<surname>Nguyen</surname>
<given-names>H. H.</given-names>
</name>
<name>
<surname>Barton</surname>
<given-names>R.</given-names>
</name>
<name>
<surname>Henry</surname>
<given-names>J.</given-names>
</name>
</person-group> (<year>2021</year>). <article-title>UAV trajectory planning in wireless sensor networks for energy consumption minimization by deep reinforcement learning</article-title>. <source>IEEE Trans. Veh. Technol.</source> <volume>70</volume> (<issue>9</issue>), <fpage>9540</fpage>&#x2013;<lpage>9554</lpage>. <pub-id pub-id-type="doi">10.1109/tvt.2021.3102161</pub-id>
</mixed-citation>
</ref>
</ref-list>
</back>
</article>