<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<?covid-19-tdm?>
<article xml:lang="EN" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="brief-report">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Digit. Health</journal-id>
<journal-title>Frontiers in Digital Health</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Digit. Health</abbrev-journal-title>
<issn pub-type="epub">2673-253X</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fdgth.2021.745674</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Digital Health</subject>
<subj-group>
<subject>Perspective</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Vaccine Development in the Time of COVID-19: The Relevance of the Risklick AI to Assist in Risk Assessment and Optimize Performance</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name><surname>Haas</surname> <given-names>Quentin</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/535876/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Borisov</surname> <given-names>Nikolay</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1483904/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Alvarez</surname> <given-names>David Vicente</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Ferdowsi</surname> <given-names>Sohrab</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>von Meyenn</surname> <given-names>Leonhard</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1480519/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Teodoro</surname> <given-names>Douglas</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1291198/overview"/>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name><surname>Amini</surname> <given-names>Poorya</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="corresp" rid="c001"><sup>&#x0002A;</sup></xref>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>Risklick AG, Spin-off University of Bern</institution>, <addr-line>Bern</addr-line>, <country>Switzerland</country></aff>
<aff id="aff2"><sup>2</sup><institution>Clinical Trial Unit Bern, University of Bern</institution>, <addr-line>Bern</addr-line>, <country>Switzerland</country></aff>
<aff id="aff3"><sup>3</sup><institution>HES-SO University of Applied Sciences and Arts Western Switzerland</institution>, <addr-line>Geneva</addr-line>, <country>Switzerland</country></aff>
<aff id="aff4"><sup>4</sup><institution>Department of Radiology and Medical Informatics, University of Geneva</institution>, <addr-line>Geneva</addr-line>, <country>Switzerland</country></aff>
<author-notes>
<fn fn-type="edited-by"><p>Edited by: Giovanni Ferrara, University of Alberta, Canada</p></fn>
<fn fn-type="edited-by"><p>Reviewed by: Petar Radanliev, University of Oxford, United Kingdom; Farid Rahimi, Australian National University, Australia</p></fn>
<corresp id="c001">&#x0002A;Correspondence: Poorya Amini <email>poorya.amini&#x00040;ctu.unibe.ch</email></corresp>
<fn fn-type="other" id="fn001"><p>This article was submitted to Health Technology Innovation, a section of the journal Frontiers in Digital Health</p></fn></author-notes>
<pub-date pub-type="epub">
<day>02</day>
<month>11</month>
<year>2021</year>
</pub-date>
<pub-date pub-type="collection">
<year>2021</year>
</pub-date>
<volume>3</volume>
<elocation-id>745674</elocation-id>
<history>
<date date-type="received">
<day>22</day>
<month>07</month>
<year>2021</year>
</date>
<date date-type="accepted">
<day>24</day>
<month>09</month>
<year>2021</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x000A9; 2021 Haas, Borisov, Alvarez, Ferdowsi, von Meyenn, Teodoro and Amini.</copyright-statement>
<copyright-year>2021</copyright-year>
<copyright-holder>Haas, Borisov, Alvarez, Ferdowsi, von Meyenn, Teodoro and Amini</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p></license> </permissions>
<abstract><p>The 2019 coronavirus (COVID-19) pandemic revealed the urgent need for the acceleration of vaccine development worldwide. Rapid vaccine development poses numerous risks for each category of vaccine technology. By using the Risklick artificial intelligence (AI), we estimated the risks associated with all types of COVID-19 vaccine during the early phase of vaccine development. We then performed a postmortem analysis of the probability and the impact matrix calculations by comparing the 2020 prognosis to the contemporary situation. We used the Risklick AI to evaluate the risks and their incidence associated with vaccine development in the early stage of the COVID-19 pandemic. Our analysis revealed the diversity of risks among vaccine technologies currently used by pharmaceutical companies providing vaccines. This analysis highlighted the current and future potential pitfalls connected to vaccine production during the COVID-19 pandemic. Hence, the Risklick AI appears as an essential tool in vaccine development for the treatment of COVID-19 in order to formally anticipate the risks, and increases the overall performance from the production to the distribution of the vaccines. The Risklick AI could, therefore, be extended to other fields of research and development and represent a novel opportunity in the calculation of production-associated risks.</p></abstract>
<kwd-group>
<kwd>artificial intelligence</kwd>
<kwd>pharmacology</kwd>
<kwd>COVID-19</kwd>
<kwd>vaccine</kwd>
<kwd>risk analysis</kwd>
</kwd-group>
<contract-sponsor id="cn001">Innosuisse - Schweizerische Agentur f&#x000FC;r Innovationsf&#x000F6;rderung<named-content content-type="fundref-id">10.13039/501100013348</named-content></contract-sponsor>
<counts>
<fig-count count="1"/>
<table-count count="1"/>
<equation-count count="0"/>
<ref-count count="29"/>
<page-count count="5"/>
<word-count count="3683"/>
</counts>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="s1">
<title>Introduction</title>
<p>The 2019 coronavirus (COVID-19) pandemic revealed an urgent need for the acceleration of vaccine development worldwide (<xref ref-type="bibr" rid="B1">1</xref>). Within months of the start of the COVID-19 pandemic, several pharmaceutical companies announced their objective to develop vaccines against COVID-19 by using different technologies (<xref ref-type="bibr" rid="B2">2</xref>&#x02013;<xref ref-type="bibr" rid="B4">4</xref>). Each of the technologies presents advantages and pitfalls regarding the development of efficient COVID-19 vaccines (<xref ref-type="bibr" rid="B5">5</xref>&#x02013;<xref ref-type="bibr" rid="B7">7</xref>). However, the situation requires that all the alternatives should be explored to shorten the course of the COVID-19 pandemic (<xref ref-type="bibr" rid="B8">8</xref>, <xref ref-type="bibr" rid="B9">9</xref>). An increased speed of development induces risks, which must be taken seriously (<xref ref-type="bibr" rid="B10">10</xref>, <xref ref-type="bibr" rid="B11">11</xref>). Hence, we analyzed the risks of the different technologies to produce the vaccines proposed by the pharmaceutical companies providing vaccines against COVID-19. This analysis was performed using scientific publications from early 2020 and utilizing the artificial intelligence (AI)-based search engine Risklick (<xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B13">13</xref>). By calculating the risk score using a Probability and Impact Matrix and a semi-automated AI interface, we estimated the major risks faced by the pharmaceutical companies providing the vaccines.</p>
<p>A year later, our risk analysis on COVID-19 vaccine development was compared to the contemporary situation. Overall, we observed that the AI-based analysis anticipated the shortage of production and distribution faced by the pharmaceutical companies providing the RNA vaccines. The AI-based analysis highlighted the other issues faced by the pharmaceutical companies providing the vaccines, such as the delayed arrival of vaccines based on technologies other than RNA. Each issue faced by the vaccine providers is, as expected in the analysis, connected to the technology of their vaccine. Hence, our analysis exposes how AI-based technologies could become essential to the future of treatment development and the anticipation of associated risks.</p></sec>
<sec sec-type="methods" id="s2">
<title>Methods</title>
<sec>
<title>Data</title>
<p>The Risklick AI collects and updates clinical trial data from a wide variety of sources such as the Clinical Trials Registr and datasets from the WHO each day. The metadata related to the publications are acquired from 14 international sources [BioRxiv; MedRxiv; Medline; Embase; Pubmed; Cinahl; Web of Science; Scopus; Cochrane; the International Clinical Trials Registry Platform (ICTRP); Dimensions; Living Evidence; Kaggle Cord-19 Dataset; and Google Scholar] allowing superior search performance relative to other acknowledged scientific publications search engines (<xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B13">13</xref>). The Risklick AI is able to find and process COVID-19 references more effectively in terms of precision, F1 score, and recall, compared to the baseline platforms. Additionally, this living evidence management tool processes data in an automated manner, allowing analysis to be performed in a short amount of time. In total, 34 publications providing risk analysis on different vaccine technologies were gathered on June 10, 2020 (<xref ref-type="supplementary-material" rid="SM1">Supplementary Figure 1</xref>). These publications were used to evaluate the different risks in the development of the different vaccine technologies against COVID-19. The Risklick AI is available at <ext-link ext-link-type="uri" xlink:href="http://www.risklick.ch">www.risklick.ch</ext-link>.</p></sec>
<sec>
<title>Selection of Variables</title>
<p>In this study, seven different types of vaccines were investigated: DNA and RNA based, inactivated virus, virus-like particle, live attenuated virus, protein subunit, and viral vector. The risks associated with vaccine production were then measured across seven categories: safety, cost of goods sold, manufacturing scalability, manufacturing process, shipping, and duration of immunity. Each category was defined in <xref ref-type="supplementary-material" rid="SM3">Supplementary Table 2</xref>.</p></sec>
<sec>
<title>Risk Analysis</title>
<p>Using the set of publications, the risk for each vaccine technology for all the individual categories of risk was measured by using the Risklick semi-automated interface allowing the Probability and Impact Matrix calculation (<xref ref-type="bibr" rid="B14">14</xref>, <xref ref-type="bibr" rid="B15">15</xref>). The probability of the risk happening (or &#x0201C;likelihood&#x0201D;) was categorized as &#x0201C;remote, unlikely, possible, likely, or certain&#x0201D; for each vaccine technology. These likelihood marks graded risk from lowest risk (remote) to highest risk (certain). Each of the grades was attributed a numeric value from one to five. Then, the potential impact of such a risk happening for each vaccine technology was categorized as &#x0201C;insignificant, minor, moderate, major, or critical.&#x0201D; These impact marks graded impact from lowest (insignificant) to highest (critical). Each of the grades was attributed a numeric value rising from one to five. The risk score of all the individual risk categories in each vaccine technology was calculated with the likelihood multiplied by the impact score. Visualization of risks per technology was performed by using radar charts, where all the risks were represented in the percentage of maximum risk per category based on the scores from <xref ref-type="table" rid="T1">Table 1</xref>. Risks were first analyzed on July 28, 2020, and compared with the situation a year later on May 22, 2021.</p>
<table-wrap position="float" id="T1">
<label>Table 1</label>
<caption><p>Risk score of the different categories of risk for each vaccine technology.</p></caption>
<table frame="hsides" rules="groups">
<thead><tr>
<th/>
<th valign="top" align="center"><bold>DNA-based</bold></th>
<th valign="top" align="center"><bold>RNA-based</bold></th>
<th valign="top" align="center"><bold>Inactivated virus</bold></th>
<th valign="top" align="center"><bold>Live attenuated virus</bold></th>
<th valign="top" align="center"><bold>Protein subunit</bold></th>
<th valign="top" align="center"><bold>Viral vector</bold></th>
<th valign="top" align="center"><bold>VLP</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">Safety</td>
<td valign="top" align="center">8</td>
<td valign="top" align="center">6</td>
<td valign="top" align="center">15</td>
<td valign="top" align="center">15</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">12</td>
<td valign="top" align="center">4</td>
</tr>
<tr>
<td valign="top" align="left">Cost of good sold</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">3</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">6</td>
<td valign="top" align="center">6</td>
<td valign="top" align="center">6</td>
</tr>
<tr>
<td valign="top" align="left">Manufacturing scalability</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">8</td>
<td valign="top" align="center">10</td>
<td valign="top" align="center">10</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">16</td>
</tr>
<tr>
<td valign="top" align="left">Manufacturing process</td>
<td valign="top" align="center">3</td>
<td valign="top" align="center">12</td>
<td valign="top" align="center">5</td>
<td valign="top" align="center">5</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">6</td>
</tr>
<tr>
<td valign="top" align="left">Time of availability</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">5</td>
<td valign="top" align="center">10</td>
<td valign="top" align="center">8</td>
<td valign="top" align="center">12</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">25</td>
</tr>
<tr>
<td valign="top" align="left">Shipping</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">10</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">3</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">9</td>
<td valign="top" align="center">4</td>
</tr>
<tr>
<td valign="top" align="left">Duration of immunity</td>
<td valign="top" align="center">25</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">2</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">12</td>
<td valign="top" align="center">12</td>
</tr>
</tbody>
</table>
</table-wrap></sec>
<sec>
<title>Validation</title>
<p>Verification and validation procedures were performed by two independent immunologists. All risk assessments, likelihood measurements, and impact estimations were analyzed and verified through non-automated input of the researchers.</p></sec></sec>
<sec sec-type="results" id="s3">
<title>Results</title>
<p>The COVID-19 pandemic revealed the necessity for accelerated vaccine development worldwide (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B7">7</xref>). In this critical context, we performed a risk analysis on the different vaccine technologies proposed by the manufacturers against COVID-19 in the early months of 2020 by using the AI-based search engine Risklick (<xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B13">13</xref>). We graded seven different risks across seven different vaccine technologies by using a Probability and Impact Matrix (as illustrated in <xref ref-type="supplementary-material" rid="SM1">Supplementary Figure 1</xref>, for the risks linked to safety for each vaccine). Then, the risk score of every risk category for each vaccine technology was calculated. Risk calculation was limited by the low number of available publications at the time. An ideal quantitative risk assessment would require more data to reduce the limitations and strengthen the conclusions of the analysis. This resulted in a table of risks for each individual vaccine technology (<xref ref-type="table" rid="T1">Table 1</xref>). In this table, we already noticed that each vaccine technology possessed strengths and weaknesses different from one another, indicating the potential critical issues for the development of future vaccines. To simplify the visualization of the risks, a resume of risks per technology was performed using radar charts (<xref ref-type="fig" rid="F1">Figure 1</xref>). To allow comparison, all technologies of the vaccines were compared to virus-like particle (VLP). This choice was made in the perspective of VLP being the least probable vaccine technology to effectively reach the market against COVID-19 in the near future. There are currently only four VLP-based COVID-19 vaccines in clinical trials and even in the optimistic cases, production could not reach more than 100 million doses before 2022 (<xref ref-type="bibr" rid="B16">16</xref>). This was confirmed in analysis with VLP presenting a particularly high risk for the time to availability and manufacturing scalability.</p>
<fig id="F1" position="float">
<label>Figure 1</label>
<caption><p>The radar charts of the seven different vaccine technologies illustrating the major risks estimated by risk score, expressed in percentage. Risks represented are safety, cost of goods sold, manufacturing scalability, manufacturing process, time of availability, shipping, and duration of immunity. Each technology is represented in blue while being compared to virus-like particle (VLP) in orange. Risk associated to <bold>(A)</bold> DNA-Based, <bold>(B)</bold> RNA-based, <bold>(C)</bold> inactivated virus, <bold>(D)</bold> live attenuated virus, <bold>(E)</bold> protein subunit, and <bold>(F)</bold> viral vector vaccines are presented. The most important risks point toward 100, while less important risks point toward 0.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fdgth-03-745674-g0001.tif"/>
</fig>
<p>In June 2020, we already observed that the pharmaceutical companies providing vaccines would face different issues. Taken separately, DNA-based vaccines have the highest risk of providing only short-term immunity (<xref ref-type="table" rid="T1">Table 1</xref>, <xref ref-type="fig" rid="F1">Figure 1</xref>). Shipping is a particular risk for the RNA-based vaccines and, to a lesser degree, viral vectors, which require extremely low temperatures. Inactivated viruses and live attenuated viruses raised concerns regarding their safety and the cost of goods sold. Protein subunits appear less risky across most categories, but the duration of immunity and time of availability still appeared as a considerable risk. Finally, the viral vectors presented serious risks connected to production, particularly for the categories of th manufacturing scalability and manufacturing process.</p>
<p>After 1 year, we compared our results to the contemporary situation (May 22, 2021). Presently, 18 different vaccines are approved in one or more countries (<xref ref-type="bibr" rid="B17">17</xref>). When selecting the vaccines with authorization in more than two countries, this number falls to 10 (three inactivated viruses, five virus vectors, and two RNA-based vectors).</p>
<p>First, we noticed that an extremely low number of the vaccines we classified as &#x0201C;high risky&#x0201D; managed to reach market authorization so far (these being inactivated virus, live attenuated virus, and VLP; <xref ref-type="table" rid="T1">Table 1</xref>). Moreover, based on the WHO database (<xref ref-type="bibr" rid="B18">18</xref>), the most widely represented technology in the clinical phase of the vaccine development is the protein subunit, which was also the least risky technology according to our analysis (<xref ref-type="fig" rid="F1">Figure 1</xref>). However, none of the vaccines using this technology have reached market authorization to date. In fact, one of the major risks of the protein subunit compared to the competitors was its relatively important time to availability (<xref ref-type="fig" rid="F1">Figure 1</xref>), which was much lower in the RNA-based vaccines and viral vectors. Hence, the major risk associated with the protein subunit vaccine might be the reason why vaccines based on such technology are slower to reach approval compared to vaccines based on the other technologies, despite the high number of clinical trials running using this technology at the time this article was written.</p>
<p>With respect to the other technologies, RNA-based vaccines were rapidly commercialized probably due to low cost and high scalability of production, which translate into an extremely low risk for the time to availability (<xref ref-type="supplementary-material" rid="SM1">Supplementary Figure 1</xref>). However, their high risk associated with shipping was verified when supply chains started suffering from storage temperature issues following market authorization (<xref ref-type="bibr" rid="B19">19</xref>). With respect to the viral vector technologies, the major risk was associated with manufacturing. This was demonstrated by the shortfall of the Oxford/AstraZeneca doses in Europe for 2021 (<xref ref-type="bibr" rid="B20">20</xref>) and Sputnik in Latin America (<xref ref-type="bibr" rid="B21">21</xref>). Hence, the major risk calculated in 2020 appeared to play a critical role in the deployment of the viral vector vaccines today as it did for the RNA vaccines. With respect to vaccine distribution, the early stage of vaccine distribution in 2021 highlighted difficulties with respect to the cold chain distribution process (<xref ref-type="bibr" rid="B22">22</xref>). This situation provoked the evident risks of supply chain failures and caused an important pressure on vaccine supply chain infrastructures (<xref ref-type="bibr" rid="B23">23</xref>, <xref ref-type="bibr" rid="B24">24</xref>). Such risks will need to be addressed in the future, particularly from the perspective of mRNA vaccine distribution (<xref ref-type="bibr" rid="B25">25</xref>). Altogether, the risk analysis performed using the Risklick AI in early 2020 offered an accurate prediction of risks that would be faced by the different pharmaceutical companies providing vaccines in the year 2021.</p></sec>
<sec sec-type="discussion" id="s4">
<title>Discussion</title>
<p>The COVID-19 outbreak resulted in one of the biggest waves of publications in the history of modern science (<xref ref-type="bibr" rid="B26">26</xref>, <xref ref-type="bibr" rid="B27">27</xref>). This influx of COVID-19-related data made literature retrieval and monitoring one of the greatest challenges of the pandemic (<xref ref-type="bibr" rid="B28">28</xref>). The COVID-19 pandemic highlighted the need for vaccine developers and health authorities to provide general guidance for the faster development and preparation of vaccines while offering the highest safety and efficacy standards (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B29">29</xref>). Shortly after the start of the COVID-19 pandemic, several pharmaceutical companies providing vaccines announced their objective to develop vaccines against COVID-19, using different technologies, all within a matter of months (<xref ref-type="bibr" rid="B2">2</xref>, <xref ref-type="bibr" rid="B3">3</xref>).</p>
<p>The choice of the type of vaccine is crucial: carrier or vector, adjuvant, excipients, dosage form, and route of administration; all directly impact the potential efficacy of the vaccine against COVID-19, but also the logistics of manufacturing, storing, and distributing the vaccine might affect the availability of the product (<xref ref-type="bibr" rid="B5">5</xref>, <xref ref-type="bibr" rid="B6">6</xref>, <xref ref-type="bibr" rid="B9">9</xref>). Hence, this simple variation in the composition of the vaccines will directly impact the policy of mass vaccination and the access to vaccine doses worldwide, affecting the duration of the COVID-19 pandemic (<xref ref-type="bibr" rid="B8">8</xref>).</p>
<p>In this context, we performed a risk analysis on the different technologies used by the pharmaceutical companies providing vaccines against COVID-19 in the early months of 2020 by using the AI-based search engine Risklick (<xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B13">13</xref>).</p>
<p>Our results demonstrate that the AI-based analysis of risks allows the identification of the vaccine category with the highest chances of success. Moreover, the Risklick AI allowed us to foresee the different issues pharmaceutical companies providing vaccines would face, notably during the production and distribution phases of their product.</p>
<p>Our results highlight how AI-based technologies will become essential in the development of various therapies in the future. Moreover, our results demonstrate that, by using in-depth risk analysis, the global performance of vaccine developers could be increased by anticipating major issues, allowing corrective, and preventive actions. Such an approach, if generalized, would represent an important gain of time and efficiency for every scientist and manufacturer involved in providing care and drugs. The adoption of detailed risk analysis by all the scientists and manufacturers involved in the COVID-19 pandemic could represent a game-changer that could positively impact the future of therapy development worldwide.</p></sec>
<sec sec-type="data-availability" id="s5">
<title>Data Availability Statement</title>
<p>The original contributions presented in the study are included in the article/<xref ref-type="sec" rid="s9">Supplementary Material</xref>, further inquiries can be directed to the corresponding author/s.</p></sec>
<sec id="s6">
<title>Author Contributions</title>
<p>PA designed the study. QH and NB wrote the manuscript. QH, PA, and LM performed the experimental work. DA, SF, NB, and DT developed the AI models. All authors had full access to the data, helped draft the report or critically revised the draft, contributed to data interpretation, reviewed, and approved the final version of the report.</p></sec>
<sec sec-type="funding-information" id="s7">
<title>Funding</title>
<p>This study was funded by the Innosuisse project funding number 41013.1 IP-ICT.</p></sec>
<sec sec-type="COI-statement" id="conf1">
<title>Conflict of Interest</title>
<p>QH, NB, LM, and PA are work for Risklick. The remaining authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p></sec>
<sec sec-type="disclaimer" id="s8">
<title>Publisher&#x00027;s Note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p></sec> </body>
<back>
<sec sec-type="supplementary-material" id="s9">
<title>Supplementary Material</title>
<p>The Supplementary Material for this article can be found online at: <ext-link ext-link-type="uri" xlink:href="https://www.frontiersin.org/articles/10.3389/fdgth.2021.745674/full#supplementary-material">https://www.frontiersin.org/articles/10.3389/fdgth.2021.745674/full#supplementary-material</ext-link></p>
<supplementary-material xlink:href="Image_1.TIF" id="SM1" mimetype="image/tif" xmlns:xlink="http://www.w3.org/1999/xlink">
<label>Supplementary Figure 1</label>
<caption><p>The Probability and Impact Matrix for the safety of the six different vaccine technologies.</p></caption> </supplementary-material>
<supplementary-material xlink:href="Table_1.XLSX" id="SM2" mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" xmlns:xlink="http://www.w3.org/1999/xlink">
<label>Supplementary Table 1</label>
<caption><p>List of publications used for the semi-automated artificial intelligence (AI) risk calculation.</p></caption> </supplementary-material>
<supplementary-material xlink:href="Table_2.XLSX" id="SM3" mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" xmlns:xlink="http://www.w3.org/1999/xlink">
<label>Supplementary Table 2</label>
<caption><p>Definition of the variables used to assess the risk of vaccine technologies.</p></caption> </supplementary-material></sec>
<ref-list>
<title>References</title>
<ref id="B1">
<label>1.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Thiel</surname> <given-names>N</given-names></name> <name><surname>Selwyn</surname> <given-names>C</given-names></name> <name><surname>Murphy</surname> <given-names>G</given-names></name> <name><surname>Simpson</surname> <given-names>S</given-names></name> <name><surname>Chakrabarti</surname> <given-names>AC</given-names></name></person-group>. <article-title>Recommendations for acceleration of vaccine development and emergency use filings for COVID-19 leveraging lessons from the novel oral polio vaccine</article-title>. <source>npj Vaccines.</source> (<year>2021</year>) <volume>6</volume>:<fpage>63</fpage>. <pub-id pub-id-type="doi">10.1038/s41541-021-00325-4</pub-id><pub-id pub-id-type="pmid">33888722</pub-id></citation></ref>
<ref id="B2">
<label>2.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Izda</surname> <given-names>V</given-names></name> <name><surname>Jeffries</surname> <given-names>MA</given-names></name> <name><surname>Sawalha</surname> <given-names>AH</given-names></name></person-group>. <article-title>COVID-19: a review of therapeutic strategies and vaccine candidates</article-title>. <source>Clin Immunol.</source> (<year>2021</year>) <volume>222</volume>:<fpage>108634</fpage>. <pub-id pub-id-type="doi">10.1016/j.clim.2020.108634</pub-id><pub-id pub-id-type="pmid">33217545</pub-id></citation></ref>
<ref id="B3">
<label>3.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Russell</surname> <given-names>RL</given-names></name> <name><surname>Pelka</surname> <given-names>P</given-names></name> <name><surname>Mark</surname> <given-names>BL</given-names></name></person-group>. <article-title>Frontrunners in the race to develop a sars-cov-2 vaccine</article-title>. <source>Can J Microbiol.</source> (<year>2021</year>) <volume>67</volume>:<fpage>189</fpage>&#x02013;<lpage>212</lpage>. <pub-id pub-id-type="doi">10.1139/cjm-2020-0465</pub-id><pub-id pub-id-type="pmid">33264067</pub-id></citation></ref>
<ref id="B4">
<label>4.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Amanat</surname> <given-names>F</given-names></name> <name><surname>Krammer</surname> <given-names>F</given-names></name></person-group>. <article-title>SARS-CoV-2 vaccines: status report</article-title>. <source>Immunity.</source> (<year>2020</year>) <volume>52</volume>:<fpage>583</fpage>&#x02013;<lpage>9</lpage>. <pub-id pub-id-type="doi">10.1016/j.immuni.2020.03.007</pub-id><pub-id pub-id-type="pmid">32259480</pub-id></citation></ref>
<ref id="B5">
<label>5.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Wang</surname> <given-names>J</given-names></name> <name><surname>Peng</surname> <given-names>Y</given-names></name> <name><surname>Xu</surname> <given-names>H</given-names></name> <name><surname>Cui</surname> <given-names>Z</given-names></name> <name><surname>Williams</surname> <given-names>RO</given-names></name></person-group>. <article-title>The COVID-19 vaccine race: challenges and opportunities in vaccine formulation</article-title>. <source>AAPS PharmSciTech.</source> (<year>2020</year>) <volume>21</volume>:<fpage>225</fpage>. <pub-id pub-id-type="doi">10.1208/s12249-020-01744-7</pub-id><pub-id pub-id-type="pmid">32761294</pub-id></citation></ref>
<ref id="B6">
<label>6.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Haynes</surname> <given-names>BF</given-names></name> <name><surname>Corey</surname> <given-names>L</given-names></name> <name><surname>Fernandes</surname> <given-names>P</given-names></name> <name><surname>Gilbert</surname> <given-names>PB</given-names></name> <name><surname>Hotez</surname> <given-names>PJ</given-names></name> <name><surname>Rao</surname> <given-names>S</given-names></name> <etal/></person-group>. <article-title>Prospects for a safe COVID-19 vaccine</article-title>. <source>Sci Transl Med.</source> (<year>2020</year>) <volume>12</volume>:<fpage>eabe0948</fpage>. <pub-id pub-id-type="doi">10.1126/scitranslmed.abe0948</pub-id><pub-id pub-id-type="pmid">33077678</pub-id></citation></ref>
<ref id="B7">
<label>7.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mullard</surname> <given-names>A</given-names></name></person-group>. <article-title>COVID-19 vaccine development pipeline gears up</article-title>. <source>Lancet.</source> (<year>2020</year>) <volume>395</volume>:<fpage>1751</fpage>&#x02013;<lpage>2</lpage>. <pub-id pub-id-type="doi">10.1016/S0140-6736(20)31252-6</pub-id><pub-id pub-id-type="pmid">32505245</pub-id></citation></ref>
<ref id="B8">
<label>8.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Rossman</surname> <given-names>H</given-names></name> <name><surname>Shilo</surname> <given-names>S</given-names></name> <name><surname>Meir</surname> <given-names>T</given-names></name> <name><surname>Gorfine</surname> <given-names>M</given-names></name> <name><surname>Shalit</surname> <given-names>U</given-names></name> <name><surname>Segal</surname> <given-names>E</given-names></name></person-group>. <article-title>COVID-19 dynamics after a national immunization program in Israel</article-title>. <source>Nat Med.</source> (<year>2021</year>) <volume>27</volume>:<fpage>1055</fpage>&#x02013;<lpage>61</lpage>. <pub-id pub-id-type="doi">10.1038/s41591-021-01337-2</pub-id><pub-id pub-id-type="pmid">33875890</pub-id></citation></ref>
<ref id="B9">
<label>9.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Singh</surname> <given-names>JA</given-names></name> <name><surname>Upshur</surname> <given-names>REG</given-names></name></person-group>. <article-title>The granting of emergency use designation to COVID-19 candidate vaccines: implications for COVID-19 vaccine trials</article-title>. <source>Lancet Infect Dis.</source> (<year>2021</year>) <volume>21</volume>:<fpage>e103</fpage>&#x02013;<lpage>9</lpage>. <pub-id pub-id-type="doi">10.1016/S1473-3099(20)30923-3</pub-id><pub-id pub-id-type="pmid">33306980</pub-id></citation></ref>
<ref id="B10">
<label>10.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Rawat</surname> <given-names>K</given-names></name> <name><surname>Kumari</surname> <given-names>P</given-names></name> <name><surname>Saha</surname> <given-names>L</given-names></name></person-group>. <article-title>COVID-19 vaccine: a recent update in pipeline vaccines, their design and development strategies</article-title>. <source>Eur J Pharmacol.</source> (<year>2021</year>) <volume>892</volume>:<fpage>173751</fpage>. <pub-id pub-id-type="doi">10.1016/j.ejphar.2020.173751</pub-id><pub-id pub-id-type="pmid">33245898</pub-id></citation></ref>
<ref id="B11">
<label>11.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Iserson K</surname> <given-names>V</given-names></name></person-group>. <article-title>Sars-cov-2 (covid-19) vaccine development and production: an ethical way forward</article-title>. <source>Cambridge Q Healthc Ethics.</source> (<year>2020</year>) <volume>30</volume>:<fpage>59</fpage>&#x02013;<lpage>68</lpage>. <pub-id pub-id-type="doi">10.1017/S096318012000047X</pub-id><pub-id pub-id-type="pmid">32498742</pub-id></citation></ref>
<ref id="B12">
<label>12.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Haas</surname> <given-names>Q</given-names></name> <name><surname>Alvarez</surname> <given-names>DV</given-names></name> <name><surname>Borissov</surname> <given-names>N</given-names></name> <name><surname>Ferdowsi</surname> <given-names>S</given-names></name> <name><surname>Von Meyenn</surname> <given-names>L</given-names></name> <name><surname>Trelle</surname> <given-names>S</given-names></name> <etal/></person-group>. <article-title>Utilizing artificial intelligence to manage COVID-19 scientific evidence torrent with risklick ai: a critical tool for pharmacology and therapy development</article-title>. <source>Pharmacology.</source> (<year>2021</year>) <volume>106</volume>:<fpage>244</fpage>&#x02013;<lpage>53</lpage>. <pub-id pub-id-type="doi">10.1159/000515908</pub-id><pub-id pub-id-type="pmid">33910199</pub-id></citation></ref>
<ref id="B13">
<label>13.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Teodoro</surname> <given-names>D</given-names></name> <name><surname>Ferdowsi</surname> <given-names>S</given-names></name> <name><surname>Borissov</surname> <given-names>N</given-names></name> <name><surname>Kashani</surname> <given-names>E</given-names></name> <name><surname>Vicente Alvarez</surname> <given-names>D</given-names></name> <name><surname>Copara</surname> <given-names>J</given-names></name> <name><surname>Gouareb</surname> <given-names>R</given-names></name> <name><surname>Naderi</surname> <given-names>N</given-names></name> <name><surname>Amini</surname> <given-names>P</given-names></name></person-group>. <article-title>Information Retrieval in an Infodemic: The Case of COVID-19 Publications</article-title>. <source>J Med Internet Res.</source> (<year>2021</year>) <volume>23</volume>:<fpage>e30161</fpage>. <pub-id pub-id-type="doi">10.2196/30161</pub-id><pub-id pub-id-type="pmid">34375298</pub-id></citation></ref>
<ref id="B14">
<label>14.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Dumbrav&#x00103;</surname> <given-names>V</given-names></name> <name><surname>Iacob</surname> <given-names>V-S</given-names></name></person-group>. <article-title>Using probability &#x02013; impact matrix in analysis and risk assessment projects</article-title>. <source>J Knowl Manag Econ Inf Technol.</source> (<year>2013</year>) <volume>3</volume>:<fpage>1</fpage>&#x02013;<lpage>7</lpage>.</citation>
</ref>
<ref id="B15">
<label>15.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sohrab</surname> <given-names>Ferdowsi</given-names></name> <name><surname>Nikolay</surname> <given-names>Borissov</given-names></name> <name><surname>Julien</surname> <given-names>Knafou</given-names></name> <name><surname>Poorya</surname> <given-names>Amini</given-names></name> <name><surname>Douglas</surname> <given-names>Teodoro</given-names></name></person-group>. <article-title>Classification of hierarchical text using geometric deep learning: the case of clinical trials corpus. Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing</article-title>. (<year>2021</year>).</citation>
</ref>
<ref id="B16">
<label>16.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nooraei</surname> <given-names>S</given-names></name> <name><surname>Bahrulolum</surname> <given-names>H</given-names></name> <name><surname>Hoseini</surname> <given-names>ZS</given-names></name> <name><surname>Katalani</surname> <given-names>C</given-names></name> <name><surname>Hajizade</surname> <given-names>A</given-names></name> <name><surname>Easton</surname> <given-names>AJ</given-names></name> <etal/></person-group>. <article-title>Virus-like particles: preparation, immunogenicity and their roles as nanovaccines and drug nanocarriers</article-title>. <source>J Nanobiotechnol.</source> (<year>2021</year>) <volume>19</volume>:<fpage>1</fpage>&#x02013;<lpage>27</lpage>. <pub-id pub-id-type="doi">10.1186/s12951-021-00806-7</pub-id><pub-id pub-id-type="pmid">33632278</pub-id></citation></ref>
<ref id="B17">
<label>17.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>Vaccines &#x02013; COVID19 Vaccine Tracker</collab></person-group>. Available online at: <ext-link ext-link-type="uri" xlink:href="https://covid19.trackvaccines.org/vaccines/">https://covid19.trackvaccines.org/vaccines/</ext-link> (accessed September 26, 2021)</citation>
</ref>
<ref id="B18">
<label>18.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>COVID-19 Vaccine Tracker and Landscape</collab></person-group>. Available online at: <ext-link ext-link-type="uri" xlink:href="https://www.who.int/publications/m/item/draft-landscape-of-covid-19-candidate-vaccines">https://www.who.int/publications/m/item/draft-landscape-of-covid-19-candidate-vaccines</ext-link> (accessed September 26, 2021)</citation>
</ref>
<ref id="B19">
<label>19.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sagili Anthony</surname> <given-names>DP</given-names></name> <name><surname>Sivakumar</surname> <given-names>K</given-names></name> <name><surname>Venugopal</surname> <given-names>P</given-names></name> <name><surname>Sriram</surname> <given-names>DK</given-names></name> <name><surname>George</surname> <given-names>M</given-names></name></person-group>. <article-title>Can mRNA vaccines turn the tables during the COVID-19 pandemic? Current status and challenges</article-title>. <source>Clin Drug Investig.</source> (<year>2021</year>) <volume>41</volume>:<fpage>499</fpage>&#x02013;<lpage>509</lpage>. <pub-id pub-id-type="doi">10.1007/s40261-021-01022-9</pub-id><pub-id pub-id-type="pmid">34089504</pub-id></citation></ref>
<ref id="B20">
<label>20.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>Update on EU Supply of COVID-19 Vaccine AstraZeneca</collab></person-group>. Available online at: <ext-link ext-link-type="uri" xlink:href="https://www.astrazeneca.com/media-centre/articles/2021/update-on-eu-supply-of-covid-19-vaccine-astrazeneca.html">https://www.astrazeneca.com/media-centre/articles/2021/update-on-eu-supply-of-covid-19-vaccine-astrazeneca.html</ext-link> (accessed September 26, 2021)</citation>
</ref>
<ref id="B21">
<label>21.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>The Moscow Times</collab></person-group>. <source>Argentina, Mexico Face Second-Dose Shortages of Russia&#x00027;s Sputnik V Vaccine &#x02013; Reports - The Moscow Times</source>. Available online at: <ext-link ext-link-type="uri" xlink:href="https://www.themoscowtimes.com/2021/06/21/argentina-mexico-face-second-dose-shortages-of-russias-sputnik-v-vaccine-reports-a74279">https://www.themoscowtimes.com/2021/06/21/argentina-mexico-face-second-dose-shortages-of-russias-sputnik-v-vaccine-reports-a74279</ext-link> (accessed September 26, 2021)</citation>
</ref>
<ref id="B22">
<label>22.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Carneiro</surname> <given-names>DC</given-names></name> <name><surname>Sousa</surname> <given-names>JD</given-names></name> <name><surname>Monteiro-Cunha</surname> <given-names>JP</given-names></name></person-group>. <article-title>The COVID-19 vaccine development: a pandemic paradigm</article-title>. <source>Virus Res.</source> (<year>2021</year>) <volume>301</volume>:<fpage>198454</fpage>. <pub-id pub-id-type="doi">10.1016/j.virusres.2021.198454</pub-id><pub-id pub-id-type="pmid">34015363</pub-id></citation></ref>
<ref id="B23">
<label>23.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Radanliev</surname> <given-names>P</given-names></name> <name><surname>De Roure</surname> <given-names>D</given-names></name> <name><surname>Ani</surname> <given-names>U</given-names></name> <name><surname>Carvalho</surname> <given-names>G</given-names></name></person-group>. <article-title>The ethics of shared Covid-19 risks: an epistemological framework for ethical health technology assessment of risk in vaccine supply chain infrastructures</article-title>. <source>Heal Technol.</source> (<year>2021</year>) <volume>11</volume>, p.<fpage>1083</fpage>&#x02013;<lpage>1091</lpage>. <pub-id pub-id-type="doi">10.1007/s12553-021-00565-3</pub-id><pub-id pub-id-type="pmid">34123697</pub-id></citation></ref>
<ref id="B24">
<label>24.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Grau</surname> <given-names>S</given-names></name> <name><surname>Ferr&#x000E1;ndez</surname> <given-names>O</given-names></name> <name><surname>Mart&#x000ED;n-Garc&#x000ED;a</surname> <given-names>E</given-names></name> <name><surname>Maldonado</surname> <given-names>R</given-names></name></person-group>. <article-title>Accidental interruption of the cold chain for the preservation of the moderna COVID-19 vaccine</article-title>. <source>Vaccines.</source> (<year>2021</year>) <volume>9</volume>:<fpage>512</fpage>. <pub-id pub-id-type="doi">10.3390/vaccines9050512</pub-id><pub-id pub-id-type="pmid">34067511</pub-id></citation></ref>
<ref id="B25">
<label>25.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Crommelin</surname> <given-names>DJA</given-names></name> <name><surname>Anchordoquy</surname> <given-names>TJ</given-names></name> <name><surname>Volkin</surname> <given-names>DB</given-names></name> <name><surname>Jiskoot</surname> <given-names>W</given-names></name> <name><surname>Mastrobattista</surname> <given-names>E</given-names></name></person-group>. <article-title>Addressing the cold reality of mRNA vaccine stability</article-title>. <source>J Pharm Sci.</source> (<year>2021</year>) <volume>110</volume>:<fpage>997</fpage>&#x02013;<lpage>1001</lpage>. <pub-id pub-id-type="doi">10.1016/j.xphs.2020.12.006</pub-id><pub-id pub-id-type="pmid">33321139</pub-id></citation></ref>
<ref id="B26">
<label>26.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Palayew</surname> <given-names>A</given-names></name> <name><surname>Norgaard</surname> <given-names>O</given-names></name> <name><surname>Safreed-Harmon</surname> <given-names>K</given-names></name> <name><surname>Andersen</surname> <given-names>TH</given-names></name> <name><surname>Rasmussen</surname> <given-names>LN</given-names></name> <name><surname>Lazarus J</surname> <given-names>V</given-names></name></person-group>. <article-title>Pandemic publishing poses a new COVID-19 challenge</article-title>. <source>Nat Hum Behav.</source> (<year>2020</year>) <volume>4</volume>:<fpage>666</fpage>&#x02013;<lpage>9</lpage>. <pub-id pub-id-type="doi">10.1038/s41562-020-0911-0</pub-id><pub-id pub-id-type="pmid">32576981</pub-id></citation></ref>
<ref id="B27">
<label>27.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Callaway</surname> <given-names>E</given-names></name></person-group>. <article-title>Will the pandemic permanently alter scientific publishing?</article-title> <source>Nature.</source> (<year>2020</year>) <volume>582</volume>:<fpage>167</fpage>&#x02013;<lpage>8</lpage>. <pub-id pub-id-type="doi">10.1038/d41586-020-01520-4</pub-id><pub-id pub-id-type="pmid">32504015</pub-id></citation></ref>
<ref id="B28">
<label>28.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Muhammad</surname> <given-names>LJ</given-names></name> <name><surname>Islam</surname> <given-names>MM</given-names></name> <name><surname>Usman</surname> <given-names>SS</given-names></name> <name><surname>Ayon</surname> <given-names>SI</given-names></name></person-group>. <article-title>Predictive data mining models for novel coronavirus (COVID-19) infected patients&#x00027; recovery</article-title>. <source>SN Comput Sci.</source> (<year>2020</year>) <volume>1</volume>:<fpage>206</fpage>. <pub-id pub-id-type="doi">10.1007/s42979-020-00216-w</pub-id><pub-id pub-id-type="pmid">33063049</pub-id></citation></ref>
<ref id="B29">
<label>29.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lurie</surname> <given-names>N</given-names></name> <name><surname>Saville</surname> <given-names>M</given-names></name> <name><surname>Hatchett</surname> <given-names>R</given-names></name> <name><surname>Halton</surname> <given-names>J</given-names></name></person-group>. <article-title>Developing Covid-19 vaccines at pandemic speed</article-title>. <source>N Engl J Med.</source> (<year>2020</year>) <volume>382</volume>:<fpage>1969</fpage>&#x02013;<lpage>73</lpage>. <pub-id pub-id-type="doi">10.1056/NEJMp2005630</pub-id><pub-id pub-id-type="pmid">32227757</pub-id></citation></ref>
</ref-list> 
</back>
</article>