<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article xml:lang="EN" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="brief-report">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Med.</journal-id>
<journal-title>Frontiers in Medicine</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Med.</abbrev-journal-title>
<issn pub-type="epub">2296-858X</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fmed.2022.1073720</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Medicine</subject>
<subj-group>
<subject>Brief Research Report</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Manual and semi-automated approaches to MIBG myocardial scintigraphy in patients with Parkinson&#x2019;s disease</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name><surname>Boccalini</surname> <given-names>Cecilia</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<xref ref-type="author-notes" rid="fn002"><sup>&#x2020;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1933699/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Carli</surname> <given-names>Giulia</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
<xref ref-type="author-notes" rid="fn002"><sup>&#x2020;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/2111532/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Vanoli</surname> <given-names>Emilia Giovanna</given-names></name>
<xref ref-type="aff" rid="aff5"><sup>5</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Cocco</surname> <given-names>Antoniangela</given-names></name>
<xref ref-type="aff" rid="aff6"><sup>6</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Albanese</surname> <given-names>Alberto</given-names></name>
<xref ref-type="aff" rid="aff6"><sup>6</sup></xref>
<xref ref-type="aff" rid="aff7"><sup>7</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/10126/overview"/>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name><surname>Garibotto</surname> <given-names>Valentina</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<xref ref-type="aff" rid="aff8"><sup>8</sup></xref>
<xref ref-type="aff" rid="aff9"><sup>9</sup></xref>
<xref ref-type="corresp" rid="c001"><sup>&#x002A;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/747941/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Perani</surname> <given-names>Daniela</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff5"><sup>5</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/154801/overview"/>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>School of Psychology, Vita-Salute San Raffaele University</institution>, <addr-line>Milan</addr-line>, <country>Italy</country></aff>
<aff id="aff2"><sup>2</sup><institution>In vivo Human Molecular and Structural Neuroimaging Unit, Division of Neuroscience, Scientific Institute for Research, Hospitalization and Healthcare (IRCCS) San Raffaele Scientific Institute</institution>, <addr-line>Milan</addr-line>, <country>Italy</country></aff>
<aff id="aff3"><sup>3</sup><institution>Laboratory of Neuroimaging and Innovative Molecular Tracers (NIMTlab), Geneva University Neurocenter and Faculty of Medicine, University of Geneva</institution>, <addr-line>Geneva</addr-line>, <country>Switzerland</country></aff>
<aff id="aff4"><sup>4</sup><institution>Department of Nuclear Medicine and Molecular Imaging, University of Groningen, University Medical Center Groningen</institution>, <addr-line>Groningen</addr-line>, <country>Netherlands</country></aff>
<aff id="aff5"><sup>5</sup><institution>Nuclear Medicine Unit, San Raffaele Hospital</institution>, <addr-line>Milan</addr-line>, <country>Italy</country></aff>
<aff id="aff6"><sup>6</sup><institution>Department of Neurology, Scientific Institute for Research, Hospitalization and Healthcare (IRCCS) Humanitas Research Hospital</institution>, <addr-line>Milan</addr-line>, <country>Italy</country></aff>
<aff id="aff7"><sup>7</sup><institution>Department of Neuroscience, Catholic University</institution>, <addr-line>Milan</addr-line>, <country>Italy</country></aff>
<aff id="aff8"><sup>8</sup><institution>Division of Nuclear Medicine and Molecular Imaging, Geneva University Hospitals</institution>, <addr-line>Geneva</addr-line>, <country>Switzerland</country></aff>
<aff id="aff9"><sup>9</sup><institution>Center for Biomedical Imaging</institution>, <addr-line>Geneva</addr-line>, <country>Switzerland</country></aff>
<author-notes>
<fn fn-type="edited-by"><p>Edited by: Giorgio Treglia, Ente Ospedaliero Cantonale (EOC), Switzerland</p></fn>
<fn fn-type="edited-by"><p>Reviewed by: Alberto Miceli, Universit&#x00E0; di Genova, Italy; Agostino Chiaravalloti, University of Rome Tor Vergata, Italy</p></fn>
<corresp id="c001">&#x002A;Correspondence: Valentina Garibotto, <email>valentina.garibotto@hcuge.ch</email></corresp>
<fn fn-type="equal" id="fn002"><p><sup>&#x2020;</sup>These authors have contributed equally to this work</p></fn>
<fn fn-type="other" id="fn004"><p>This article was submitted to Nuclear Medicine, a section of the journal Frontiers in Medicine</p></fn>
</author-notes>
<pub-date pub-type="epub">
<day>02</day>
<month>12</month>
<year>2022</year>
</pub-date>
<pub-date pub-type="collection">
<year>2022</year>
</pub-date>
<volume>9</volume>
<elocation-id>1073720</elocation-id>
<history>
<date date-type="received">
<day>18</day>
<month>10</month>
<year>2022</year>
</date>
<date date-type="accepted">
<day>17</day>
<month>11</month>
<year>2022</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x00A9; 2022 Boccalini, Carli, Vanoli, Cocco, Albanese, Garibotto and Perani.</copyright-statement>
<copyright-year>2022</copyright-year>
<copyright-holder>Boccalini, Carli, Vanoli, Cocco, Albanese, Garibotto and Perani</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p></license>
</permissions>
<abstract>
<sec>
<title>Objective</title>
<p>This study investigates the effects of manual and semi-automatic methods for assessing MIBG semi-quantitative indices in a clinical setting.</p>
</sec>
<sec>
<title>Materials and methods</title>
<p>We included <sup>123</sup>I-MIBG scans obtained in 35 patients with idiopathic Parkinson&#x2019;s Disease. Early and late heart-to-mediastinum (H/M) ratios were calculated from <sup>123</sup>I-MIBG images using regions of interest (ROIs) placed over the heart and the mediastinum. The ROIs were derived using two approaches: (i) manually drawn and (ii) semi-automatic fixed-size ROIs using anatomical landmarks. Expert, moderate-expert, and not expert raters applied the ROIs procedures and interpreted the <sup>123</sup>I-MIBG images. We evaluated the inter and intra-rater agreements in assessing <sup>123</sup>I-MIBG H/M ratios.</p>
</sec>
<sec>
<title>Results</title>
<p>A moderate agreement in the raters&#x2019; classification of pathological and non-pathological scores emerged regarding early and late H/M ratio values (&#x03BA; = 0.45 and 0.69 respectively), applying the manual method, while the early and late H/M ratios obtained with the semi-automatic method reached a good agreement among observers (&#x03BA; = 0.78). Cohen-Kappa values revealed that the semi-automatic method improved the agreement between expert and inexpert raters: the agreement improved from a minimum of 0.29 (fair, for early H/M) and 0.69 (substantial, in late H/M) with the manual method, to 0.90 (perfect, in early H/M) and 0.87 (perfect, in late H/M) with the semi-automatic method.</p>
</sec>
<sec>
<title>Conclusion</title>
<p>The use of the semi-automatic method improves the agreement among raters in classifying&#x2019; H/M ratios as pathological or non-pathological, namely for inexpert readers. These results have important implications for semi-quantitative assessment of <sup>123</sup>I-MIBG images in clinical routine.</p>
</sec>
</abstract>
<kwd-group>
<kwd>Parkinson&#x2019;s disease</kwd>
<kwd>MIBG scintigraphy</kwd>
<kwd>H/M ratios</kwd>
<kwd>manual-drawing approach</kwd>
<kwd>semi-automatic fixed-size ROIs</kwd>
</kwd-group>
<contract-sponsor id="cn001">Fondazione Cariplo<named-content content-type="fundref-id">10.13039/501100002803</named-content></contract-sponsor>
<counts>
<fig-count count="1"/>
<table-count count="1"/>
<equation-count count="0"/>
<ref-count count="23"/>
<page-count count="6"/>
<word-count count="3887"/>
</counts>
</article-meta>
</front>
<body>
<sec id="S1" sec-type="intro">
<title>Introduction</title>
<p>Parkinson&#x2019;s disease (PD) and parkinsonisms are a spectrum of multisystem neurodegenerative diseases characterised by early involvement of the autonomic and enteric nervous systems (<xref ref-type="bibr" rid="B1">1</xref>). PD-related neuropathology (&#x03B1;-synuclein aggregates) is not restricted to the central nervous system but affects several organs and tissues (<xref ref-type="bibr" rid="B2">2</xref>, <xref ref-type="bibr" rid="B3">3</xref>). The denervation of the autonomic nervous system can be assessed by <sup>123</sup>I-metaiodobenzylguanidine (MIBG) scintigraphy (<xref ref-type="bibr" rid="B4">4</xref>), which can be clinically helpful as a supportive diagnostic tool (<xref ref-type="bibr" rid="B5">5</xref>). <sup>123</sup>I-MIBG, a norepinephrine analogue (<xref ref-type="bibr" rid="B6">6</xref>), is taken up into the presynaptic cardiac sympathetic nerves by the norepinephrine uptake-1 transporter, and the amount of <sup>123</sup>I-MIBG retention over several hours after administration reflects neuronal integrity (<xref ref-type="bibr" rid="B7">7</xref>). Thus, <sup>123</sup>I-MIBG cardiac scintigraphy measures the postganglionic presynaptic cardiac sympathetic nerve endings of the noradrenergic nervous system. It was developed for the assessment of heart diseases (<xref ref-type="bibr" rid="B8">8</xref>). Subsequently, cardiac <sup>123</sup>I-MIBG scintigraphy was clinically applied in the PD spectrum, showing a reduced cardiac uptake in the vast majority of patients (<xref ref-type="bibr" rid="B9">9</xref>), representing a sensitive test for this clinical condition. Therefore, positive results at <sup>123</sup>I-MIBG scintigraphy are considered a supportive biomarker for PD detection (<xref ref-type="bibr" rid="B5">5</xref>) and differential diagnosis (<xref ref-type="bibr" rid="B10">10</xref>). <sup>123</sup>I-MIBG scintigraphy can accurately distinguish Lewy Bodies (LB) related disorders [i.e., PD, Dementia with LB (DLB)] and non-LB related disorders (e.g., Alzheimer&#x2019;s disease and progressive supranuclear palsy) (<xref ref-type="bibr" rid="B10">10</xref>).</p>
<p>The most used quantitative measurements of <sup>123</sup>I-MIBG uptake are the heart-to-mediastinum (H/M) ratios calculated from early (15 min) and late (3&#x2013;4 h) <sup>123</sup>I-MIBG static planar images. The method used to obtain these parameters is mainly based on manually drawing cardiac regions of interest (ROIs). However, the position and size of the cardiac ROI, as well as the level of experience in drawing the ROIs on the planar <sup>123</sup>I-MIBG images, might affect the final H/M ratio measures.</p>
<p>The effects of different methods to assess <sup>123</sup>I-MIBG parameters for clinical interpretation should be carefully considered (<xref ref-type="bibr" rid="B11">11</xref>, <xref ref-type="bibr" rid="B12">12</xref>). Therefore, this study aimed to compare two different methods assessing the H/M ratios in a series of PD patients, (i) the typical manual-drawing method and (ii) the fixed-size ROIs approach following Veltman&#x2019;s guidelines for the standardisation of <sup>123</sup>I-MIBG parameters. Veltman et al. (<xref ref-type="bibr" rid="B13">13</xref>) demonstrated good reproducibility of fixed-size mediastinal and cardiac ROIs in assessing H/M ratios in patients with heart failure; here, we wanted to extend these findings in PD. <sup>123</sup>I-MIBG images of 35 PD patients were analysed by highly expert, moderate-expert, and inexpert raters. We evaluated the influences of cardiac ROI size and position and the impact of raters&#x2019; experience on assessing the H/M ratios.</p>
</sec>
<sec id="S2">
<title>Methods</title>
<sec id="S2.SS1">
<title>Participants</title>
<p>The study included 35 patients fulfilling the PD clinical diagnostic criteria (<xref ref-type="bibr" rid="B5">5</xref>). All participants gave their written informed consent to the experimental procedure, as approved by the Ethical Committees of Scientific Institute for Research, Hospitalization and Healthcare (IRCCS) San Raffaele Hospital. The protocol conformed to the ethical standards of the Declaration of Helsinki for the protection of human subjects.</p>
</sec>
<sec id="S2.SS2">
<title><sup>123</sup>I-metaiodobenzylguanidine scintigraphy data acquisition</title>
<p><sup>123</sup>I-MIBG myocardial scintigraphy was performed at the Nuclear Medicine Unit, San Raffaele Hospital, using (NM/CT 670, Milwaukee, WI, United States) manufactured by GE Healthcare medical systems, according to the standard procedure (<xref ref-type="bibr" rid="B13">13</xref>). All patients have suspended medication and stopped eating any food that could influence <sup>123</sup>I-MIBG myocardial uptake (<xref ref-type="bibr" rid="B14">14</xref>). A total of 111 MBq of <sup>123</sup>I-MIBG was injected intravenously. <sup>123</sup>I-MIBG planar images were obtained with a gamma camera in two temporal windows: 15 min after the injection (early-image) and 4 h after the injection (delayed-image). Static planar imaging was obtained with 256 &#x00D7; 256 matrix. Only planar images in thoracic anterior view were used for quantitative evaluation. All camera heads were equipped with low-energy high-resolution collimators and all images were acquired with a 15% energy window centred at the 159 keV photopeak of <sup>123</sup>I. The image acquisition time was approximately 5 min. All planar images were analysed using Xeleris (Discovery NM/CT 670, GE Healthcare, Milwaukee, WI, United States) and PMOD software (PMOD Technologies Ltd.), for manual and semi-automatic methods respectively.</p>
</sec>
<sec id="S2.SS3">
<title><sup>123</sup>I-metaiodobenzylguanidine scintigraphy planar image analyses</title>
<p>Early and late H/M ratios were calculated from <sup>123</sup>I-MIBG planar images using ROIs placed over the heart and the upper mediastinum. The ROIs were derived using two different approaches: manually drawn ROIs and semi-automatic fixed-size ROIs. In the first approach, a rectangular mediastinal ROI has been used with unspecified size. The cardiac ROI was a polygonal manually drawn ROI including the myocardium and the left ventricular cavity (<xref ref-type="fig" rid="F1">Figure 1A</xref>). The manual method was performed on a Xeleris workstation, following the recommendations described in the proposal for standardisation by Flotats et al. (<xref ref-type="bibr" rid="B14">14</xref>).</p>
<fig id="F1" position="float">
<label>FIGURE 1</label>
<caption><p>Manual and fixed-size regions of interest (ROIs) on planar <sup>123</sup>I-metaiodobenzylguanidine (MIBG) images. Panel <bold>(A)</bold> depicts the manual ROIs method. A rectangular mediastinal ROI is used with unspecified size and located on planar MIBG images below the thyroid. The manual-drawn cardiac ROI is polygonal and includes the myocardium and the left ventricular cavity. Panel <bold>(B)</bold> represents the semi-automatic ROIs method. An &#x201C;anatomical landmark square&#x201D; is formed by the lung apexes (upper square border), the upper cardiac border (lower square border), and the medial contours of the lungs (medial square borders) to place fixed-size mediastinum and oval cardiac ROIs. Precisely, a fixed-size rectangular mediastinal ROI (13 &#x00D7; 20 pixel) and an oval cardiac ROI are placed on MIBG planar images. The mediastinal ROI is placed in the half of the &#x201C;anatomical landmark square&#x201D; formed by lungs apexes, the upper cardiac border, and the medial contours of the lung. The cardiac oval ROI (60 &#x00D7; 70 pixel) is placed in the lower-right quadrant, over the myocardium, including the left ventricular cavity.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fmed-09-1073720-g001.tif"/>
</fig>
<p>For the semi-automatic method, we followed the guidelines of Veltman et al. (<xref ref-type="bibr" rid="B13">13</xref>) (<xref ref-type="fig" rid="F1">Figure 1B</xref>). An &#x201C;anatomical landmark square&#x201D; was formed by the lung apexes (upper square border), the upper cardiac border (lower square border), and the medial contours of the lungs (medial square borders) to place fixed-size mediastinum and oval cardiac ROIs. Precisely, a fixed-size rectangular mediastinal ROI (13 &#x00D7; 20 pixel) and an oval cardiac ROI (60 &#x00D7; 70 pixel) were placed on <sup>123</sup>I-MIBG planar images. The mediastinal ROI was placed in the half of the &#x201C;anatomical landmark square&#x201D; formed by lungs apexes, the upper cardiac border, and the medial contours of the lung. The cardiac oval ROI was placed in the lower-right quadrant, over the myocardium, including the left ventricular cavity. The use of fixed-size ROI, which the operators only need to move according to the anatomical landmark square, gives partial automation to the method. This semiautomatic method was processed using the PMOD software.</p>
<p>Three raters (expert, moderately-expert, and not expert) analysed the <sup>123</sup>I-MIBG images using both procedures. Of note, the inexpert rater followed a brief training (2 h) for the manual drawing analysis. The other raters were experienced in the analysis of <sup>123</sup>I-MIBG myocardial scintigraphy, with mean more than 5 and 10 years of experience, respectively. Anonymised data were analysed in a random order to avoid bias by order effects.</p>
</sec>
<sec id="S2.SS4">
<title>Statistical analyses</title>
<p>The intra- and inter-rater agreements were tested to assess the reproducibility of the measured H/M ratios.</p>
<sec id="S2.SS4.SSS1">
<title>The inter-rater agreements</title>
<p>The intraclass correlation coefficients (ICC) were used to assess agreement among independent raters, considering the H/M ratios as a continuous variable. ICC values &#x003C; 0.5, between 0.5 and 0.75, between 0.75 and 0.9, and greater than 0.90 are indicative of poor, moderate, good, and excellent reliability, respectively (<xref ref-type="bibr" rid="B15">15</xref>).</p>
<p>We applied previously published cut-off for pathological scans (&#x003C;1.9 early ratios and &#x003C;1.7 late ratios) to classify the H/M ratios into mutually exclusive categories (pathological and not) (<xref ref-type="bibr" rid="B16">16</xref>). Following this categorisation, we calculated the Fleiss&#x2019; kappa, &#x03BA; to measure the inter-rater agreement between the raters (<xref ref-type="bibr" rid="B17">17</xref>, <xref ref-type="bibr" rid="B18">18</xref>). The following classification was suggested for assessing the strength of agreement: &#x03BA; &#x003C; 0.20 poor, 0.21&#x2013;0.40 fair, 0.41&#x2013;0.60 moderate, 0.61&#x2013;0.80 good, and 0.81&#x2013;1.00 very good (<xref ref-type="bibr" rid="B19">19</xref>). Last, Cohen&#x2019;s Kappa was also used to measure the agreement between two raters after classifying the H/M ratios as pathological or not. Cohen&#x2019;s Kappa values 0.01&#x2013;0.20, 0.21&#x2013;0.40, 0.41&#x2013;0.60, 0.61&#x2013;0.80, and 0.81&#x2013;1.00 were considered slight, fair, moderate, substantial, and perfect agreement, respectively (<xref ref-type="bibr" rid="B20">20</xref>). A Chi-square test was run to test differences in the proportion of patients classified as pathological or not by different raters.</p>
</sec>
<sec id="S2.SS4.SSS2">
<title>The intra-rater agreements</title>
<p>The correlations between the H/M ratios obtained with the two methods by each rater were analysed by the Spearman correlation coefficients. The significance level was <italic>p</italic> &#x003C; 0.001.</p>
<p>All statistical analyses were performed using the SPSS software package, version 26.0 (SPSS, Chicago, IL, United States).</p>
</sec>
</sec>
</sec>
<sec id="S3" sec-type="results">
<title>Results</title>
<p>A total of 35 PD patients [63.66 &#x00B1; 9.74 years of age, 27 (77%) men] were included in the study. <xref ref-type="table" rid="T1">Table 1</xref> reported the number of PD patients that fell into the category of &#x201C;pathological sympathetic innervation,&#x201D; according to the cut-off value of early and late H/M ratios, obtained with manual approach and semi-automatic approach for the three raters.</p>
<table-wrap position="float" id="T1">
<label>TABLE 1</label>
<caption><p>Inter and intra-rater indices on <sup>123</sup>I-metaiodobenzylguanidine (MIBG) parameters.</p></caption>
<table cellspacing="5" cellpadding="5" frame="hsides" rules="groups">
<thead>
<tr>
<td valign="top" align="left"></td>
<td valign="top" align="center" colspan="2">Early H/M ratio<hr/></td>
<td valign="top" align="center" colspan="2">Delayed H/M ratio<hr/></td>
</tr>
<tr>
<td valign="top" align="left"/><td valign="top" align="center">Manual method</td>
<td valign="top" align="center">Semi-automatic method</td>
<td valign="top" align="center">Manual method</td>
<td valign="top" align="center">Semi-automatic method</td>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left" colspan="5"><bold>Inter-raters analysis</bold></td>
</tr>
<tr>
<td valign="top" align="left" colspan="5"><bold>Continue values</bold></td>
</tr>
<tr>
<td valign="top" align="left">Expert (1)</td>
<td valign="top" align="center">1.49 &#x00B1; 0.27</td>
<td valign="top" align="center">1.59 &#x00B1; 0.27</td>
<td valign="top" align="center">1.41 &#x00B1; 0.37</td>
<td valign="top" align="center">1.51 &#x00B1; 0.35</td>
</tr>
<tr>
<td valign="top" align="left">Moderate- expert (2)</td>
<td valign="top" align="center">1.54 &#x00B1; 0.28</td>
<td valign="top" align="center">1.53 &#x00B1; 0.26</td>
<td valign="top" align="center">1.46 &#x00B1; 0.35</td>
<td valign="top" align="center">1.44 &#x00B1; 0.33</td>
</tr>
<tr>
<td valign="top" align="left">Inexpert (3)</td>
<td valign="top" align="center">1.65 &#x00B1; 0.32</td>
<td valign="top" align="center">1.62 &#x00B1; 0.28</td>
<td valign="top" align="center">1.56 &#x00B1; 0.40</td>
<td valign="top" align="center">1.51 &#x00B1; 0.36</td>
</tr>
<tr>
<td valign="top" align="left">ICC (95% CI)</td>
<td valign="top" align="center">0.94 (0.80&#x2013;0.97)</td>
<td valign="top" align="center">0.95 (0.90&#x2013;0.97)</td>
<td valign="top" align="center">0.95 (0.89&#x2013;0.97)</td>
<td valign="top" align="center">0.97 (0.95&#x2013;0.98)</td>
</tr>
<tr>
<td valign="top" align="left"><bold>Categorical values</bold><break/> <bold>(<italic>n</italic> pathological/35)</bold> </td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td valign="top" align="left">Expert (1)</td>
<td valign="top" align="center">32</td>
<td valign="top" align="center">28</td>
<td valign="top" align="center">24</td>
<td valign="top" align="center">20</td>
</tr>
<tr>
<td valign="top" align="left">Moderate-expert (2)</td>
<td valign="top" align="center">31</td>
<td valign="top" align="center">31</td>
<td valign="top" align="center">24</td>
<td valign="top" align="center">26</td>
</tr>
<tr>
<td valign="top" align="left">Inexpert (3)</td>
<td valign="top" align="center">26</td>
<td valign="top" align="center">28</td>
<td valign="top" align="center">20</td>
<td valign="top" align="center">23</td>
</tr>
<tr>
<td valign="top" align="left">Fleiss Kappa (95% CI)</td>
<td valign="top" align="center">0.45 (0.44&#x2013;0.46)</td>
<td valign="top" align="center">0.78 (0.78&#x2013;0.79)</td>
<td valign="top" align="center">0.69 (0.69&#x2013;0.71)</td>
<td valign="top" align="center">0.78 (0.78&#x2013;0.79)</td>
</tr>
<tr>
<td valign="top" align="left">Cohen Kappa 1&#x2013;2</td>
<td valign="top" align="center">0.63</td>
<td valign="top" align="center">0.76</td>
<td valign="top" align="center">0.66</td>
<td valign="top" align="center">0.68</td>
</tr>
<tr>
<td valign="top" align="left">Cohen Kappa 1&#x2013;3</td>
<td valign="top" align="center">0.29</td>
<td valign="top" align="center">0.90</td>
<td valign="top" align="center">0.69</td>
<td valign="top" align="center">0.87</td>
</tr>
<tr>
<td valign="top" align="left">Cohen Kappa 2&#x2013;3</td>
<td valign="top" align="center">0.54</td>
<td valign="top" align="center">0.68</td>
<td valign="top" align="center">0.75</td>
<td valign="top" align="center">0.79</td>
</tr>
<tr>
<td valign="top" align="left"><bold>Intra-rater analysis</bold></td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td valign="top" align="left">Spearman coefficient 1</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.92, <italic>p</italic> = 0.000</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.92, <italic>p</italic> = 0.000</td>
</tr>
<tr>
<td valign="top" align="left">Spearman coefficient 2</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.86, <italic>p</italic> = 0.000</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.93, <italic>p</italic> = 0.000</td>
</tr>
<tr>
<td valign="top" align="left">Spearman coefficient 3</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.91, <italic>p</italic> = 0.000</td>
<td valign="top" align="center" colspan="2"><italic>p</italic> = 0.91, <italic>p</italic> = 0.000</td>
</tr>
</tbody>
</table>
</table-wrap>
<sec id="S3.SS1">
<title>The inter-rater agreements results</title>
<p>The raters showed excellent reliability measured by ICC in both methods (i.e., manual and semi-automatic) (<xref ref-type="table" rid="T1">Table 1</xref>). Of note, the agreement among raters tends to be higher in the semi-automatic procedure (ICC = 0.95 and ICC = 0.97, early and delayed H/M ratio, respectively) compared to the manual one (ICC = 0.94 and ICC = 0.95, early and delayed H/M ratio, respectively) (<xref ref-type="table" rid="T1">Table 1</xref>).</p>
<p>Fleiss&#x2019; kappa showed a moderate agreement in the raters&#x2019; classification of normal and pathological scores regarding early H/M ratio values (&#x03BA; = 0.45, <italic>p</italic> &#x003C; 0.0005) by using the manual method. Instead, the agreement was good when the early H/M ratio was obtained with the semi-automatic method (&#x03BA; = 0.78). Similarly, the late H/M ratios showed increased raters compliance moving from manual (&#x03BA; = 0.69) to semi-automatic approaches (&#x03BA; = 0.78). Of note, with the semi-automatic method, fewer PD patients fell under the category &#x201C;non-pathological&#x201D; for the inexpert and the moderate-expert raters (<xref ref-type="table" rid="T1">Table 1</xref>), with a more significant agreement among raters (<xref ref-type="table" rid="T1">Table 1</xref>). Specifically, the chi-square test found significant differences between the proportion of subjects classified as pathological or not by expert versus inexpert raters and moderate expert versus inexpert raters (<italic>p</italic> &#x003C; 0.05) only with the manual method. The differences were not significant between raters when they applied the semi-automatic method (<italic>p</italic> &#x003E; 0.05).</p>
<p>Cohen Kappa values revealed that applying a semi-automatic method instead of the manual one leads to an improvement in agreement, especially between expert and inexpert raters; the agreement moved from 0.29 (fair, in early H/M ratios) and 0.69 (substantial, in late H/M ratios) to 0.90 (perfect, in early H/M ratios) and 0.87 (perfect, late H/M ratios).</p>
</sec>
<sec id="S3.SS2">
<title>The intra-rater agreements results</title>
<p>The Spearman correlation analysis showed a robust intra-rater agreement in the assessment of early and delayed H/M values (Spearman Coefficient &#x003E; 0.80, <italic>p</italic> &#x003C; 0.001).</p>
</sec>
</sec>
<sec id="S4" sec-type="discussion">
<title>Discussion</title>
<p>This study investigated the effects of different methods assessing <sup>123</sup>I-MIBG parameters on the clinical interpretation of <sup>123</sup>I-MIBG images. To do this, we evaluated the reproducibility of delayed and early H/M ratios using two types of approach: (i) manual-drawing method and (ii) fixed-size ROIs approach following literature guidelines proposed for heart failure (<xref ref-type="bibr" rid="B13">13</xref>). Our results indicate that the use of fixed-size ROIs located following an &#x201C;anatomical landmark square&#x201D; (semi-automatic approach) improves the agreement among raters in classifying&#x2019; H/M ratios as PD pathological or non-pathological. Of note, the semi-automatic procedure produced a perfect agreement between experienced and inexperienced observers (<xref ref-type="table" rid="T1">Table 1</xref>).</p>
<p>The use of a rectangular mediastinal ROI with unspecified size and manually drawn cardiac ROI is commonly used (<xref ref-type="bibr" rid="B21">21</xref>). However, fixed-size ROIs are recommended in the proposal to standardise <sup>123</sup>I-MIBG parameters, but they were validated in patients with heart failure and without comparing the semi-automatic method with the standard visual one (<xref ref-type="bibr" rid="B13">13</xref>). Our results support and extend these findings in PD. The raters showed good inter and intra-rater agreements in early and late H/M ratios (continue values) applying manual-drawn and semiautomatic methods. This finding suggests that both methods for the calculation of the H/M ratios benefit from excellent reproducibility. However, the semi-automatic procedure showed a higher percentage of raters&#x2019; consistency, assessing both early and late H/M ratios (<xref ref-type="table" rid="T1">Table 1</xref>).</p>
<p>The classification of the H/M ratios into pathological or not according to clinical cut-off (<xref ref-type="bibr" rid="B16">16</xref>) further highlighted the differences in the approaches. The agreement among raters in classifying patients into pathological or not, according to H/M ratios measures (categorical values) was higher in the semi-automatic procedure than in the manual one (<xref ref-type="table" rid="T1">Table 1</xref>). This variation might be explained by the fact that the expert and inexpert raters showed low agreement, especially in assessing early H/M ratio values using the manual method (Cohen Kappa 0.29). Instead, in the semi-automatic procedure, the consistency between the raters became perfect (Cohen Kappa 0.90). This improvement suggests that the semi-automatic procedure improves the interpretation of planar <sup>123</sup>I-MIBG images in clinical settings.</p>
<p>This study has some limitations, particularly the small sample size and the lack of a control group. The data used in the present study have been acquired in the context of a protocol aiming to profile the dementia risk in PD with imaging biomarkers, where the acquisition of healthy controls was not planned. Moreover, we are aware that the superimposition of non-cardiac structures in the heart ROI might affect the <sup>123</sup>I-MIBG planar measures. This bias is intrinsic to the planar image technique because of the lack of three-dimensional information. A volumetric single photon emission computed tomography (SPECT) acquisition protocol would help to overcome this limitation, because of three-dimensional information. <sup>123</sup>I-MIBG-SPECT imaging can minimise the superimposition of the anatomic structures seen in two-dimensional planar images (e.g., lungs overlapping the heart) (<xref ref-type="bibr" rid="B22">22</xref>). We used the planar procedure in our study to obtain the <sup>123</sup>I-MIBG images because the quantification of global cardiac innervation using volumetric SPECT is less established than planar <sup>123</sup>I-MIBG imaging (<xref ref-type="bibr" rid="B23">23</xref>). This bias would equally affect the H/M ratios obtained with manual and semi-automatic ROI placement, most likely not impacting our study&#x2019;s main result.</p>
</sec>
<sec id="S5" sec-type="data-availability">
<title>Data availability statement</title>
<p>The raw data supporting the conclusions of this article will be made available by the authors, without undue reservation.</p>
</sec>
<sec id="S6">
<title>Ethics statement</title>
<p>The studies involving human participants were reviewed and approved by the Ethical Committees of IRCCS San Raffaele Hospital. The patients/participants provided their written informed consent to participate in this study.</p>
</sec>
<sec id="S7">
<title>Author contributions</title>
<p>CB and GC contributed to the conception and study design, statistical analysis, interpretation of results, and drafted the manuscript. EV contributed to the conception and study design and revised the manuscript. AC and AA provided clinical data and revised the manuscript. DP contributed to the study design, interpretation of results, and revision of the manuscript. VG contributed to the interpretation of results and revision of the manuscript. All authors read and approved the final manuscript.</p>
</sec>
</body>
<back>
<sec id="S8" sec-type="funding-information">
<title>Funding</title>
<p>This study was supported by the CARIPLO Project &#x201C;Evaluation of autonomic, genetic, imaging, and biochemical markers for Parkinson related dementia: Longitudinal assessment of a PD cohort&#x201D; 2016&#x2013;2020 (grant agreement no. 2014-0832) (DP). Open access funding was provided by the University of Geneva.</p>
</sec>
<sec id="S9" sec-type="COI-statement">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec id="S10" sec-type="disclaimer">
<title>Publisher&#x2019;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<ref-list>
<title>References</title>
<ref id="B1"><label>1.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Obeso</surname> <given-names>JA</given-names></name> <name><surname>Stamelou</surname> <given-names>M</given-names></name> <name><surname>Goetz</surname> <given-names>CG</given-names></name> <name><surname>Poewe</surname> <given-names>W</given-names></name> <name><surname>Lang</surname> <given-names>AE</given-names></name> <name><surname>Weintraub</surname> <given-names>D</given-names></name><etal/></person-group> <article-title>Past, present, and future of Parkinson&#x2019;s disease: a special essay on the 200<sup>th</sup> Anniversary of the Shaking Palsy.</article-title> <source><italic>Mov Disord.</italic></source> (<year>2017</year>). <volume>32</volume>:<fpage>1264</fpage>&#x2013;<lpage>310</lpage>. <pub-id pub-id-type="doi">10.1002/mds.27115</pub-id> <pub-id pub-id-type="pmid">28887905</pub-id></citation></ref>
<ref id="B2"><label>2.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Borghammer</surname> <given-names>P</given-names></name></person-group>. <article-title>How does parkinson&#x2019;s disease begin? Perspectives on neuroanatomical pathways, prions, and histology.</article-title> <source><italic>Mov Disord.</italic></source> (<year>2018</year>) <volume>33</volume>:<fpage>48</fpage>&#x2013;<lpage>57</lpage>. <pub-id pub-id-type="doi">10.1002/mds.27138</pub-id> <pub-id pub-id-type="pmid">28843014</pub-id></citation></ref>
<ref id="B3"><label>3.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Orimo</surname> <given-names>S</given-names></name> <name><surname>Ghebremedhin</surname> <given-names>E</given-names></name> <name><surname>Gelpi</surname> <given-names>E</given-names></name></person-group>. <article-title>Peripheral and central autonomic nervous system: does the sympathetic or parasympathetic nervous system bear the brunt of the pathology during the course of sporadic PD?</article-title> <source><italic>Cell Tissue Res.</italic></source> (<year>2018</year>) <volume>373</volume>:<fpage>267</fpage>&#x2013;<lpage>86</lpage>. <pub-id pub-id-type="doi">10.1007/s00441-018-2851-9</pub-id> <pub-id pub-id-type="pmid">29869180</pub-id></citation></ref>
<ref id="B4"><label>4.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Knudsen</surname> <given-names>K</given-names></name> <name><surname>Borghammer</surname> <given-names>P</given-names></name></person-group>. <article-title>Imaging the Autonomic Nervous System in Parkinson&#x2019;s Disease.</article-title> <source><italic>Curr Neurol Neurosci Rep.</italic></source> (<year>2018</year>) <volume>18</volume>:<fpage>79</fpage>. <pub-id pub-id-type="doi">10.1007/s11910-018-0889-4</pub-id> <pub-id pub-id-type="pmid">30232650</pub-id></citation></ref>
<ref id="B5"><label>5.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Postuma</surname> <given-names>RB</given-names></name> <name><surname>Berg</surname> <given-names>D</given-names></name> <name><surname>Stern</surname> <given-names>M</given-names></name> <name><surname>Poewe</surname> <given-names>W</given-names></name> <name><surname>Olanow</surname> <given-names>CW</given-names></name> <name><surname>Oertel</surname> <given-names>W</given-names></name><etal/></person-group> <article-title>MDS clinical diagnostic criteria for Parkinson&#x2019;s disease.</article-title> <source><italic>Mov Disord.</italic></source> (<year>2015</year>) <volume>30</volume>:<fpage>1591</fpage>&#x2013;<lpage>601</lpage>. <pub-id pub-id-type="doi">10.1002/mds.26424</pub-id> <pub-id pub-id-type="pmid">26474316</pub-id></citation></ref>
<ref id="B6"><label>6.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Wieland</surname> <given-names>DM</given-names></name> <name><surname>Brown</surname> <given-names>LE</given-names></name> <name><surname>Les Rogers</surname> <given-names>W</given-names></name> <name><surname>Worthington</surname> <given-names>KC</given-names></name> <name><surname>Wu</surname> <given-names>JL</given-names></name> <name><surname>Clinthorne</surname> <given-names>NH</given-names></name><etal/></person-group> <article-title>Myocardial imaging with a radioiodinated norepinephrine storage analog.</article-title> <source><italic>J Nucl Med.</italic></source> (<year>1981</year>) <volume>22</volume>:<fpage>22</fpage>&#x2013;<lpage>31</lpage>.</citation></ref>
<ref id="B7"><label>7.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sisson</surname> <given-names>JC</given-names></name> <name><surname>Wieland</surname> <given-names>DM</given-names></name></person-group>. <article-title>Radiolabeled meta-iodobenzylguanidine: pharmacology and clinical studies.</article-title> <source><italic>Am J Physiol Imaging.</italic></source> (<year>1986</year>) <volume>1</volume>:<fpage>96</fpage>&#x2013;<lpage>103</lpage>.</citation></ref>
<ref id="B8"><label>8.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Carri&#x00F3;</surname> <given-names>I</given-names></name></person-group>. <article-title>Cardiac neurotransmission imaging.</article-title> <source><italic>J Nucl Med.</italic></source> (<year>2001</year>) <volume>42</volume>:<fpage>1062</fpage>&#x2013;<lpage>76</lpage>.</citation></ref>
<ref id="B9"><label>9.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Orimo</surname> <given-names>S</given-names></name> <name><surname>Suzuki</surname> <given-names>M</given-names></name> <name><surname>Inaba</surname> <given-names>A</given-names></name> <name><surname>Mizusawa</surname> <given-names>H</given-names></name></person-group>. <article-title>123I-MIBG myocardial scintigraphy for differentiating Parkinson&#x2019;s disease from other neurodegenerative parkinsonism: a systematic review and meta-analysis.</article-title> <source><italic>Parkinsonism Relat Disord.</italic></source> (<year>2012</year>) <volume>18</volume>:<fpage>494</fpage>&#x2013;<lpage>500</lpage>. <pub-id pub-id-type="doi">10.1016/j.parkreldis.2012.01.009</pub-id> <pub-id pub-id-type="pmid">22321865</pub-id></citation></ref>
<ref id="B10"><label>10.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>King</surname> <given-names>AE</given-names></name> <name><surname>Mintz</surname> <given-names>J</given-names></name> <name><surname>Royall</surname> <given-names>DR</given-names></name></person-group>. <article-title>Meta-analysis of 123I-MIBG cardiac scintigraphy for the diagnosis of Lewy body&#x2013;related disorders.</article-title> <source><italic>Mov Disord.</italic></source> (<year>2011</year>) <volume>26</volume>:<fpage>1218</fpage>&#x2013;<lpage>24</lpage>. <pub-id pub-id-type="doi">10.1002/mds.23659</pub-id> <pub-id pub-id-type="pmid">21480373</pub-id></citation></ref>
<ref id="B11"><label>11.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nuvoli</surname> <given-names>S</given-names></name> <name><surname>Spanu</surname> <given-names>A</given-names></name> <name><surname>Fravolini</surname> <given-names>ML</given-names></name> <name><surname>Bianconi</surname> <given-names>F</given-names></name> <name><surname>Cascianelli</surname> <given-names>S</given-names></name> <name><surname>Madeddu</surname> <given-names>G</given-names></name><etal/></person-group> <article-title>[<sup>123</sup>I] Metaiodobenzylguanidine (MIBG) cardiac scintigraphy and automated classification techniques in parkinsonian disorders.</article-title> <source><italic>Mol Imaging Biol.</italic></source> (<year>2019</year>) <volume>22</volume>:<fpage>703</fpage>&#x2013;<lpage>10</lpage>. <pub-id pub-id-type="doi">10.1007/s11307-019-01406-6</pub-id> <pub-id pub-id-type="pmid">31309370</pub-id></citation></ref>
<ref id="B12"><label>12.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Roberts</surname> <given-names>G</given-names></name> <name><surname>Kane</surname> <given-names>JPM</given-names></name> <name><surname>Lloyd</surname> <given-names>JJ</given-names></name> <name><surname>Petrides</surname> <given-names>GS</given-names></name> <name><surname>Howe</surname> <given-names>K</given-names></name> <name><surname>O&#x2019;Brien</surname> <given-names>JT</given-names></name><etal/></person-group> <article-title>A comparison of visual and semiquantitative analysis methods for planar cardiac 123I-MIBG scintigraphy in dementia with Lewy bodies.</article-title> <source><italic>Nucl Med Commun.</italic></source> (<year>2019</year>) <volume>40</volume>:<fpage>734</fpage>&#x2013;<lpage>43</lpage>. <pub-id pub-id-type="doi">10.1097/MNM.0000000000001024</pub-id> <pub-id pub-id-type="pmid">31058746</pub-id></citation></ref>
<ref id="B13"><label>13.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Veltman</surname> <given-names>CE</given-names></name> <name><surname>Boogers</surname> <given-names>MJ</given-names></name> <name><surname>Meinardi</surname> <given-names>JE</given-names></name> <name><surname>Al Younis</surname> <given-names>I</given-names></name> <name><surname>Dibbets-Schneider</surname> <given-names>P</given-names></name> <name><surname>Van derWall</surname> <given-names>EE</given-names></name><etal/></person-group> <article-title>Reproducibility of planar 123I-meta-iodobenzylguanidine (MIBG) myocardial scintigraphy in patients with heart failure.</article-title> <source><italic>Eur J Nucl Med Mol Imaging.</italic></source> (<year>2012</year>) <volume>39</volume>:<fpage>1599</fpage>&#x2013;<lpage>608</lpage>. <pub-id pub-id-type="doi">10.1007/s00259-012-2180-2</pub-id> <pub-id pub-id-type="pmid">22790878</pub-id></citation></ref>
<ref id="B14"><label>14.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Flotats</surname> <given-names>A</given-names></name> <name><surname>Carri&#x00F3;</surname> <given-names>I</given-names></name> <name><surname>Agostini</surname> <given-names>D</given-names></name> <name><surname>Le Guludec</surname> <given-names>D</given-names></name> <name><surname>Marcassa</surname> <given-names>C</given-names></name> <name><surname>Schaffers</surname> <given-names>M</given-names></name><etal/></person-group> <article-title>Proposal for standardisation of 123 I-metaiodobenzylguanidine (MIBG) cardiac sympathetic imaging by the EANM Cardiovascular Committee and the European Council of Nuclear Cardiology.</article-title> <source><italic>Eur J Nucl Med Mol Imaging.</italic></source> (<year>2010</year>) <volume>37</volume>:<fpage>1802</fpage>&#x2013;<lpage>12</lpage>. <pub-id pub-id-type="doi">10.1007/s00259-010-1491-4</pub-id> <pub-id pub-id-type="pmid">20577740</pub-id></citation></ref>
<ref id="B15"><label>15.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Koo</surname> <given-names>TK</given-names></name> <name><surname>Li</surname> <given-names>MY</given-names></name></person-group>. <article-title>A guideline of selecting and reporting intraclass correlation coefficients for reliability research.</article-title> <source><italic>J Chiropr Med.</italic></source> (<year>2016</year>) <volume>15</volume>:<fpage>155</fpage>&#x2013;<lpage>63</lpage>. <pub-id pub-id-type="doi">10.1016/j.jcm.2016.02.012</pub-id> <pub-id pub-id-type="pmid">27330520</pub-id></citation></ref>
<ref id="B16"><label>16.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nagamachi</surname> <given-names>S</given-names></name> <name><surname>Wakamatsu</surname> <given-names>H</given-names></name> <name><surname>Kiyohara</surname> <given-names>S</given-names></name> <name><surname>Fujita</surname> <given-names>S</given-names></name> <name><surname>Futami</surname> <given-names>S</given-names></name> <name><surname>Tamura</surname> <given-names>S</given-names></name><etal/></person-group> <article-title>Usefulness of rCBF analysis in diagnosing Parkinson&#x2019;s disease: supplemental role with MIBG myocardial scintigraphy.</article-title> <source><italic>Ann Nucl Med.</italic></source> (<year>2008</year>) <volume>22</volume>:<fpage>557</fpage>&#x2013;<lpage>64</lpage>. <pub-id pub-id-type="doi">10.1007/s12149-008-0156-x</pub-id> <pub-id pub-id-type="pmid">18756357</pub-id></citation></ref>
<ref id="B17"><label>17.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fleiss</surname> <given-names>JL</given-names></name></person-group>. <article-title>Measuring nominal scale agreement among many raters.</article-title> <source><italic>Psychol Bull.</italic></source> (<year>1971</year>) <volume>76</volume>:<fpage>378</fpage>. <pub-id pub-id-type="doi">10.1037/h0031619</pub-id></citation></ref>
<ref id="B18"><label>18.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fleiss</surname> <given-names>JL</given-names></name> <name><surname>Levin</surname> <given-names>B</given-names></name> <name><surname>Paik</surname> <given-names>MC.</given-names></name></person-group> <source><italic>Statistical Methods for Rates and Proportions.</italic></source> <publisher-loc>Hoboken, NJ</publisher-loc>: <publisher-name>John wiley and sons</publisher-name> (<year>2013</year>).</citation></ref>
<ref id="B19"><label>19.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Landis</surname> <given-names>JR</given-names></name> <name><surname>Koch</surname> <given-names>GG</given-names></name></person-group>. <article-title>The measurement of observer agreement for categorical data.</article-title> <source><italic>Biometrics.</italic></source> (<year>1977</year>) <volume>33</volume>:<fpage>159</fpage>&#x2013;<lpage>74</lpage>. <pub-id pub-id-type="doi">10.2307/2529310</pub-id></citation></ref>
<ref id="B20"><label>20.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>McHugh</surname> <given-names>ML</given-names></name></person-group>. <article-title>Interrater reliability: the kappa statistic.</article-title> <source><italic>Biochem Med.</italic></source> (<year>2012</year>) <volume>22</volume>:<fpage>276</fpage>&#x2013;<lpage>82</lpage>. <pub-id pub-id-type="doi">10.11613/BM.2012.031</pub-id></citation></ref>
<ref id="B21"><label>21.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>van der Veen</surname> <given-names>L</given-names></name> <name><surname>Scholte</surname> <given-names>A</given-names></name> <name><surname>Stokkel</surname> <given-names>M</given-names></name></person-group>. <article-title>Mathematical methods to determine quantitative parameters of myocardial 123I-MIBG studies: a review of the literature.</article-title> <source><italic>Nucl Med Commun.</italic></source> (<year>2010</year>) <volume>31</volume>:<fpage>617</fpage>&#x2013;<lpage>28</lpage>. <pub-id pub-id-type="doi">10.1097/MNM.0b013e328337a99b</pub-id> <pub-id pub-id-type="pmid">20220545</pub-id></citation></ref>
<ref id="B22"><label>22.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Oh</surname> <given-names>J-K</given-names></name> <name><surname>Choi</surname> <given-names>E-K</given-names></name> <name><surname>Song</surname> <given-names>I-U</given-names></name> <name><surname>Kim</surname> <given-names>J-S</given-names></name> <name><surname>Chung</surname> <given-names>Y-A</given-names></name></person-group>. <article-title>Comparison of I-123 MIBG planar imaging and SPECT for the detection of decreased heart uptake in Parkinson disease.</article-title> <source><italic>J Neural Trans.</italic></source> (<year>2015</year>) <volume>122</volume>:<fpage>1421</fpage>&#x2013;<lpage>7</lpage>. <pub-id pub-id-type="doi">10.1007/s00702-015-1409-1</pub-id> <pub-id pub-id-type="pmid">26003179</pub-id></citation></ref>
<ref id="B23"><label>23.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>van der Veen</surname> <given-names>BJ</given-names></name> <name><surname>Al Younis</surname> <given-names>I</given-names></name> <name><surname>de Roos</surname> <given-names>A</given-names></name> <name><surname>Stokkel</surname> <given-names>MPM</given-names></name></person-group>. <article-title>Assessment of global cardiac I-123 MIBG uptake and washout using volumetric quantification of SPECT acquisitions.</article-title> <source><italic>J Nucl Cardiol.</italic></source> (<year>2012</year>) <volume>19</volume>:<fpage>752</fpage>&#x2013;<lpage>62</lpage>. <pub-id pub-id-type="doi">10.1007/s12350-012-9539-4</pub-id> <pub-id pub-id-type="pmid">22669736</pub-id></citation></ref>
</ref-list>
</back>
</article>