<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" article-type="research-article" dtd-version="2.3" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Psychol.</journal-id>
<journal-title>Frontiers in Psychology</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Psychol.</abbrev-journal-title>
<issn pub-type="epub">1664-1078</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fpsyg.2024.1410396</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Psychology</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Predicting implementation of response to intervention in math using elastic net logistic regression</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name><surname>Wang</surname> <given-names>Qi</given-names></name>
<uri xlink:href="https://loop.frontiersin.org/people/2672191/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/formal-analysis/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/visualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author" corresp="yes" equal-contrib="yes">
<name><surname>Hall</surname> <given-names>Garret J.</given-names></name>
<xref ref-type="corresp" rid="c001"><sup>&#x002A;</sup></xref>
<xref ref-type="author-notes" rid="fn0003"><sup>&#x2020;</sup></xref>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/funding-acquisition/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author" corresp="yes" equal-contrib="yes">
<name><surname>Zhang</surname> <given-names>Qian</given-names></name>
<xref ref-type="corresp" rid="c002"><sup>&#x002A;</sup></xref>
<xref ref-type="author-notes" rid="fn0003"><sup>&#x2020;</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/1112777/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/funding-acquisition/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Comella</surname> <given-names>Sara</given-names></name>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
</contrib-group>
<aff><institution>Department of Educational Psychology and Learning Systems, College of Education, Health, and Human Sciences, Florida State University</institution>, <addr-line>Tallahassee, FL</addr-line>, <country>United States</country></aff>
<author-notes>
<fn fn-type="edited-by" id="fn0004">
<p>Edited by: Chong Ho Yu, Azusa Pacific University, United States</p>
</fn>
<fn fn-type="edited-by" id="fn0005">
<p>Reviewed by: Seongah Im, University of Hawaii at Manoa, United States</p>
<p>Jiacong Du, University of Michigan, United States</p>
</fn>
<corresp id="c001">&#x002A;Correspondence: Garret J. Hall, <email>gjhall@fsu.edu</email></corresp>
<corresp id="c002">Qian Zhang, <email>qzhang4@fsu.edu</email></corresp>
<fn fn-type="equal" id="fn0003">
<p><sup>&#x2020;</sup>These authors have contributed equally to this work</p>
</fn>
</author-notes>
<pub-date pub-type="epub">
<day>02</day>
<month>10</month>
<year>2024</year>
</pub-date>
<pub-date pub-type="collection">
<year>2024</year>
</pub-date>
<volume>15</volume>
<elocation-id>1410396</elocation-id>
<history>
<date date-type="received">
<day>01</day>
<month>04</month>
<year>2024</year>
</date>
<date date-type="accepted">
<day>04</day>
<month>09</month>
<year>2024</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x00A9; 2024 Wang, Hall, Zhang and Comella.</copyright-statement>
<copyright-year>2024</copyright-year>
<copyright-holder>Wang, Hall, Zhang and Comella</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p>
</license>
</permissions>
<abstract>
<sec>
<title>Introduction</title>
<p>The primary objective of this study was to identify variables that significantly influence the implementation of math Response to Intervention (RTI) at the school level, utilizing the ECLS-K: 2011 dataset.</p>
</sec>
<sec>
<title>Methods</title>
<p>Due to missing values in the original dataset, a Random Forest algorithm was employed for data imputation, generating a total of 10 imputed datasets. Elastic net logistic regression, combined with nested cross-validation, was applied to each imputed dataset, potentially resulting in 10 models with different variables. Variables for the models derived from the imputed datasets were selected using four methods, leading to four candidate models for final selection. These models were assessed based on their performance of prediction accuracy, culminating in the selection of the final model that outperformed the others.</p>
</sec>
<sec>
<title>Results and discussion</title>
<p>Method<sub>50</sub> and Method<sub>coef</sub> emerged as the most effective, achieving a balanced accuracy of 0.852. The ultimate model selected relevant variables that effectively predicted RTI. The predictive accuracy of the final model was also demonstrated by the receiver operating characteristic (ROC) plot and the corresponding area under the curve (AUC) value, indicating its ability to accurately forecast math RTI implementation in schools for the following year.</p>
</sec>
</abstract>
<kwd-group>
<kwd>math achievement</kwd>
<kwd>response-to-intervention</kwd>
<kwd>elastic net logistic regression</kwd>
<kwd>multiple imputation</kwd>
<kwd>random forest algorithm</kwd>
<kwd>variable selection</kwd>
</kwd-group>
<contract-sponsor id="cn1">American Educational Research Association<named-content content-type="fundref-id">10.13039/100005165</named-content></contract-sponsor>
<contract-sponsor id="cn2">National Science Foundation<named-content content-type="fundref-id">10.13039/100000001</named-content></contract-sponsor>
<counts>
<fig-count count="3"/>
<table-count count="6"/>
<equation-count count="3"/>
<ref-count count="50"/>
<page-count count="15"/>
<word-count count="11695"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Quantitative Psychology and Measurement</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="sec1">
<title>Introduction</title>
<p>Despite the well-known recommendations for practice regarding the use of response-to-intervention (RTI; often, but not always, synonymous with multi-tiered systems of support; <xref ref-type="bibr" rid="ref3">Burns et al., 2016</xref>) to prevent and intervene on academic difficulties, there remains surprisingly little evidence on what factors help understand schools&#x2019; adoption of RTI in math. Although there is inconclusive evidence on the effectiveness of RTI as a prevention and intervention system (particularly in reading, e.g., <xref ref-type="bibr" rid="ref1">Balu et al., 2015</xref>), RTI (and tiered intervention models in general) is well-established in the best practices for making data-based decisions on intervention need and preventing school-wide academic difficulties (<xref ref-type="bibr" rid="ref25">Lane et al., 2019</xref>; <xref ref-type="bibr" rid="ref30">McIntosh and Goodman, 2016</xref>; <xref ref-type="bibr" rid="ref41">Schulte, 2016</xref>). Thus, it is necessary to better understand the conditions that predict schools&#x2019; adoption of RTI, especially in math, which has historically lagged reading in RTI research. In this study, we investigate the predictors of elementary schools&#x2019; adoption of math RTI using school-level data and school personnel surveys from a large, United States national dataset (Early Childhood Longitudinal Study: Kindergarten Cohort 2010-11; ECLS-K: 2011). Our study uses novel machine learning and cross-validation methods to identify the most prominent predictors of schools&#x2019; RTI adoption.</p>
<sec id="sec2">
<title>What is RTI?</title>
<p>Response to intervention is a tiered model of intervention service delivery for academic skills. The prototypical RTI model encompasses three tiers of services, where Tier 1 (universal supports) refers to the general education curriculum that all students receive, Tier 2 refers to supplemental intensified interventions that support Tier 1 learning (e.g., small group reading interventions twice a week), and Tier 3 refers to intensive individualized and frequent (4&#x2013;5 times a week) intervention designed to remediate substantial learning difficulties (<xref ref-type="bibr" rid="ref31">Mellard et al., 2010</xref>). Together, RTI&#x2019;s approach to implementation, prevention, and intervention is heavily rooted in public health prevention models (<xref ref-type="bibr" rid="ref41">Schulte, 2016</xref>). RTI has been frequently contrasted with &#x201C;wait-to-fail&#x201D; models of service delivery where intensive intervention might not be provided until a student is referred for special education for a learning disability (though RTI has historically demonstrated its own shortcomings; <xref ref-type="bibr" rid="ref39">Reynolds and Shaywitz, 2009</xref>). To this end, in addition to the multiple tiers of RTI providing a continuum of supports for all students, RTI also provides a mechanism to identify students that do not respond to the general supports available, which may open the potential for special education evaluation for a specific learning disability (<xref ref-type="bibr" rid="ref3">Burns et al., 2016</xref>). RTI is typically applied in reading, math, and writing subject areas, and more recent conceptualizations of RTI integrate this approach with tiered social&#x2013;emotional-behavioral supports (<xref ref-type="bibr" rid="ref25">Lane et al., 2019</xref>; <xref ref-type="bibr" rid="ref30">McIntosh and Goodman, 2016</xref>).</p>
<p>The nature of RTI warrants a school-wide implementation approach to build systems capacity for tiered service provision, including data collection (e.g., screening and progress monitoring), intervention material development/curation, implementation, and sustainment. Thus, while specific interventions <italic>within</italic> RTI may be implemented for small groups (e.g., Tier 2) or individuals (e.g., Tier 3), the overall model operates as a school-wide effort of data-based decision making based on screening and progress monitoring (including entry and exit criteria for intervention), intervention planning, development, and provision. Moreover, the public health approach that emphasizes population-wide &#x201C;inoculation&#x201D; mechanisms at tier 1 to increase the efficiency and efficacy of more intensive interventions intends to promote school effectiveness and achievement on a broad scale to mitigate the incidence of compounding academic difficulties (<xref ref-type="bibr" rid="ref31">Mellard et al., 2010</xref>).</p>
</sec>
<sec id="sec3">
<title>Limitations of current math RTI implementation research</title>
<p>Despite the systems-level focus of RTI and the popularity of this model as a prevention system in K-12 schools, there is a dearth of research on the factors that relate to whether schools use RTI. Specifically, this is the case with math, as reading has traditionally predominated in schools&#x2019; RTI practices. Although there are some case studies and qualitative inquiry into the implementation of math RTI models (<xref ref-type="bibr" rid="ref2">Bouck and Cosby, 2019</xref>; <xref ref-type="bibr" rid="ref11">Donovan and Shepherd, 2013</xref>; <xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>) as well as limited empirical investigation (<xref ref-type="bibr" rid="ref42">Schumacher et al., 2017</xref>), the empirical research on factors predicting the use of RTI is lacking. This is a significant limitation in understanding the systematic factors that may determine whether schools choose to adopt an RTI model for prevention and intervention in math. The system capacity to adopt parallel RTI models for math may be limited if schools are already devoting substantial resources to other programs and initiatives (<xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>). However, other contextual factors may play a part in the adoption of math RTI, including school resources, community contextual factors, and personnel factors (e.g., staff training background; <xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>).</p>
</sec>
<sec id="sec4">
<title>Identifying predictors</title>
<p>With seemingly innumerable factors that could possibly relate to whether schools use math RTI, <italic>a priori</italic> selection of specific predictive factors may induce bias in predictions or unintentionally limit the scope of predictive factors. Compounding the identification of relevant predictors is the limited research base on school factors associated with math RTI implementation, aside from factors that would likely be related to the time and resources schools have available to conduct math RTI (e.g., economic conditions, community context, and student-related issues that guide service priorities). Consequently, narrowing the scope of predictors is an exploratory task at this stage.</p>
<p>Due to the large number of predictors and the relatively small sample size, elastic net logistic regression with nested cross-validation (nested CV) is a viable method to address the issues of predictor selection. This choice was motivated by the potential for overfitting when using traditional logistic regression (<xref ref-type="bibr" rid="ref20">Hastie et al., 2006</xref>). The elastic net logistic regression can effectively address this risk by imposing penalties on the predictors, effectively reducing their number. This strategy not only alleviates concerns related to overfitting but also enhances the model&#x2019;s predictive accuracy (<xref ref-type="bibr" rid="ref18">Hans, 2011</xref>; <xref ref-type="bibr" rid="ref54">Zou and Hastie, 2005</xref>). Therefore, determining the appropriate penalties is crucial when using elastic net regression. The most commonly used method to select these penalties is cross-validation, with <italic>K</italic>-fold cross-validation (CV) being particularly popular. However, <xref ref-type="bibr" rid="ref50">Vabalas et al. (2019)</xref> showed that <italic>K</italic>-fold CV would inflate the accuracy of a regularized logistic regression with feature selection, especially when the sample size was small. Hence, they recommended using nested CV which could produce an accurate misclassification rate, even with a small sample size.</p>
</sec>
<sec id="sec5">
<title>The current study</title>
<p>In this study, we used data from Grades 1 (2011&#x2013;2012) and 2 (2012&#x2013;2013) of the ECLS-K: 2011 to investigate the school-level factors in school year (SY) 2011&#x2013;2012 that predicted elementary schools&#x2019; adoption of math RTI based on school administrator reports in SY2012-2013. Our primary research question was the following: What are the prominent predictors of schools&#x2019; math RTI implementation in the subsequent year? We hypothesized that factors related to school resources, achievement, and school personnel (e.g., training) would predominate as math RTI implementation predictors, as prior research has suggested that resource (e.g., staff training) and logistical factors (e.g., available time for implementation) are relevant to the adoption and implementation of math RTI (<xref ref-type="bibr" rid="ref6">Choi et al., 2022</xref>; <xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>). We used regularized regression techniques with nested CV to identify the most robust predictors. Due to missing values in this dataset, we initially used the Random Forest algorithm to create different imputed datasets. Subsequently, we applied elastic net logistic regression with nested CV to build prediction models for each imputed dataset. Since the penalties in elastic net logistic regression were evaluated separately in each dataset, this resulted in different penalties across the datasets, leading to variations in the logistic regression models. To address this, we employed four variable selection methods to identify the most robust predictors across the various elastic net logistic regression models from different imputed datasets, forming four candidate models. Then, four candidate models were evaluated using an independent dataset. The candidate model that exhibited the best performance among the four was selected as the final model. The flow chart of the data analysis procedure for this study is depicted in <xref ref-type="fig" rid="fig1">Figure 1</xref>.</p>
<fig position="float" id="fig1">
<label>Figure 1</label>
<caption>
<p>Data analysis procedure.</p>
</caption>
<graphic xlink:href="fpsyg-15-1410396-g001.tif"/>
</fig>
</sec>
</sec>
<sec sec-type="methods" id="sec6">
<title>Methods</title>
<sec id="sec7">
<title>Participating schools</title>
<p>The ECLS-K: 2011 is a multi-stage stratified nationally representative sample of United States students from kindergarten through fifth grade. The study initially samples base-year students in kindergarten, and students are followed through fifth grade with procedures to ensure the national representativeness of the student samples at each grade level. In addition, kindergarten schools are sampled to be nationally representative; all schools in the following years are representative of the students in the sample attending those schools (rather than being intrinsically nationally representative like the base year). We limit our sample of schools to public schools present in both the Grades 1 and 2 Spring data collection rounds. These grades represent the first period in the dataset in which RTI implementation data is collected from schools, so we choose these grades to avoid additional temporal dependency in the measurement of RTI in grades 3&#x2013;5. In addition, given the potential increase in the use of RTI over time (or changes in the uses of RTI over time), using data from earlier years (2012&#x2013;2013) represents an earlier stage in the uptake of RTI practices and thus includes potentially more variability in how/why schools chose to use math RTI.</p>
</sec>
<sec id="sec8">
<title>Measures</title>
<sec id="sec9">
<title>Outcome</title>
<sec id="sec10">
<title>School administrator report of RTI implementation</title>
<p>The outcome of our study is the school administrator&#x2019;s report of whether their school implemented math RTI in the Spring of 2012&#x2013;2013 (the Grade 2 round of data collection at the child level). This data were collected as part of the school administrator survey portion of the ECLS-K: 2011. Responses were recorded as &#x201C;not applicable&#x201D; (e.g., schools in which RTI would not have been implemented), &#x201C;no,&#x201D; &#x201C;partially implemented,&#x201D; or &#x201C;fully implemented.&#x201D; We collapse partial and full implementation into a single category, resulting in a binary 0 (no)/1 (yes) indicator of RTI. It is unclear what exactly would differentiate &#x201C;partial&#x201D; vs. &#x201C;full&#x201D; implementation in this survey, so we focus on the presence of any math RTI practices rather than a gradient of implementation (e.g., different forms of partial implementation could exist but there is no way to determine this, which reduces the practical value of differentiating the responses).</p>
</sec>
</sec>
<sec id="sec11">
<title>Predictors</title>
<p>We selected a thorough set of predictors that represented contextual (e.g., economic, community), student (e.g., enrollment, reports of safety issues), personnel factors (e.g., teacher training), and implementation factors (e.g., previous implementation of behavior and writing RTI) to cover the reasonable potential range of reasons schools may adopt math RTI. A sample of the predictors is displayed in <xref ref-type="table" rid="tab1">Table 1</xref>. All predictors are from the Spring semester of Grade 1 and have been aggregated at the school level. The mean was utilized for continuous variables for aggregation purposes, while the mode (we focus on the most frequent category within a school) was employed for categorical variables. The mean was chosen for aggregating continuous variables because it is the most commonly used measure to describe central tendency in continuous data (<xref ref-type="bibr" rid="ref8">Clowes and Duke, 2022</xref>). This method is also widely utilized in applied research for aggregating such variables (e.g., <xref ref-type="bibr" rid="ref16">Geronimus and Bound, 1998</xref>; <xref ref-type="bibr" rid="ref21">Jacob et al., 2014</xref>; <xref ref-type="bibr" rid="ref32">Moineddin and Urquia, 2014</xref>). For categorical variables, the mode was selected because it is often the most appropriate, and sometimes the only, method to effectively describe the central tendency for nominal variables (<xref ref-type="bibr" rid="ref8">Clowes and Duke, 2022</xref>). The outliers in our dataset may not pose a problem, as most continuous variables we used are at the school level, meaning that students within the same school share the same value. However, one variable, X4SESL_I, which is the continuous Socioeconomic Status (SES) index, may have potential outliers. Therefore, we checked for outliers in X4SESL_I using Z-scores for each school with at least 10 observations. If a <italic>Z</italic>-score exceeded 3, the observation was identified as an outlier. The results showed that only one school (id = 1,816) had a single outlier, while other schools did not have any outliers.</p>
<table-wrap position="float" id="tab1">
<label>Table 1</label>
<caption>
<p>Covariates domains considered for RTI prediction.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th align="left" valign="top">Covariate block</th>
<th align="left" valign="top">Covariate description</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top">School demographic factors</td>
<td align="left" valign="top">School-level sociodemographic factors, including funding mechanisms (Title 1 and 3), poverty, geographic locale (rural, suburban, etc.), changes in enrollment/funding/staffing/class sizes, school-level race/ethnicity, and language status</td>
</tr>
<tr>
<td align="left" valign="top">Policy or procedural features</td>
<td align="left" valign="top">School policies and procedures for behavioral and academic intervention (e.g., implementation of other tiered systems for behavior and writing, staff procedures for implementing interventions and using data to make decisions, and professional development of staff)</td>
</tr>
<tr>
<td align="left" valign="top">Staffing and administration</td>
<td align="left" valign="top">General school policies related to staffing and administrative procedures (e.g., academic standards, resource allocation, staffing, school-level procedures for teaching such as using achievement groupings)</td>
</tr>
<tr>
<td align="left" valign="top">Areas of concern for school</td>
<td align="left" valign="top">Student problems at school (e.g., reports of whether the school has problems with weapons, theft, classroom disorder, and absenteeism)</td>
</tr>
<tr>
<td align="left" valign="top">Community engagement</td>
<td align="left" valign="top">Community engagement with the school (e.g., before and after-school care, having parent teacher conferences, community support)</td>
</tr>
</tbody>
</table>
</table-wrap>
</sec>
</sec>
<sec id="sec12">
<title>Procedure</title>
<p>Public-use ECLS-K: 2011 data were downloaded from the National Center for Education Statistics website. Data cleaning and analysis took place in Stata 17 (<xref ref-type="bibr" rid="ref9004">StataCorp, 2021</xref>) and R (<xref ref-type="bibr" rid="ref9003">R Core Team, 2021</xref>). To be specific, elastic net logistic regression with nested CV was conducted by using <italic>nestcv</italic> package in R (<xref ref-type="bibr" rid="ref27">Lewis et al., 2023</xref>). In addition, missing value imputation carried out with the <italic>missForest</italic> package in R (<xref ref-type="bibr" rid="ref44">Stekhoven and B&#x00FC;hlmann, 2012</xref>; <xref ref-type="bibr" rid="ref9001">Stekhoven, 2013</xref>).</p>
</sec>
<sec id="sec13">
<title>Data analysis</title>
<sec id="sec14">
<title>Multiple imputation for missing data</title>
<p>In this study, the percentage of missing values is 16.4%. <xref ref-type="bibr" rid="ref28">Little and Rubin (2019)</xref> described three missing data mechanisms: Missing Completely at Random (MCAR), Missing at Random (MAR), and Missing Not at Random (MNAR). MCAR situations are characterized by missing values that do not correlate with either observed or unobserved variables. MAR, on the other hand, describes instances where missing values are linked to observed variables but not to unobserved ones. MNAR pertains to cases where missing values are related to unobserved variables. We assume that the missingness present among school-level variables in this dataset is MCAR or MAR given that much of the data comes from administrative information about schools or information that is aggregated to the school level. We would not expect missing data to be related to endogenous, unobserved factors about the schools themselves (MNAR). Rather, we expect that missing responses would be due to random survey nonresponse or non-response related to other systematic factors for which we have information (e.g., other school and teacher characteristics).</p>
<p>Given MAR data, it was necessary to employ an imputation method to address these missing values (<xref ref-type="bibr" rid="ref28">Little and Rubin, 2019</xref>). We employed the Random Forest (RF) algorithm for multiple imputation. This choice was made because the RF algorithm (1) is well-suited for data missing at random, (2) can effectively handle both continuous and categorical variables, and most importantly, (3) does not require parametric forms and can effectively account for any non-linear relationships, complex interactions, and high dimensionality in the imputation model (<xref ref-type="bibr" rid="ref44">Stekhoven and B&#x00FC;hlmann, 2012</xref>).</p>
<p>The last advantage of RF is theoretically shared by other nonparametric machine-learning methods. We selected RF based on existing comparisons of RF against other parametric and nonparametric machine-learning imputation methods. <xref ref-type="bibr" rid="ref36">Pantanowitz and Marwala (2009)</xref> conducted an analysis using empirical data to compare five imputation methods: RF, Autoassociative Neural Network, Autoassociative Adaptive Neuro-Fuzzy Inference System, a hybrid of Random Forest and Autoassociative Neural Network, and a hybrid of Autoassociative Neural Network and Random Forest. Their findings revealed that Random Forest outperforms the other methods in terms of both accuracy and computational efficiency. Similarly, <xref ref-type="bibr" rid="ref44">Stekhoven and B&#x00FC;hlmann (2012)</xref> explored four different imputation techniques: RF, <italic>k</italic>-Nearest Neighbors algorithm, Missingness Pattern Alternating Imputation and <italic>l</italic><sub>1</sub>-penalty algorithm, and Multivariate Imputation by Chained Equations, across various empirical datasets. Their findings also revealed that RF had better performance than the other methods generally, particularly in datasets containing both continuous and categorical variables. <xref ref-type="bibr" rid="ref47">Tang and Ishwaran (2017)</xref> conducted a simulation study to evaluate the performance of various Random Forest (RF) algorithms under three missing data mechanisms. Their findings revealed that RF algorithms generally performed well when data were MCAR or MAR, and the proportion of missing data was low to moderate.</p>
<p>Furthermore, RF managed to maintain acceptable performance in MNAR conditions when the variables were highly correlated (<xref ref-type="bibr" rid="ref47">Tang and Ishwaran, 2017</xref>). To maximize the information contained in the datasets and to capture possible relationships between missing values and other variables, all variables were used for imputation. This further makes the imputation model robust to account for variables responsible for missingness. Additionally, an independent complete dataset was utilized to evaluate the generalizability of the models, which also served as a sensitivity analysis for missing data imputation.</p>
<p>The RF imputation process was carried out 10 times, resulting in a total of 10 datasets (<xref ref-type="bibr" rid="ref34">Musoro et al., 2014</xref>; <xref ref-type="bibr" rid="ref40">Rubin, 1987</xref>; <xref ref-type="bibr" rid="ref34">Musoro et al., 2014</xref>; <xref ref-type="bibr" rid="ref53">Zahid et al., 2020</xref>). Generally, the RF imputation employed in this study involves modeling each variable with missing values as a function of all other available variables, with the missing values being predicted using a fitted random forest model. Specifically, an initial guess for the missing values, such as the mean, is made. Variables are then sorted by their percentage of missing values, and the one with the fewest missing values is imputed first. This imputed variable is treated as the response variable, with the others serving as predictors. An RF model is constructed to predict the missing values of this variable, and these missing values are updated by the RF-model-based predicted values. Then, the variable having the next fewest missing data is imputed based on the already imputed variable and others, following the same procedure until the stopping criteria are met. The stopping criteria involve observing an increase in the difference between the newly imputed values and the previously imputed values across observations for the first time, at which point the RF algorithm is stopped. Importantly, this increase should be observed across all types of variables. This non-parametric method, detailed by <xref ref-type="bibr" rid="ref44">Stekhoven and B&#x00FC;hlmann (2012)</xref>, is effective regardless of whether the missing values occur in independent or outcome variables.</p>
<p>When conducting the RF imputation, missing values in the two variables &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; were not imputed, as they represent the strata and primary sampling units (PSU) from the complex survey design. Due to their nature, these variables were not suitable for imputation using the RF algorithm like the other variables. To address missing values of these two variables while retaining as many observations as possible, a value of 0 was assigned to indicate &#x201C;not specified&#x201D; for both variables. Although &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; cannot be directly imputed by the RF algorithm, they were included in the RF imputation process to facilitate the imputation of other variables. We chose to include these sampling variables in the RF algorithm because including them in the model allows for control over the sampling design for the ECLS-K dataset (<xref ref-type="bibr" rid="ref43">Stapleton and Kang, 2018</xref>).</p>
</sec>
<sec id="sec15">
<title>Elastic net logistic regression with nested cross-validation under four methods of variable selection</title>
<p>After completing the imputation phase, we conducted elastic net logistic regression analyses using nested CV with 5-fold on each imputed dataset. The elastic net regression was proposed by <xref ref-type="bibr" rid="ref54">Zou and Hastie (2005)</xref>, which is expressed as:</p>
<disp-formula id="E1">
<mml:math id="M3">
<mml:mo>arg</mml:mo>
<mml:mo>min</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mrow>
<mml:mn>2</mml:mn>
<mml:mi>n</mml:mi>
</mml:mrow>
</mml:mfrac>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>N</mml:mi>
</mml:munderover>
<mml:msup>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mi>y</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mn>0</mml:mn>
</mml:msub>
<mml:mo>+</mml:mo>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:munderover>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mn>2</mml:mn>
</mml:msup>
<mml:mo>+</mml:mo>
<mml:mi>&#x03BB;</mml:mi>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:mfrac>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>&#x03B1;</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mn>2</mml:mn>
</mml:mfrac>
<mml:msubsup>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>2</mml:mn>
<mml:mn>2</mml:mn>
</mml:msubsup>
<mml:mo>+</mml:mo>
<mml:mi>&#x03B1;</mml:mi>
<mml:msub>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
</mml:math>
</disp-formula>
<p>where <inline-formula>
<mml:math id="M4">
<mml:msub>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>1</mml:mn>
</mml:msub>
<mml:mo>=</mml:mo>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:munderover>
<mml:mo stretchy="true">|</mml:mo>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
<mml:mo stretchy="true">|</mml:mo>
</mml:math>
</inline-formula> and <inline-formula>
<mml:math id="M5">
<mml:msub>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>2</mml:mn>
</mml:msub>
<mml:mo>=</mml:mo>
<mml:msqrt>
<mml:mrow>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:munderover>
<mml:msubsup>
<mml:mi>&#x03B2;</mml:mi>
<mml:mi>j</mml:mi>
<mml:mn>2</mml:mn>
</mml:msubsup>
</mml:mrow>
</mml:msqrt>
</mml:math>
</inline-formula>. In our case, <inline-formula>
<mml:math id="M6">
<mml:msub>
<mml:mi>y</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
<mml:mo>=</mml:mo>
<mml:mo>log</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mfrac>
<mml:msub>
<mml:mi>&#x03D5;</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:msub>
<mml:mi>&#x03D5;</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfrac>
</mml:mfenced>
</mml:math>
</inline-formula>, the logarithm of the odds ratio of RTI implementation to no RTI implementation (or the logit).</p>
<p>Here, <inline-formula>
<mml:math id="M7">
<mml:msub>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:math>
</inline-formula> is the sum of absolute values of the coefficients for the predictors, which is also called <inline-formula>
<mml:math id="M8">
<mml:msub>
<mml:mi>l</mml:mi>
<mml:mn>1</mml:mn>
</mml:msub>
</mml:math>
</inline-formula>penalty, and <inline-formula>
<mml:math id="M9">
<mml:msubsup>
<mml:mrow>
<mml:mo stretchy="true">&#x2016;</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x2016;</mml:mo>
</mml:mrow>
<mml:mn>2</mml:mn>
<mml:mn>2</mml:mn>
</mml:msubsup>
</mml:math>
</inline-formula> is the sum of squared coefficients for the predictors, which is also called <inline-formula>
<mml:math id="M10">
<mml:msub>
<mml:mi>l</mml:mi>
<mml:mn>2</mml:mn>
</mml:msub>
</mml:math>
</inline-formula> penalty. In addition, &#x03BB; controls the overall strength of regularization, while &#x03B1; balances between LASSO and Ridge regression penalties. When &#x03B1;=1, elastic net regression transforms to LASSO regression, and when &#x03B1;=0, it becomes Ridge regression. The factor 1/2 before the <italic>l</italic><sub>2</sub> penalty is included for mathematical optimization convenience and does not alter the fundamental behavior of elastic net regression. Both LASSO and Ridge regressions have limitations with highly correlated predictors: Ridge regression tends to retain both variables but produces similar coefficient estimates for these predictors, whereas LASSO typically selects one predictor and discards the other. However, elastic net regression strikes a balance between LASSO and Ridge. This allows elastic net regression not only to retain both correlated predictors but also to generate stable coefficient estimates. Therefore, elastic net regression is chosen because it combines the advantages of both LASSO and Ridge regressions (<xref ref-type="bibr" rid="ref54">Zou and Hastie, 2005</xref>).</p>
<p>Elastic net regression selects variables by imposing two penalties on variable coefficients; therefore, choosing the appropriate values for <inline-formula>
<mml:math id="M15">
<mml:mi>&#x03BB;</mml:mi>
</mml:math>
</inline-formula> and <inline-formula>
<mml:math id="M16">
<mml:mi>&#x03B1;</mml:mi>
</mml:math>
</inline-formula>is crucial. In this study, these penalty parameters were chosen via nested CV because <xref ref-type="bibr" rid="ref50">Vabalas et al. (2019)</xref> have demonstrated that nested CV is an effective cross-validation method, especially when the sample size is small. The procedure for nested CV is presented in <xref ref-type="fig" rid="fig2">Figure 2</xref>. A 5-fold nested CV is used in this study as an example. The standard nested CV is conducted in six steps:</p>
<list list-type="order">
<list-item><p>The entire dataset is divided into several folds; for example, into 5-fold. These serve as the outer folds of the nested CV. Each outer fold is further split into an outer testing set and an outer training set.</p></list-item>
<list-item><p>Each outer training set is further divided into several inner folds. These inner folds consist of their own training and testing sets, referred to as the inner training fold and inner testing fold, respectively. These are used for feature selection, hyperparameter tuning, and model building. The best model for each inner fold is selected based on the smallest discrepancy in performance between the inner training and testing sets, indicating minimal overfitting.</p></list-item>
<list-item><p>The best model from an inner fold (for example, the model from the third inner fold, highlighted in yellow) is then tested using the corresponding outer testing set (in this case, outer test fold 1, also highlighted in yellow).</p></list-item>
<list-item><p>Steps 2 and 3 are repeated for each outer fold, with each outer fold producing a model.</p></list-item>
<list-item><p>The best features and tuning parameters are chosen from the model associated with the outer fold that shows the least overfitting. These are then used to train a model on the entire dataset to create the final model.</p></list-item>
<list-item><p>The final model is applied to an independent dataset to validate its generalizability.</p></list-item>
</list>
<fig position="float" id="fig2">
<label>Figure 2</label>
<caption>
<p>An illustration of nested CV.</p>
</caption>
<graphic xlink:href="fpsyg-15-1410396-g002.tif"/>
</fig>
<p>All predictors were standardized before conducting elastic net logistic regression. While elastic net logistic regression offers benefits, it can potentially yield 10 distinct models corresponding to the 10 imputation datasets. This diversity presents a challenge in synthesizing an overarching model from the 10 distinct versions. Consequently, upon obtaining these 10 models, we employed four distinct methods to select the final candidate models, resulting in four separate candidate models. Each method yields its own model for consideration in the final selection process. Four methods of selecting the candidate models were comprehensively described in the next section. To determine the final model, the candidate models were subsequently assessed in an independent dataset to evaluate their generalizability. All candidate models used predictors from the independent dataset to generate predictions for math RTI. Subsequently, the balanced accuracy between these predicted math RTI values and the observed math RTI from the independent dataset was calculated. The balanced accuracy is calculated as <inline-formula>
<mml:math id="M17">
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi mathvariant="italic">sensitivity</mml:mi>
<mml:mo>+</mml:mo>
<mml:mi mathvariant="italic">specificity</mml:mi>
</mml:mrow>
</mml:mfenced>
<mml:mo stretchy="true">/</mml:mo>
<mml:mn>2</mml:mn>
</mml:math>
</inline-formula>, where in our case sensitivity is the rate of correctly labeling schools as implementing RTI and specificity is the rate of correctly labeling schools as not implementing RTI (<xref ref-type="bibr" rid="ref5">Chicco et al., 2021</xref>; <xref ref-type="bibr" rid="ref15">Garc&#x00ED;a et al., 2009</xref>).</p>
<p>Because the predicted math RTI from the model is expressed as a percentage, we utilized the optimal threshold to categorize the predicted math RTI into two distinct categories. The optimal threshold applied in this study is widely recognized in research for maximizing both sensitivity and specificity (<xref ref-type="bibr" rid="ref9">Coffin and Sukhatme, 1997</xref>; <xref ref-type="bibr" rid="ref38">Perkins and Schisterman, 2006</xref>; <xref ref-type="bibr" rid="ref49">Unal, 2017</xref>). This approach is also called the &#x201C;closest-top-left-corner&#x201D; method. This method utilizes the receiver operating characteristic (ROC) plot, a graphical representation that maps the True Positive Rate (<italic>Sensitivity</italic>) against the False Positive Rate (1&#x2014;<italic>Specificity</italic>) for various threshold values. Typically, the <italic>x</italic>-axis denotes 1-<italic>Specificity</italic>, while the <italic>y</italic>-axis corresponds to <italic>Sensitivity</italic>. Ideally, the best classifier would achieve 100% Sensitivity and 0% False Positive Rate, which would be represented by the point (0, 1) on the ROC plot, situated at the top-left corner of the graph. However, attaining this perfect point is nearly impossible in practical scenarios. Therefore, the &#x201C;closest-top-left-corner&#x201D; method selects the point on the ROC curve that is nearest to (0, 1) as the optimal threshold, representing the most effective balance between <italic>Sensitivity</italic> and <italic>Specificity</italic> based on the available data. Beyond determining the optimal threshold, the ROC plot is also utilized to calculate the Area Under the Curve (AUC), which quantifies the area beneath the ROC curve. An AUC of 1 indicates that the elastic net logistic regression model is an ideal classifier, perfectly distinguishing schools by their math RTI status. Conversely, an AUC of 0.5 suggests that the model lacks discriminative ability (<xref ref-type="bibr" rid="ref13">Fan et al., 2006</xref>; <xref ref-type="bibr" rid="ref35">Obuchowski and Bullen, 2018</xref>). The final model which exhibited the best performance was chosen.<xref ref-type="fn" rid="fn0001"><sup>1</sup></xref></p>
<p>Considering the necessity for an independent dataset to assess generalizability, we divided the dataset into two parts: one for model building, which includes training and validating the model, and another to serve as an independent dataset for conducting the generalizability check of the model. In addition, to ensure the validity of the generalizability assessment, we employed complete cases from the original dataset, which were free from missing values, to serve as the independent dataset. This independent dataset consisted of 127 schools. The rest of the cases were used for training and validating the elastic net logistic regression model.</p>
<p>Given the prevalence of Likert scale items in our data, we faced the challenge of potentially expanding the number of predictors significantly if each were to be treated categorically. To address this and streamline the modeling process, we adopted the guidance provided by <xref ref-type="bibr" rid="ref19">Harpe (2015)</xref>, treating ordinal variables with five or more categories as continuous, while those with four or fewer categories were handled as categorical variables. This decision was made to balance the granularity of the Likert-scale responses with the practical considerations of model complexity and interpretability.</p>
<p>The second consideration is related to the sampling design. Again, we applied the same strategy to control for the sampling design as we did in the missing value imputation stage. We included &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU,&#x201D; representing strata and PSU, to account for the sampling design. Consequently, these two variables were included in the elastic net logistic regression analysis without undergoing variable selection. The variables &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; are inherently categorical and would typically necessitate the creation of dummy variables. Upon thorough examination of the cross-tabulations between &#x201C;W4C4P_4TSTR&#x201D; and the response variable &#x201C;math RTI,&#x201D; as well as &#x201C;W4C4P_4TPSU&#x201D; and &#x201C;math RTI,&#x201D; we identified many cells with zero observations. The details of these cross-tabulations are presented in <xref ref-type="table" rid="tab2">Tables 2</xref>, <xref ref-type="table" rid="tab3">3</xref>. Such a distribution poses a risk of the complete separation or quasi-complete separation problem in logistic regression (<xref ref-type="bibr" rid="ref10">Devika et al., 2016</xref>). To mitigate this issue, we strategically combined certain categories within &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU.&#x201D; This not only resolved the separation problem but also curbed the risk of overfitting in the final model.</p>
<table-wrap position="float" id="tab2">
<label>Table 2</label>
<caption>
<p>Cross-tabulation between &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;math RTI&#x201D; based on school level.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th/>
<th/>
<th align="left" valign="top" colspan="12">W4C4P_4TSTR</th>
<th/>
</tr>
</thead>
<tbody>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">5</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">9</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">11</td>
<td align="center" valign="bottom">12</td>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="middle">0</td>
<td align="center" valign="bottom">34</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
</tr>
<tr>
<td align="center" valign="middle">1</td>
<td align="center" valign="bottom">185</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">11</td>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">15</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">2</td>
</tr>
<tr>
<td/>
<td/>
<td align="left" valign="middle" colspan="12">W4C4P_4TSTR</td>
<td/>
</tr>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">15</td>
<td align="center" valign="bottom">16</td>
<td align="center" valign="bottom">19</td>
<td align="center" valign="bottom">20</td>
<td align="center" valign="bottom">21</td>
<td align="center" valign="bottom">22</td>
<td align="center" valign="bottom">23</td>
<td align="center" valign="bottom">24</td>
<td align="center" valign="bottom">26</td>
<td align="center" valign="bottom">28</td>
<td align="center" valign="bottom">29</td>
<td align="center" valign="bottom">33</td>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="middle">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
</tr>
<tr>
<td align="center" valign="middle">1</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
</tr>
<tr>
<td/>
<td/>
<td align="left" valign="bottom" colspan="12">W4C4P_4TSTR</td>
<td/>
</tr>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">35</td>
<td align="center" valign="bottom">36</td>
<td align="center" valign="bottom">37</td>
<td align="center" valign="bottom">38</td>
<td align="center" valign="bottom">39</td>
<td align="center" valign="bottom">40</td>
<td align="center" valign="bottom">41</td>
<td align="center" valign="bottom">42</td>
<td align="center" valign="bottom">43</td>
<td align="center" valign="bottom">44</td>
<td align="center" valign="bottom">45</td>
<td align="center" valign="bottom">46</td>
<td align="center" valign="bottom">47</td>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="middle">0</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">5</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">2</td>
</tr>
<tr>
<td align="center" valign="middle">1</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">15</td>
<td align="center" valign="bottom">9</td>
<td align="center" valign="bottom">15</td>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">16</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">16</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">16</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">13</td>
</tr>
<tr>
<td/>
<td/>
<td align="left" valign="bottom" colspan="12">W4C4P_4TSTR</td>
<td/>
</tr>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">48</td>
<td align="center" valign="bottom">49</td>
<td align="center" valign="bottom">50</td>
<td align="center" valign="bottom">51</td>
<td align="center" valign="bottom">52</td>
<td align="center" valign="bottom">53</td>
<td align="center" valign="bottom">54</td>
<td align="center" valign="bottom">55</td>
<td align="center" valign="bottom">56</td>
<td align="center" valign="bottom">57</td>
<td align="center" valign="bottom">58</td>
<td align="center" valign="bottom">59</td>
<td align="center" valign="bottom">60</td>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
</tr>
<tr>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">17</td>
<td align="center" valign="bottom">19</td>
<td align="center" valign="bottom">18</td>
<td align="center" valign="bottom">30</td>
<td align="center" valign="bottom">19</td>
<td align="center" valign="bottom">21</td>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">24</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">12</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">13</td>
<td align="center" valign="bottom">7</td>
</tr>
<tr>
<td/>
<td/>
<td align="left" valign="bottom" colspan="12">W4C4P_4TSTR</td>
<td/>
</tr>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">61</td>
<td align="center" valign="bottom">62</td>
<td align="center" valign="bottom">63</td>
<td align="center" valign="bottom">64</td>
<td align="center" valign="bottom">65</td>
<td align="center" valign="bottom">66</td>
<td align="center" valign="bottom">67</td>
<td align="center" valign="bottom">68</td>
<td align="center" valign="bottom">69</td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">20</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">9</td>
<td align="center" valign="bottom">8</td>
<td align="center" valign="bottom">16</td>
<td align="center" valign="bottom">14</td>
<td align="center" valign="bottom">21</td>
<td/>
<td/>
<td/>
<td/>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>W4C4P_4TSTR: sampling strata; Math RTI: whether math RTI is implemented.</p>
</table-wrap-foot>
</table-wrap>
<table-wrap position="float" id="tab3">
<label>Table 3</label>
<caption>
<p>Cross-tabulation between &#x201C;W4C4P_4TPSU&#x201D; and &#x201C;math RTI&#x201D; based on school level.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th/>
<th/>
<th align="left" valign="top" colspan="9">W4C4P_4TPSU</th>
</tr>
</thead>
<tbody>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">5</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">7</td>
<td align="center" valign="bottom">8</td>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">34</td>
<td align="center" valign="bottom">24</td>
<td align="center" valign="bottom">24</td>
<td align="center" valign="bottom">3</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">0</td>
</tr>
<tr>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">185</td>
<td align="center" valign="bottom">221</td>
<td align="center" valign="bottom">262</td>
<td align="center" valign="bottom">60</td>
<td align="center" valign="bottom">12</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">11</td>
<td align="center" valign="bottom">6</td>
<td align="center" valign="bottom">4</td>
</tr>
<tr>
<td/>
<td/>
<td align="left" valign="middle" colspan="9">W4C4P_4TPSU</td>
</tr>
<tr>
<td/>
<td/>
<td align="center" valign="bottom">9</td>
<td align="center" valign="bottom">10</td>
<td align="center" valign="bottom">15</td>
<td align="center" valign="bottom">18</td>
<td align="center" valign="bottom">19</td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="middle" rowspan="2">Math RTI</td>
<td align="center" valign="middle">0</td>
<td align="center" valign="bottom">2</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">0</td>
<td/>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="center" valign="middle">1</td>
<td align="center" valign="bottom">5</td>
<td align="center" valign="bottom">4</td>
<td align="center" valign="bottom">0</td>
<td align="center" valign="bottom">1</td>
<td align="center" valign="bottom">2</td>
<td/>
<td/>
<td/>
<td/>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>W4C4P_4TPSU: sampling PSU; Math RTI: whether math RTI is implemented.</p>
</table-wrap-foot>
</table-wrap>
<p>Based on these two cross-tabulations, the re-categorization of &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; is primarily determined by the category of the response variable where math RTI equals 0, given the majority of zero observations occur when math RTI is 0. Additionally, the recategorization was implemented to balance the sample sizes between the math RTI&#x2009;=&#x2009;0 group and the math RTI&#x2009;=&#x2009;1 group. This recategorization of &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; was conducted at the school level. For &#x201C;W4C4P_4TSTR,&#x201D; excluding those with missing values (NAs), schools were categorized based on the math RTI sample size. Schools with math RTI cell values of 14 or fewer were grouped together into the first category and assigned a code of 1. Schools with math RTI cell values in the range [15, 20) were grouped into the second category and coded as 2, and those with math RTI cell values are equal to or greater than 20 were grouped into the third category and coded as 3. In addition, those who have NAs were coded as 0. Regarding &#x201C;W4C4P_4TPSU,&#x201D; except for those who have NAs, the original categories 1 and 2 were retained due to the substantial number of values present in the math RTI cell, as they provided a sufficient sample size for both the math RTI&#x2009;=&#x2009;0 group and the math RTI&#x2009;=&#x2009;1 group. The rest of the categories were combined together and coded as 3. Also, those who have NAs were coded as 0.</p>
<p>It is important to note that this category consolidation was uniquely applied to &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU.&#x201D; The rationale behind this selective approach is twofold: First, other categorical variables were subjected to the selection process of the elastic net logistic regression, which inherently manages the separation problem (<xref ref-type="bibr" rid="ref14">Friedman et al., 2010</xref>; <xref ref-type="bibr" rid="ref33">M&#x00FC;nch et al., 2021</xref>). Second, &#x201C;W4C4P_4TSTR&#x201D; (49 categories) and &#x201C;W4C4P_4TPSU&#x201D; (12 categories) contain too many categories. After dummy coding these variables, the number of predictors increases dramatically, potentially leading to overfitting in the final model. This issue arises because &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; are integral to accounting for the sampling design and were retained without penalties. Moreover, these two variables were not the central focus of our study, further justifying their fixed inclusion in the model without the application of elastic net penalties.</p>
</sec>
<sec id="sec16">
<title>Four methods of aggregating results from multiple imputation</title>
<p>As previously discussed, the process could yield 10 distinct models, each corresponding to one of the 10 imputation datasets. Consequently, the crucial challenge lies in determining the candidate models for the final selection, especially considering that these models may contain feature-varying variables. Some studies have discussed the variable selection of regularized linear regression (<xref ref-type="bibr" rid="ref34">Musoro et al., 2014</xref>; <xref ref-type="bibr" rid="ref17">Gunn et al., 2023</xref>; <xref ref-type="bibr" rid="ref53">Zahid et al., 2020</xref>). Although the studies by <xref ref-type="bibr" rid="ref17">Gunn et al. (2023)</xref>, <xref ref-type="bibr" rid="ref53">Zahid et al. (2020)</xref>, and <xref ref-type="bibr" rid="ref34">Musoro et al. (2014)</xref> primarily utilized LASSO regression, the methodologies they explored can be adaptable to elastic net regression. Consequently, we extended their approaches to our elastic net framework, corresponding to Method<sub>50</sub>, Method<sub>coef</sub>, and Method<sub>adj</sub>, respectively. Furthermore, we proposed a new method to select the candidate model from 10 distinct models based on the generalizability (Method<sub>gen</sub>).</p>
<sec id="sec17">
<title>Method<sub>50</sub></title>
<p>Method<sub>50</sub> initially performs elastic net logistic regression separately on each imputed dataset, retaining variables selected in more than 50% of the cases. The variables &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; were not selected because they were consistently retained in the model. Average the coefficients of the selected variables across the 10 models derived from imputed data to construct the final candidate model. Then, the final candidate model was fitted to independent data to evaluate its generalizability. This method was selected based on empirical evidence from <xref ref-type="bibr" rid="ref17">Gunn et al. (2023)</xref>, which demonstrated its superior performance compared to two other variable selection methods. The first alternative method involves using stacked datasets, where training and testing datasets from multiple imputed datasets are combined into a single stacked training dataset and a corresponding stacked testing dataset. A regularized regression model is then developed using the stacked training dataset and validated using the stacked testing dataset. The second method employs a group penalty approach,<xref ref-type="fn" rid="fn0002"><sup>2</sup></xref> where the group penalty parameter is calculated jointly across the training datasets for each imputed dataset. This approach ensures that models derived from multiple imputed datasets incorporate consistent variables, with each model&#x2019;s performance evaluated against its respective testing dataset before aggregating the final performance metrics across all models.</p>
</sec>
<sec id="sec18">
<title>Method<sub>coef</sub></title>
<p>Method<sub>coef</sub>, proposed by <xref ref-type="bibr" rid="ref53">Zahid et al. (2020)</xref>, begins by independently conducting elastic net logistic regression on each imputed dataset. It retains variables that meet specific criteria of <inline-formula>
<mml:math id="M18">
<mml:mn>1</mml:mn>
<mml:mo stretchy="true">/</mml:mo>
<mml:mi>p</mml:mi>
</mml:math>
</inline-formula>, where <inline-formula>
<mml:math id="M19">
<mml:mi>p</mml:mi>
</mml:math>
</inline-formula> is the number of predictors. Notably, the variables &#x201C;W4C4P_4TSTR&#x201D; and &#x201C;W4C4P_4TPSU&#x201D; were excluded because they were consistently retained in the model. To be more specific, the criteria for choosing variables are:</p>
<list list-type="simple">
<list-item><p>1. For continuous variable <inline-formula>
<mml:math id="M20">
<mml:msub>
<mml:mi>X</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:math>
</inline-formula> associated with a parameter <inline-formula>
<mml:math id="M21">
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula>, indicating the <italic>j</italic>th (<inline-formula>
<mml:math id="M22">
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mi>p</mml:mi>
</mml:math>
</inline-formula>) predictor within the <italic>m</italic>th (<inline-formula>
<mml:math id="M23">
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mi>M</mml:mi>
</mml:math>
</inline-formula>) imputed dataset. The criterion for retaining the variable is as follows:</p></list-item>
</list>
<disp-formula id="E2">
<mml:math id="M24">
<mml:mfrac>
<mml:mrow>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mi>M</mml:mi>
</mml:mfrac>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:msubsup>
<mml:mo stretchy="true">|</mml:mo>
<mml:msub>
<mml:mover>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="true">|</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:msubsup>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mi>M</mml:mi>
</mml:mfrac>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:msubsup>
<mml:mo stretchy="true">|</mml:mo>
<mml:msub>
<mml:mover>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="true">|</mml:mo>
</mml:mrow>
</mml:mfrac>
<mml:mo>&#x2265;</mml:mo>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mi>p</mml:mi>
</mml:mfrac>
</mml:math>
</disp-formula>
<list list-type="simple">
<list-item><p>2. For a categorical variable <inline-formula>
<mml:math id="M25">
<mml:msub>
<mml:mi>X</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:math>
</inline-formula> with <inline-formula>
<mml:math id="M26">
<mml:mi>K</mml:mi>
<mml:mo>+</mml:mo>
<mml:mn>1</mml:mn>
</mml:math>
</inline-formula> categories, <inline-formula>
<mml:math id="M27">
<mml:mi>K</mml:mi>
</mml:math>
</inline-formula> dummy variables are required. Each dummy variable is associated with a parameter <inline-formula>
<mml:math id="M28">
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula>, which represents the <italic>k</italic>th (<inline-formula>
<mml:math id="M29">
<mml:mi>k</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:msub>
<mml:mi>K</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:math>
</inline-formula>) dummy variable for the <italic>j</italic>th (<inline-formula>
<mml:math id="M30">
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mi>p</mml:mi>
</mml:math>
</inline-formula>) predictor within the <italic>m</italic>th (<inline-formula>
<mml:math id="M31">
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mi>M</mml:mi>
</mml:math>
</inline-formula>) imputed dataset. The criterion for retaining the variable is as follows:</p></list-item>
</list>
<disp-formula id="E3">
<mml:math id="M32">
<mml:mfrac>
<mml:mrow>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mrow>
<mml:mi>M</mml:mi>
<mml:mo>&#x22C5;</mml:mo>
<mml:msub>
<mml:mi>K</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfrac>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:msub>
<mml:mi>K</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:msubsup>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:msubsup>
<mml:msub>
<mml:mover>
<mml:mrow>
<mml:mo stretchy="true">|</mml:mo>
<mml:mi>&#x03B2;</mml:mi>
</mml:mrow>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="true">|</mml:mo>
</mml:mrow>
<mml:mrow>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:msubsup>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mrow>
<mml:mi>M</mml:mi>
<mml:mo>&#x22C5;</mml:mo>
<mml:msub>
<mml:mi>K</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfrac>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>k</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:msub>
<mml:mi>K</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:msubsup>
<mml:msubsup>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:msubsup>
<mml:mo stretchy="true">|</mml:mo>
<mml:msub>
<mml:mover>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mi>k</mml:mi>
<mml:mo>,</mml:mo>
<mml:mi>m</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo stretchy="true">|</mml:mo>
</mml:mrow>
</mml:mfrac>
<mml:mo>&#x2265;</mml:mo>
<mml:mfrac>
<mml:mn>1</mml:mn>
<mml:mi>p</mml:mi>
</mml:mfrac>
</mml:math>
</disp-formula>
<p>Then, average the coefficients of the selected variables across the 10 models derived from imputed data, and fit the final candidate model to independent data to evaluate its generalizability.</p>
<p><xref ref-type="bibr" rid="ref53">Zahid et al. (2020)</xref> explored the influence of the number of predictors (<inline-formula>
<mml:math id="M33">
<mml:mn>40</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>80</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>200</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>500</mml:mn>
</mml:math>
</inline-formula>) and proportions of missingness (<inline-formula>
<mml:math id="M34">
<mml:mn>0.05</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>0.1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>0.2</mml:mn>
<mml:mo>,</mml:mo>
<mml:mn>0.3</mml:mn>
</mml:math>
</inline-formula>) on Method<sub>coef</sub> with a small sample size (<inline-formula>
<mml:math id="M35">
<mml:mn>100</mml:mn>
</mml:math>
</inline-formula>). The results revealed that Method<sub>coef</sub> can relatively balance the trade-off between selecting relevant and irrelevant variables. Typically, selecting more relevant variables tends to also increase the selection of irrelevant variables. In addition, Method<sub>coef</sub> can be used when the number of predictors exceeds the sample size, a scenario where the method using group penalty fails to select variables.</p>
</sec>
<sec id="sec19">
<title>Method<sub>adj</sub></title>
<p>Method<sub>adj</sub>, proposed by <xref ref-type="bibr" rid="ref34">Musoro et al. (2014)</xref>, differs from previous methods. While the earlier methods finalize the model through variable selection, this method does not engage in selecting variables. Instead, its aim is to adjust the parameter estimates of the final model. The procedure for Method<sub>adj</sub> is outlined as follows:</p>
<list list-type="simple">
<list-item><p>1. Run elastic net logistic regression independently for each imputed dataset.</p></list-item>
<list-item><p>2. Disregard the variable distinctions among each model and compute the average coefficients for all parameters across the 10 models, <inline-formula>
<mml:math id="M36">
<mml:msub>
<mml:mover>
<mml:mi>Y</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mi mathvariant="italic">fin</mml:mi>
</mml:msub>
<mml:mo>=</mml:mo>
<mml:mover>
<mml:mi>&#x03B1;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mo>+</mml:mo>
<mml:msub>
<mml:mover>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mi>j</mml:mi>
</mml:msub>
<mml:msub>
<mml:mi>X</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:math>
</inline-formula>, where <inline-formula>
<mml:math id="M37">
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:math>
</inline-formula> is the coefficient for the predictor <inline-formula>
<mml:math id="M38">
<mml:mi>j</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mi>p</mml:mi>
</mml:mrow>
</mml:mfenced>
</mml:math>
</inline-formula>.</p></list-item>
<list-item><p>3. Use bootstrapping to obtain the calibration parameters for the parameter adjustment of the final model. Here is the procedure of bootstrapping:</p></list-item>
</list>
<list list-type="simple">
<list-item><p>&#x2003; For a bootstrap run, the same observations were selected across 10 imputed datasets (<inline-formula>
<mml:math id="M39">
<mml:mi>i</mml:mi>
<mml:mi>m</mml:mi>
<mml:msub>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
</mml:msub>
</mml:math>
</inline-formula>, where <inline-formula>
<mml:math id="M40">
<mml:mi>i</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mn>10</mml:mn>
</mml:math>
</inline-formula>) to obtain corresponding bootstrapping datasets, <inline-formula>
<mml:math id="M41">
<mml:mi>i</mml:mi>
<mml:mi>m</mml:mi>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mo>&#x2217;</mml:mo>
</mml:msubsup>
</mml:math>
</inline-formula> (<inline-formula>
<mml:math id="M42">
<mml:mi>i</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
<mml:mo>,</mml:mo>
<mml:mo>&#x2026;</mml:mo>
<mml:mo>,</mml:mo>
<mml:mn>10</mml:mn>
</mml:math>
</inline-formula>).</p></list-item>
<list-item><p>&#x2003; Rebuild the elastic net logistic regression model based on the bootstrapping datasets, <inline-formula>
<mml:math id="M43">
<mml:mi>i</mml:mi>
<mml:mi>m</mml:mi>
<mml:msubsup>
<mml:mi>p</mml:mi>
<mml:mi>i</mml:mi>
<mml:mo>&#x2217;</mml:mo>
</mml:msubsup>
</mml:math>
</inline-formula>.</p></list-item>
<list-item><p>&#x2003; Repeat a and b 100 times.</p></list-item>
</list>
<list list-type="simple">
<list-item><p>4. Aggregate each variable in the imputed datasets into a long-stacked dataset, so that the size of the stacked dataset is 10 times that of the original dataset. For each bootstrap run, the predicted response variable is calculated using the coefficients derived from the elastic net regression model built on the bootstrap sample. Specifically, the predicted response variable is computed by multiplying the elastic net regression coefficients with the predictors in the stacked dataset. For predictors not selected by the elastic net regression, their coefficients are set to zero. Subsequently, calibration parameters are computed by regressing the response variable in the stacked dataset against the predicted response variable. The predicted response variable from bootstrapping is calculated using the final model from step 3 and predictors from the stacked dataset. This relationship is symbolized as <inline-formula>
<mml:math id="M44">
<mml:msub>
<mml:mi>Y</mml:mi>
<mml:mi mathvariant="italic">imp</mml:mi>
</mml:msub>
<mml:mo>=</mml:mo>
<mml:msub>
<mml:mi>&#x03B1;</mml:mi>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>+</mml:mo>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mover>
<mml:mi>Y</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mi>m</mml:mi>
<mml:msup>
<mml:mi>p</mml:mi>
<mml:mo>&#x2217;</mml:mo>
</mml:msup>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula>.</p></list-item>
<list-item><p>5. Then, compute the average of <inline-formula>
<mml:math id="M45">
<mml:msub>
<mml:mi>&#x03B1;</mml:mi>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula> and <inline-formula>
<mml:math id="M46">
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula> over these 100 bootstrap iterations to obtain <inline-formula>
<mml:math id="M47">
<mml:msub>
<mml:mover accent="true">
<mml:mi>&#x03B1;</mml:mi>
<mml:mo stretchy="true">&#x00AF;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula> and <inline-formula>
<mml:math id="M48">
<mml:msub>
<mml:mover accent="true">
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x00AF;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:math>
</inline-formula>. Finally, adjust the final model for intercepts and all coefficients from the imputed datasets by using the equation <inline-formula>
<mml:math id="M49">
<mml:msub>
<mml:mover accent="true">
<mml:mi>&#x03B1;</mml:mi>
<mml:mo stretchy="true">&#x00AF;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>+</mml:mo>
<mml:msub>
<mml:mover accent="true">
<mml:mi>&#x03B2;</mml:mi>
<mml:mo stretchy="true">&#x00AF;</mml:mo>
</mml:mover>
<mml:mrow>
<mml:mi>a</mml:mi>
<mml:mi>d</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mover>
<mml:mi>&#x03B1;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mo>+</mml:mo>
<mml:msub>
<mml:mover>
<mml:mi>&#x03B2;</mml:mi>
<mml:mo>&#x0302;</mml:mo>
</mml:mover>
<mml:mi>j</mml:mi>
</mml:msub>
<mml:msub>
<mml:mi>X</mml:mi>
<mml:mi>j</mml:mi>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:math>
</inline-formula>.</p></list-item>
<list-item><p>6. Fit the final model to the independent data to assess the generalizability.</p></list-item>
</list>
</sec>
<sec id="sec20">
<title>Method<sub>gen</sub></title>
<p>The final method, Method<sub>gen</sub>, is a novel approach introduced by this study. Method<sub>gen</sub> initially conducts elastic net logistic regression independently on each imputed dataset. Each model is then fitted to independent data, with the final candidate model being selected based on achieving the highest balanced accuracy.</p>
</sec>
</sec>
<sec id="sec21">
<title>Evaluation criteria for the model performance</title>
<p>Given that the response variable math RTI is binary, accuracy serves as a suitable metric to assess the model performance. However, the response variable, math RTI, in this study is imbalanced. <xref ref-type="bibr" rid="ref46">Sun et al. (2009)</xref> claimed that standard classifiers, such as the logistic regression and decision tree, tended to ignore the rare cases, potentially compromising the accuracy of the model&#x2019;s predictions. Given the presence of an imbalanced response variable in the dataset, weights were computed for each observation. These weights were then applied to the regularized logistic regression to ensure a balanced 50:50 weight ratio across the two categories. Also, as we focus on the prediction accuracy for both implementing and not implementing RTI, balanced accuracy is a more precise metric as compared to regular accuracy. Furthermore, we also used the receiver operating characteristic (ROC) plot and the corresponding area under the curve (AUC) to delineate the performance of the elastic net logistic regression.</p>
</sec>
</sec>
</sec>
<sec sec-type="results" id="sec22">
<title>Results</title>
<sec id="sec23">
<title>Descriptive statistics</title>
<p>The total sample size at the individual level was 6,647. After aggregating variables to the school level, the sample size was reduced to 1,130. Since math RTI was investigated at the school level, the final sample size should be 1,130. Within the selected data, there were 37 continuous predictors and 54 categorical ones. Upon creating dummy variables for categorical variables, the analysis included a total of 160 predictors. Across the 10 imputed datasets, the count of cases where math RTI was not implemented ranged from 94 to 97. Conversely, for cases where math RTI was implemented, the count lied between 1,033 and 1,036. This data suggest that the majority of schools have adopted math RTI. However, due to the large number of predictors, descriptive statistics of predictors were omitted.</p>
</sec>
<sec id="sec24">
<title>Four methods of variable selection and the final model</title>
<p><xref ref-type="table" rid="tab4">Table 4</xref> displays the candidate models derived from four methods. Except for variables of the sampling design, W4C4P_4TSTR and W4C4P_4TPSU, we found Method<sub>gen</sub>, Method<sub>50</sub>, and Method<sub>coef</sub> selected the same variables. Method<sub>adj</sub> does not select variables from the 10 models derived from imputed data. Instead, it retains all variables from these 10 models and adjusts their coefficients. As a result, Method<sub>adj</sub> includes all variables produced by the imputed data, resulting in a slightly larger variable count.</p>
<table-wrap position="float" id="tab4">
<label>Table 4</label>
<caption>
<p>Candidate models from four methods.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th align="left" valign="top">Methods</th>
<th align="left" valign="top">Variables</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="middle">Method<sub>gen</sub></td>
<td align="left" valign="middle">W4C4P_4TSTR.1, W4C4P_4TSTR.2, W4C4P_4TSTR.3, W4C4P_4TPSU.1, W4C4P_4TPSU.2, W4C4P_4TPSU.3, S4RTLMTH.2</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>50</sub></td>
<td align="left" valign="middle">W4C4P_4TSTR.1, W4C4P_4TSTR.2, W4C4P_4TSTR.3, W4C4P_4TPSU.1, W4C4P_4TPSU.2, W4C4P_4TPSU.3, S4RTLMTH.2</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>coef</sub></td>
<td align="left" valign="middle">W4C4P_4TSTR.1, W4C4P_4TSTR.2, W4C4P_4TSTR.3, W4C4P_4TPSU.1, W4C4P_4TPSU.2, W4C4P_4TPSU.3, S4RTLMTH.2</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>adj</sub></td>
<td align="left" valign="middle">W4C4P_4TSTR.1, W4C4P_4TSTR.2, W4C4P_4TSTR.3, W4C4P_4TPSU.1, W4C4P_4TPSU.2, W4C4P_4TPSU.3, S4RTLMTH.2, S4RTLSOC.2</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>Variables beginning with S4 are administrator-reported; variables beginning with A4 are teacher-reported; variables beginning with X4 are data collected by/reported to the ECLS-K. W4C4P_4TSTR: sampling strata, W4C4P_4TPSU: sampling PSU, S4RTLMTH: whether math RTI is implemented in Grade 1, S4RTLSOC: whether behavior/social RTI is implemented in Grade 1. Variables with numbers indicate those are dummy variables for corresponding categorical variables. For dummy variables with &#x201C;RTL&#x201D; (which indicates the variable measuring administrator-reported RTI implementation), the baseline represents full implementation. A value of &#x201C;2&#x201D; denotes no implementation, while &#x201C;1&#x201D; is partial implementation.</p>
</table-wrap-foot>
</table-wrap>
<p><xref ref-type="table" rid="tab5">Table 5</xref> presents the balanced accuracy and AUC values associated with the four methods. Method<sub>50</sub> and Method<sub>coef</sub> achieved the highest balanced accuracy, both equal to 0.852, and also demonstrated the highest AUC values. In contrast, Method<sub>gen</sub> also showed commendable balanced accuracy at 0.829, while Method<sub>adj</sub> recorded the lowest balanced accuracy of 0.491 among the four methods. Given the superior performance of Method<sub>50</sub> and Method<sub>coef</sub>, the final model is based on these methods, as they produced identical results. Therefore, the final model is defined by the following logistic regression model: <inline-formula>
<mml:math id="M50">
<mml:mtable columnalign="left">
<mml:mtr>
<mml:mtd>
<mml:mo>log</mml:mo>
<mml:mfenced open="(" close=")">
<mml:mfrac>
<mml:mrow>
<mml:mi>P</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>Y</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>P</mml:mi>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:mi>Y</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfrac>
</mml:mfenced>
<mml:mo>=</mml:mo>
<mml:mn>.186</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>.038</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TPSU</mml:mi>
<mml:mn>.1</mml:mn>
<mml:mo>+</mml:mo>
<mml:mn>.166</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TPSU</mml:mi>
<mml:mn>.2</mml:mn>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>.310</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TPSU</mml:mi>
<mml:mn>.3</mml:mn>
<mml:mo>+</mml:mo>
<mml:mn>.799</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TSTR</mml:mi>
<mml:mn>.1</mml:mn>
<mml:mo>+</mml:mo>
<mml:mn>.112</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TSTR</mml:mi>
<mml:mn>.2</mml:mn>
</mml:mtd>
</mml:mtr>
<mml:mtr>
<mml:mtd>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>.102</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>W</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>C</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi>P</mml:mi>
<mml:mo>_</mml:mo>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">TSTR</mml:mi>
<mml:mn>.3</mml:mn>
<mml:mo>&#x2212;</mml:mo>
<mml:mn>1.072</mml:mn>
<mml:mo>&#x00D7;</mml:mo>
<mml:mi>S</mml:mi>
<mml:mn>4</mml:mn>
<mml:mi mathvariant="italic">RTLMTH</mml:mi>
<mml:mn>.2</mml:mn>
<mml:mtext>.</mml:mtext>
</mml:mtd>
</mml:mtr>
</mml:mtable>
</mml:math>
</inline-formula><xref ref-type="fig" rid="fig3">Figure 3</xref> is the ROC plot for the final model derived from Method<sub>50</sub> and Method<sub>coef</sub>. Based on the balanced accuracy and the ROC plot, it is concluded that the final model effectively classifies math RTI. <xref ref-type="table" rid="tab6">Table 6</xref> presents the tuning parameters for the 10 imputed datasets.</p>
<table-wrap position="float" id="tab5">
<label>Table 5</label>
<caption>
<p>Generalizability of candidate models from four methods using the independent dataset.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th/>
<th align="center" valign="top">Balanced accuracy</th>
<th align="center" valign="top">AUC</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="middle">Method<sub>gen</sub></td>
<td align="center" valign="middle">0.829</td>
<td align="center" valign="middle">0.855</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>50</sub></td>
<td align="center" valign="middle">0.852</td>
<td align="center" valign="middle">0.883</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>coef</sub></td>
<td align="center" valign="middle">0.852</td>
<td align="center" valign="middle">0.883</td>
</tr>
<tr>
<td align="left" valign="middle">Method<sub>adj</sub></td>
<td align="center" valign="middle">0.491</td>
<td align="center" valign="middle">0.329</td>
</tr>
</tbody>
</table>
</table-wrap>
<fig position="float" id="fig3">
<label>Figure 3</label>
<caption>
<p>The ROC plot for the final model.</p>
</caption>
<graphic xlink:href="fpsyg-15-1410396-g003.tif"/>
</fig>
<table-wrap position="float" id="tab6">
<label>Table 6</label>
<caption>
<p>Tuning parameters of 10 elastic net logistic regression models.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th/>
<th align="center" valign="top">
<inline-formula>
<mml:math id="M51">
<mml:mi>&#x03BB;</mml:mi>
</mml:math>
</inline-formula>
</th>
<th align="center" valign="top">
<inline-formula>
<mml:math id="M52">
<mml:mi>&#x03B1;</mml:mi>
</mml:math>
</inline-formula>
</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="middle">Imputed dataset 1</td>
<td align="center" valign="middle">0.235</td>
<td align="center" valign="middle">0.500</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 2</td>
<td align="center" valign="middle">0.102</td>
<td align="center" valign="middle">0.800</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 3</td>
<td align="center" valign="middle">0.136</td>
<td align="center" valign="middle">1.00</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 4</td>
<td align="center" valign="middle">0.160</td>
<td align="center" valign="middle">1.00</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 5</td>
<td align="center" valign="middle">0.109</td>
<td align="center" valign="middle">0.600</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 6</td>
<td align="center" valign="middle">0.133</td>
<td align="center" valign="middle">1.00</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 7</td>
<td align="center" valign="middle">0.131</td>
<td align="center" valign="middle">1.00</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 8</td>
<td align="center" valign="middle">0.133</td>
<td align="center" valign="middle">1.00</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 9</td>
<td align="center" valign="middle">0.137</td>
<td align="center" valign="middle">0.600</td>
</tr>
<tr>
<td align="left" valign="middle">Imputed dataset 10</td>
<td align="center" valign="middle">0.135</td>
<td align="center" valign="middle">0.800</td>
</tr>
</tbody>
</table>
</table-wrap>
</sec>
</sec>
<sec sec-type="discussion" id="sec25">
<title>Discussion</title>
<p>Response to intervention has been prominent in research for several decades, and since the early 2000s, it has become much more prominent in practice both through individual districts&#x2019; uptake of the practice as well as through recommendations in federal and state policies (<xref ref-type="bibr" rid="ref23">Jimerson et al., 2016</xref>). The overall process, typically involving a three-tiered system to classify the intensity of students&#x2019; academic needs, involves many moving components, including implementation of assessment practices like screening and progress monitoring, managing data-based decisions based on screening and monitoring data, and selecting appropriate curricula and interventions to implement at each tier. Each component of RTI requires substantial human, material, and financial resources, and there is substantial between-school variability in the capacity to allocate resources and sustain implementation. However, little is known about the factors that relate to schools&#x2019; decisions to use and sustain RTI, though it is reasonable to assume economic and human resources would be a primary driver.</p>
<p>In this study, we investigated the predictors of schools&#x2019; implementation of math RTI in Grade 2 using predictors from the prior year. There is a dearth of research examining the factors that relate to schools&#x2019; decision to use RTI in math. Historically, large-scale studies of reading RTI and its implementation have predominated the literature (e.g., <xref ref-type="bibr" rid="ref1">Balu et al., 2015</xref>), and the decisions to implement math RTI in addition to other initiatives in place (<xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>) may be unique relative to decisions guiding implementation of other prevention and intervention systems. Thus, it is important to establish an empirical basis for the factors that relate to schools&#x2019; uptake of an initiative like math RTI, which is both complex in terms of the school system dynamics needed to sustain its implementation and resource-demanding (<xref ref-type="bibr" rid="ref6">Choi et al., 2022</xref>; <xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>).</p>
<p>Results of our analyses indicate that three of the four methods evaluated for selecting predictors were equivalent in model balanced accuracy. Of these three methods, Method<sub>50</sub> and Method<sub>coef</sub> equally demonstrated the strongest performance based on the AUC and balanced accuracy. All models demonstrated that previous math RTI implementation was a predictor of future math RTI implementation. Additionally, RTI implementation for social/behavioral skills in Grade 1 emerged as a potential predictor for math RTI implementation, as it was included in several elastic net regression models. This finding is further supported by the correlation analysis, which showed that RTI implementation for math and social/behavioral skills in Grade 1 were the top two variables most strongly correlated with math RTI implementation in Grade 2. Specifically, the correlation for math RTI implementation in Grade 1 was 0.602, while the correlation for RTI implementation for social/behavioral skills in Grade 1 was 0.224. Notably, RTI implementation for math in Grade 1 exhibited a significantly higher correlation with math RTI implementation in Grade 2 compared to other variables. This could suggest that concurrent RTI infrastructure may be a determinant of math RTI implementation, though this may not be as robust of a predictor given the inconsistency of its selection into the models. Given that this is the first study to employ these methods for predicting RTI implementation, it is essential to conduct additional research examining the school-level and contextual factors relating to schools&#x2019; math RTI implementation decisions. Qualitative reports of math RTI implementation indicate student economic conditions, teacher professional development, and other implementation priorities are relevant factors in the math RTI implementation process (<xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>).</p>
<sec id="sec26">
<title>Different methods of variable selection</title>
<p>In this paper, we used four methods, Method<sub>gen</sub>, Method<sub>50</sub>, Method<sub>coef</sub>, and Method<sub>adj</sub>, to select candidate models from 10 distinct models derived from 10 imputed datasets. Then, four candidate models were evaluated in the independent dataset and the final model was determined based on the best performance. From <xref ref-type="table" rid="tab4">Tables 4</xref>, <xref ref-type="table" rid="tab5">5</xref>, we could observe that Method<sub>50</sub>, and Method<sub>coef</sub> selected the same variables and exhibited identical balanced accuracy and AUC in the independent dataset. This consistency is due to the robustness of the RF imputation method used in our study against multiple imputations. To further confirm this robustness, we generated an additional 30 imputed datasets, bringing the total to 40. Among these, only two yielded elastic net regressions with different variables, while the remainder produced identical elastic net regressions, albeit with varying parameter estimates. This consistency ensures that both Method<sub>50</sub> and Method<sub>coef</sub> select the same variables across multiple imputations. Moreover, the variable S4RTLMTH.2, which is included in the final model, consistently appears in all elastic net models. Consequently, the average coefficient for S4RTLMTH.2 was calculated across the multiple imputed datasets, resulting in the same final model being produced by both Method<sub>50</sub> and Method<sub>coef</sub>.</p>
<p>In contrast, although Method<sub>gen</sub> selected the same variables as Method<sub>50</sub> and Method<sub>coef</sub>, it achieved different balanced accuracy and AUC on independent data. This variation arises because Method<sub>gen</sub> selects only one model from the candidate models based solely on AUC, without averaging the coefficients of the selected variables across these models. Consequently, the parameter estimates in the final model of Method<sub>gen</sub> differ from those in Method<sub>50</sub> and Method<sub>coef</sub>. In scenarios where the outcome is imbalanced, the AUC can provide an overly optimistic view of a model&#x2019;s performance, particularly favoring the majority class. Therefore, when the outcome is imbalanced, balanced accuracy is a more appropriate index than AUC, because balanced accuracy takes this imbalance into consideration. Given the similar performance observed with Method<sub>gen</sub>, Method<sub>50</sub>, and Method<sub>coef</sub>, researchers are encouraged to employ all these approaches in future studies. Doing so can allow these methods to complement one another, providing a more comprehensive understanding of the study. However, Method<sub>adj</sub> is not recommended as it underperformed compared to the other three methods and potentially includes too many variables, which may lead to overfitting.</p>
<p>Method<sub>adj</sub> is different from the other three methods because it retains all variables from 10 distinct models derived from 10 imputed datasets and adjusted model coefficients rather than selecting variables. This retention of all variables can result in an overly complex final model, prone to overfitting. The relatively poor performance of Method<sub>adj</sub> may be attributed to its approach of calculating the calibration parameters based on the stacked dataset of these 10 imputed datasets. The 10 imputed datasets were used both for constructing the elastic net logistic regression model and for computing the calibration parameters. This repeated use of the same datasets may cause model overfitting, which could degrade performance and compromise generalizability. Another potential reason for Method<sub>adj</sub>&#x2019;s poor performance might be that the bootstrapping datasets do not accurately represent the original dataset, due to the nature of bootstrapping involving repeated sampling from the original dataset with replacement. Given the significant imbalance in the outcome, the class distributions in the bootstrapping data could differ markedly. Moreover, since the RF imputation produced relatively stable imputed datasets, and Method<sub>adj</sub> consistently selected the same observations across these imputed datasets for calibration in each run, this approach could cause all bootstrapping datasets in each run to differ significantly in outcome categories from the original dataset, potentially biasing the calibration parameters.</p>
<p>When selecting variables for regularized models with multiple imputations, there are generally two approaches to variable selection. The first involves fitting regularized models separately for each imputed dataset, which may result in distinct models, and then applying thresholds to select variables. The second approach aims to create a unified set of variables across all regularized models. Method<sub>50</sub> and Method<sub>coef</sub> adhere to the first approach. Initially, each method fits a regularized model separately for each imputed dataset. Variables are then selected based on specific thresholds: for Method<sub>50</sub>, variables with non-zero coefficients must appear in more than 50% of the cases, whereas for Method<sub>coef</sub>, the magnitude of the coefficients must be equal to or greater than <inline-formula>
<mml:math id="M53">
<mml:mn>1</mml:mn>
<mml:mo stretchy="true">/</mml:mo>
<mml:mi>p</mml:mi>
</mml:math>
</inline-formula>. The second approach can be achieved either by using a stacked dataset or by applying a group penalty, as described by <xref ref-type="bibr" rid="ref17">Gunn et al. (2023)</xref> in their second and third methods. As previously mentioned, the stacked method combines multiple imputed datasets into a single stacked dataset, then applies regularized regression to this unified dataset. This approach can select unified variables because it ultimately chooses variables from one dataset. On the other hand, the method using a group penalty applies the group penalty across all imputed datasets, assuming that if a variable is important, it should be selected in all imputed datasets. This method produces unified variables by jointly fitting the group penalty to all imputed datasets. By adopting these two methods, researchers can bypass the need to select a threshold when formulating the final model.</p>
<p>No single variable selection method consistently outperforms others. Previous studies, such as <xref ref-type="bibr" rid="ref52">Wood et al. (2008)</xref>, have shown that the performance of methods using thresholds and stacked datasets is comparable. <xref ref-type="bibr" rid="ref12">Du et al. (2022)</xref> favored the method using a stacked dataset over the group penalty method for achieving better coefficient estimates and reduced computation time. Conversely, <xref ref-type="bibr" rid="ref17">Gunn et al. (2023)</xref> observed that the stacked dataset method underperformed compared to methods using thresholds and the group penalty in an empirical dataset. <xref ref-type="bibr" rid="ref53">Zahid et al. (2020)</xref> noted that while the method using the group penalty can correctly identify relevant variables, it also tends to select more non-informative variables. Moreover, this method fails to select variables when the number of predictors exceeds the sample size. Additionally, the efficacy of the method using the group penalty is highly dependent on the number of imputations. <xref ref-type="bibr" rid="ref12">Du et al. (2022)</xref> made similar observations regarding the dependency on the number of imputations for the group penalty method, noting a more significant improvement with this approach compared to the stacked dataset method as the number of imputations increased. Previous studies have not reached a consensus on which method is definitively superior. Therefore, researchers are encouraged to employ multiple variable selection methods to assess the robustness of the selected variables.</p>
</sec>
<sec id="sec27">
<title>Practical implications</title>
<p>The current results confirm practical assumptions that existing initiatives (previous social/behavioral RTI implementation) relate to RTI implementation. Although the first study to empirically demonstrate these relations, these findings are likely unsurprising to applied researchers and school personnel. The capacity and motives to initiate math RTI implementation will be constrained by other existing priorities and competing resources for an added initiative (<xref ref-type="bibr" rid="ref29">Mason et al., 2019</xref>). However, it is important to note that our results suggest that previous math RTI implementation is consistently selected as a predictor, suggesting strong stability in schools&#x2019; decisions to use math RTI over these two school years. In the absence of contextual and human resources predictors, it may be that initial uptake of RTI is a more idiosyncratic process rather than systematically attributable to specific, quantifiable factors. As a result, research and practice would benefit from additional mixed methods work to understand the experiential processes involved in RTI implementation and whether this relates to other systematic factors at the school level.</p>
</sec>
<sec id="sec28">
<title>Limitations and future directions</title>
<p>One limitation of this study is the small size of the independent dataset. The limited sample size means that some categories of categorical variables cannot be validated in the independent dataset, as it lacks representation of those categories. For example, there were three categories of W4C4P_4TPSU when constructing the elastic net logistic regression model. However, in the independent dataset, only one category of W4C4P_4TPSU was present. Therefore, limited sample size of the independent dataset compromises the generalizability of the final model to some degree. The second limitation of this study lies in our comparison of four methods using empirical data. While the empirical results offer valuable insights, a thorough simulation study is necessary to comprehensively evaluate the four variable selection methods. Moreover, the finding of robustness of the RF imputation is also based on empirical data. A simulation study is needed to fully investigate the relationship between the RF imputation method and the number of imputations. Last, the reports of math RTI implementation in Grade 2, which we further collapsed into 0&#x2009;=&#x2009;no implementation or 1&#x2009;=&#x2009;partial/full implementation are highly limited and may not accurately represent the presence of core components of math RTI implementation (<xref ref-type="bibr" rid="ref26">Lembke et al., 2012</xref>). As a result, the nature of these schools implementing math RTI is unclear. More accurate criteria for differentiating math RTI implementation is essential in future studies to accurately capture the factors that go into schools&#x2019; uptake and implementation sustainment. Moreover, the use of school-level data in this case may not accurately represent actual school-level factors: teachers reports are not representative of all teachers within each school, nor are aggregated student-level data representative of all students in that grade and school. Finally, the current study cannot differentiate how &#x201C;partial&#x201D; and &#x201C;full&#x201D; RTI would have been interpreted. Future research should examine the predictors of different degrees of implementation in addition to the specific processes that are implemented.</p>
</sec>
</sec>
<sec sec-type="conclusions" id="sec29">
<title>Conclusion</title>
<p>Given the increasing uptake of tiered intervention systems in schools (<xref ref-type="bibr" rid="ref6">Choi et al., 2022</xref>), such as RTI, there is a pressing need to identify the factors relating to schools&#x2019; implementation decisions. Our current study revealed that existing RTI systems were primary predictors of schools&#x2019; implementation. This is a first step in developing an empirical basis for predictors of school-wide math RTI implementation.</p>
</sec>
</body>
<back>
<sec sec-type="data-availability" id="sec30">
<title>Data availability statement</title>
<p>The datasets and R code are available in online repositories. The names of the repository/repositories and accession number(s) can be found at: <ext-link xlink:href="https://www.openicpsr.org/openicpsr/project/209149/version/V2/view" ext-link-type="uri">https://www.openicpsr.org/openicpsr/project/209149/version/V2/view</ext-link>.</p>
</sec>
<sec sec-type="ethics-statement" id="sec31">
<title>Ethics statement</title>
<p>Ethical approval was not required for the study involving humans in accordance with the local legislation and institutional requirements. Written informed consent to participate in this study was not required from the participants or the participants&#x2019; legal guardians/next of kin in accordance with the national legislation and the institutional requirements.</p>
</sec>
<sec sec-type="author-contributions" id="sec32">
<title>Author contributions</title>
<p>QW: Formal analysis, Methodology, Software, Validation, Visualization, Writing &#x2013; original draft, Writing &#x2013; review &#x0026; editing. GH: Conceptualization, Funding acquisition, Supervision, Writing &#x2013; original draft, Writing &#x2013; review &#x0026; editing. QZ: Conceptualization, Funding acquisition, Supervision, Writing &#x2013; review &#x0026; editing. SC: Writing &#x2013; review &#x0026; editing.</p>
</sec>
<sec sec-type="funding-information" id="sec33">
<title>Funding</title>
<p>The author(s) declare that financial support was received for the research, authorship, and/or publication of this article. This research was supported by a grant from the American Educational Research Association which receives funds for its &#x201C;AERA Grants Program&#x201D; from the National Science Foundation under NSF award NSF-DRL #1749275. Opinions reflect those of the author and do not necessarily reflect those AERA or NSF. Garret Hall is PI and Qian Zhang is CO-PI.</p>
</sec>
<ack>
<p>ChatGPT 4 was utilized to check for grammatical errors. OpenAI. (2023). ChatGPT4 [Large language model]. <ext-link xlink:href="https://chat.openai.com/chat" ext-link-type="uri">https://chat.openai.com/chat</ext-link>.</p>
</ack>
<sec sec-type="COI-statement" id="sec34">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec sec-type="disclaimer" id="sec35">
<title>Publisher&#x2019;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<fn-group>
<fn id="fn0001">
<p><sup>1</sup>The final model was selected based on having the highest balanced accuracy. In cases where two candidate models exhibit identical balanced accuracy, the model with the larger Area Under the Curve (AUC) was chosen.</p>
</fn>
<fn id="fn0002">
<p><sup>2</sup>The calculation of group penalty parameter, <inline-formula>
<mml:math id="M54">
<mml:mi>&#x03BB;</mml:mi>
</mml:math>
</inline-formula>, as jointly determined using all training datasets (<xref ref-type="bibr" rid="ref4">Chen and Wang, 2013</xref>) is <inline-formula>
<mml:math id="M55">
<mml:mo>min</mml:mo>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:munderover>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>i</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>N</mml:mi>
</mml:munderover>
<mml:msup>
<mml:mfenced open="(" close=")">
<mml:mrow>
<mml:msub>
<mml:mi>y</mml:mi>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mi>i</mml:mi>
</mml:mrow>
</mml:msub>
<mml:mo>&#x2212;</mml:mo>
<mml:mfenced open="[" close="]">
<mml:mrow>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mn>0</mml:mn>
</mml:mrow>
</mml:msub>
<mml:mo>+</mml:mo>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:munderover>
<mml:msub>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
<mml:msub>
<mml:mi>x</mml:mi>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
</mml:msub>
</mml:mrow>
</mml:mfenced>
</mml:mrow>
</mml:mfenced>
<mml:mn>2</mml:mn>
</mml:msup>
<mml:mo>+</mml:mo>
<mml:mi>&#x03BB;</mml:mi>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>j</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>p</mml:mi>
</mml:munderover>
<mml:msqrt>
<mml:mrow>
<mml:munderover>
<mml:mstyle displaystyle="true">
<mml:mo stretchy="true">&#x2211;</mml:mo>
</mml:mstyle>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>1</mml:mn>
</mml:mrow>
<mml:mi>M</mml:mi>
</mml:munderover>
<mml:msubsup>
<mml:mi>&#x03B2;</mml:mi>
<mml:mrow>
<mml:mi>m</mml:mi>
<mml:mi>j</mml:mi>
</mml:mrow>
<mml:mn>2</mml:mn>
</mml:msubsup>
</mml:mrow>
</mml:msqrt>
</mml:math>
</inline-formula> where <inline-formula><mml:math id="M56">
<mml:mi>M</mml:mi>
</mml:math></inline-formula> is the number of imputed datasets, <inline-formula><mml:math id="M57">
<mml:mi>N</mml:mi>
</mml:math></inline-formula> is the sample size, <inline-formula><mml:math id="M58">
<mml:mi>p</mml:mi>
</mml:math></inline-formula> is the number of predictors.</p>
</fn>
</fn-group>
<ref-list>
<title>References</title>
<ref id="ref1">
<citation citation-type="other"><person-group person-group-type="author"><name><surname>Balu</surname> <given-names>R.</given-names></name> <name><surname>Zhu</surname> <given-names>P.</given-names></name> <name><surname>Doolittle</surname> <given-names>F.</given-names></name> <name><surname>Schiller</surname> <given-names>E.</given-names></name> <name><surname>Jenkins</surname> <given-names>J.</given-names></name> <name><surname>Gersten</surname> <given-names>R</given-names></name></person-group>. (<year>2015</year>). Evaluation of response to intervention practices for elementary school reading (NCEE: 2016-4000). National Center for educational evaluation and regional assistance, Institute of Education Sciences, U.S. Department of Education.</citation>
</ref>
<ref id="ref2">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bouck</surname> <given-names>E. C.</given-names></name> <name><surname>Cosby</surname> <given-names>M. D.</given-names></name></person-group> (<year>2019</year>). <article-title>Response to intervention in high school mathematics: one school&#x2019;s implementation</article-title>. <source>Prev. Sch. Fail. Altern. Educ. Child. Youth</source> <volume>63</volume>, <fpage>32</fpage>&#x2013;<lpage>42</lpage>. doi: <pub-id pub-id-type="doi">10.1080/1045988X.2018.1469463</pub-id></citation>
</ref>
<ref id="ref3">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Burns</surname> <given-names>M.</given-names></name> <name><surname>Jimerson</surname> <given-names>S.</given-names></name> <name><surname>VanDerHeyden</surname> <given-names>A.</given-names></name> <name><surname>Deno</surname> <given-names>S.</given-names></name></person-group> (<year>2016</year>). <article-title>Toward a Unified Response-to-Intervention Model: Multi-Tiered Systems of Support</article-title>. In: <source>Handbook of Response to Intervention</source>. eds. <person-group person-group-type="editor"><name><surname>Jimerson</surname> <given-names>S.</given-names></name> <name><surname>Burns</surname> <given-names>M.</given-names></name> <name><surname>VanDerHeyden</surname> <given-names>A.</given-names></name></person-group> (<publisher-loc>Boston, MA</publisher-loc>: <publisher-name>Springer</publisher-name>).</citation>
</ref>
<ref id="ref4">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chen</surname> <given-names>Q.</given-names></name> <name><surname>Wang</surname> <given-names>S.</given-names></name></person-group> (<year>2013</year>). <article-title>Variable selection for multiply-imputed data with application to dioxin exposure study</article-title>. <source>Stat. Med.</source> <volume>32</volume>, <fpage>3646</fpage>&#x2013;<lpage>3659</lpage>. doi: <pub-id pub-id-type="doi">10.1002/sim.5783</pub-id>, PMID: <pub-id pub-id-type="pmid">23526243</pub-id></citation>
</ref>
<ref id="ref5">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chicco</surname> <given-names>D.</given-names></name> <name><surname>T&#x00F6;tsch</surname> <given-names>N.</given-names></name> <name><surname>Jurman</surname> <given-names>G.</given-names></name></person-group> (<year>2021</year>). <article-title>The Matthews correlation coefficient (MCC) is more reliable than balanced accuracy, bookmaker informedness, and markedness in two-class confusion matrix evaluation</article-title>. <source>Bio Data Min.</source> <volume>14</volume>, <fpage>13</fpage>&#x2013;<lpage>22</lpage>. doi: <pub-id pub-id-type="doi">10.1186/s13040-021-00244-z</pub-id>, PMID: <pub-id pub-id-type="pmid">33541410</pub-id></citation>
</ref>
<ref id="ref6">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Choi</surname> <given-names>J. H.</given-names></name> <name><surname>McCart</surname> <given-names>A. B.</given-names></name> <name><surname>Miller</surname> <given-names>D. H.</given-names></name> <name><surname>Sailor</surname> <given-names>W.</given-names></name></person-group> (<year>2022</year>). <article-title>Issues in statewide scale up of a multi-tiered system of support</article-title>. <source>J. Sch. Leadersh.</source> <volume>32</volume>, <fpage>514</fpage>&#x2013;<lpage>536</lpage>. doi: <pub-id pub-id-type="doi">10.1177/10526846211067650</pub-id></citation>
</ref>
<ref id="ref8">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Clowes</surname> <given-names>S. K.</given-names></name> <name><surname>Duke</surname> <given-names>D.</given-names></name></person-group> (<year>2022</year>). &#x201C;<article-title>Research skills: collecting, analyzing, and interpreting quantitative data</article-title>&#x201D; in <source>SAGE Skills: Student Success</source>. (<publisher-loc>Los Angeles, CA</publisher-loc>: <publisher-name>SAGE Publications, Inc.</publisher-name>).</citation>
</ref>
<ref id="ref9">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Coffin</surname> <given-names>M.</given-names></name> <name><surname>Sukhatme</surname> <given-names>S.</given-names></name></person-group> (<year>1997</year>). <article-title>Receiver operating characteristic studies and measurement errors</article-title>. <source>Biometrics</source> <volume>53</volume>:<fpage>823</fpage>. doi: <pub-id pub-id-type="doi">10.2307/2533545</pub-id></citation>
</ref>
<ref id="ref10">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Devika</surname> <given-names>S.</given-names></name> <name><surname>Jeyaseelan</surname> <given-names>L.</given-names></name> <name><surname>Sebastian</surname> <given-names>G.</given-names></name></person-group> (<year>2016</year>). <article-title>Analysis of sparse data in logistic regression in medical research: a newer approach</article-title>. <source>J. Postgrad. Med.</source> <volume>62</volume>, <fpage>26</fpage>&#x2013;<lpage>31</lpage>. doi: <pub-id pub-id-type="doi">10.4103/0022-3859.173193</pub-id>, PMID: <pub-id pub-id-type="pmid">26732193</pub-id></citation>
</ref>
<ref id="ref11">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Donovan</surname> <given-names>E.</given-names></name> <name><surname>Shepherd</surname> <given-names>K.</given-names></name></person-group> (<year>2013</year>). <article-title>Implementing multi-tiered systems of support in mathematics: findings from two schools</article-title>. <source>J. Spec. Educ. Apprent.</source> <volume>2</volume>, <fpage>1</fpage>&#x2013;<lpage>15</lpage>. doi: <pub-id pub-id-type="doi">10.58729/2167-3454.1017</pub-id></citation>
</ref>
<ref id="ref12">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Du</surname> <given-names>J.</given-names></name> <name><surname>Boss</surname> <given-names>J.</given-names></name> <name><surname>Han</surname> <given-names>P.</given-names></name> <name><surname>Beesley</surname> <given-names>L. J.</given-names></name> <name><surname>Kleinsasser</surname> <given-names>M.</given-names></name> <name><surname>Goutman</surname> <given-names>S. A.</given-names></name> <etal/></person-group>. (<year>2022</year>). <article-title>Variable selection with multiply-imputed datasets: choosing between stacked and grouped methods</article-title>. <source>J. Comput. Graph. Stat.</source> <volume>31</volume>, <fpage>1063</fpage>&#x2013;<lpage>1075</lpage>. doi: <pub-id pub-id-type="doi">10.1080/10618600.2022.2035739</pub-id>, PMID: <pub-id pub-id-type="pmid">36644406</pub-id></citation>
</ref>
<ref id="ref13">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fan</surname> <given-names>J.</given-names></name> <name><surname>Upadhye</surname> <given-names>S.</given-names></name> <name><surname>Worster</surname> <given-names>A.</given-names></name></person-group> (<year>2006</year>). <article-title>Understanding receiver operating characteristic (ROC) curves</article-title>. <source>Can. J. Emerg. Med.</source> <volume>8</volume>, <fpage>19</fpage>&#x2013;<lpage>20</lpage>. doi: <pub-id pub-id-type="doi">10.1017/S1481803500013336</pub-id></citation>
</ref>
<ref id="ref14">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Friedman</surname> <given-names>J.</given-names></name> <name><surname>Hastie</surname> <given-names>T.</given-names></name> <name><surname>Tibshirani</surname> <given-names>R.</given-names></name></person-group> (<year>2010</year>). <article-title>Regularization paths for generalized linear models via coordinate descent</article-title>. <source>J. Stat. Softw.</source> <volume>33</volume>, <fpage>1</fpage>&#x2013;<lpage>22</lpage>. doi: <pub-id pub-id-type="doi">10.18637/jss.v033.i01</pub-id>, PMID: <pub-id pub-id-type="pmid">20808728</pub-id></citation>
</ref>
<ref id="ref15">
<citation citation-type="other"><person-group person-group-type="author"><name><surname>Garc&#x00ED;a</surname> <given-names>V.</given-names></name> <name><surname>Mollineda</surname> <given-names>R. A.</given-names></name> <name><surname>S&#x00E1;nchez</surname> <given-names>J. S.</given-names></name></person-group> (<year>2009</year>). &#x201C;Index of balanced accuracy: a performance measure for skewed class distributions&#x201D; in <italic>Iberian Conference on Pattern Recognition and Image Analysis</italic>. Berlin, Heidelberg: Springer Berlin Heidelberg, 441&#x2013;448.</citation>
</ref>
<ref id="ref16">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Geronimus</surname> <given-names>A. T.</given-names></name> <name><surname>Bound</surname> <given-names>J.</given-names></name></person-group> (<year>1998</year>). <article-title>Use of census-based aggregate variables to proxy for socioeconomic group: evidence from national samples</article-title>. <source>Am. J. Epidemiol.</source> <volume>148</volume>, <fpage>475</fpage>&#x2013;<lpage>486</lpage>. doi: <pub-id pub-id-type="doi">10.1093/oxfordjournals.aje.a009673</pub-id>, PMID: <pub-id pub-id-type="pmid">9737560</pub-id></citation>
</ref>
<ref id="ref17">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Gunn</surname> <given-names>H. J.</given-names></name> <name><surname>Hayati Rezvan</surname> <given-names>P.</given-names></name> <name><surname>Fern&#x00E1;ndez</surname> <given-names>M. I.</given-names></name> <name><surname>Comulada</surname> <given-names>W. S.</given-names></name></person-group> (<year>2023</year>). <article-title>How to apply variable selection machine learning algorithms with multiply imputed data: a missing discussion</article-title>. <source>Psychol. Methods</source> <volume>28</volume>, <fpage>452</fpage>&#x2013;<lpage>471</lpage>. doi: <pub-id pub-id-type="doi">10.1037/met0000478</pub-id>, PMID: <pub-id pub-id-type="pmid">35113633</pub-id></citation>
</ref>
<ref id="ref18">
<citation citation-type="journal"><person-group person-group-type="author">
<name><surname>Hans</surname> <given-names>C.</given-names></name>
</person-group> (<year>2011</year>). <article-title>Elastic net logistic regression modeling with the orthant normal prior</article-title>. <source>J. Am. Stat. Assoc.</source> <volume>106</volume>, <fpage>1383</fpage>&#x2013;<lpage>1393</lpage>. doi: <pub-id pub-id-type="doi">10.1198/jasa.2011.tm09241</pub-id></citation>
</ref>
<ref id="ref19">
<citation citation-type="journal"><person-group person-group-type="author">
<name><surname>Harpe</surname> <given-names>S. E.</given-names></name>
</person-group> (<year>2015</year>). <article-title>How to analyze Likert and other rating scale data</article-title>. <source>Curr. Pharm. Teach. Learn.</source> <volume>7</volume>, <fpage>836</fpage>&#x2013;<lpage>850</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.cptl.2015.08.001</pub-id></citation>
</ref>
<ref id="ref20">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Hastie</surname> <given-names>T.</given-names></name> <name><surname>Tibshirani</surname> <given-names>R.</given-names></name> <name><surname>Friedman</surname> <given-names>J.</given-names></name></person-group> (<year>2006</year>). <source>The Elements of Statistical Learning: Data Mining, Inference, and Prediction</source>. <edition>2nd</edition> Edn. <publisher-loc>New York, NY</publisher-loc>: <publisher-name>Springer</publisher-name>.</citation>
</ref>
<ref id="ref21">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Jacob</surname> <given-names>R. T.</given-names></name> <name><surname>Goddard</surname> <given-names>R. D.</given-names></name> <name><surname>Kim</surname> <given-names>E. S.</given-names></name></person-group> (<year>2014</year>). <article-title>Assessing the use of aggregate data in the evaluation of school-based interventions: implications for evaluation research and state policy regarding public-use data</article-title>. <source>Educ. Eval. Policy Anal.</source> <volume>36</volume>, <fpage>44</fpage>&#x2013;<lpage>66</lpage>. doi: <pub-id pub-id-type="doi">10.3102/0162373713485814</pub-id></citation>
</ref>
<ref id="ref23">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Jimerson</surname> <given-names>S. R.</given-names></name> <name><surname>Burns</surname> <given-names>M. K.</given-names></name> <name><surname>Van DerHeyden</surname> <given-names>A. M.</given-names></name></person-group> (eds). (<year>2016</year>). <article-title>From response to intervention to multi-tiered systems of support: advances in the science and practice of assessment and intervention</article-title> in <source>Handbook of Response to Intervention</source>. (<publisher-loc>Boston, MA</publisher-loc>: <publisher-name>Springer</publisher-name>).</citation>
</ref>
<ref id="ref25">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Lane</surname> <given-names>K. L.</given-names></name> <name><surname>Manzies</surname> <given-names>H. M.</given-names></name> <name><surname>Oakes</surname> <given-names>W. P.</given-names></name> <name><surname>Kalberg</surname> <given-names>J. R.</given-names></name></person-group> (<year>2019</year>). <source>Developing a Schoolwide Framework to Prevent and Manage Learning and Behavior Problems</source>. <edition>2nd</edition> Edn. <publisher-loc>New York, NY</publisher-loc>: <publisher-name>The Guilford Press</publisher-name>.</citation>
</ref>
<ref id="ref26">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lembke</surname> <given-names>E.</given-names></name> <name><surname>Hampton</surname> <given-names>D.</given-names></name> <name><surname>Beyers</surname> <given-names>S.</given-names></name></person-group> (<year>2012</year>). <article-title>Response to intervention in mathematics: critical elements: addressing response to intervention implementation: questions from the field</article-title>. <source>Psychol. Sch.</source> <volume>49</volume>, <fpage>257</fpage>&#x2013;<lpage>272</lpage>. doi: <pub-id pub-id-type="doi">10.1002/pits.21596</pub-id></citation>
</ref>
<ref id="ref27">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lewis</surname> <given-names>M. J.</given-names></name> <name><surname>Spiliopoulou</surname> <given-names>A.</given-names></name> <name><surname>Goldmann</surname> <given-names>K.</given-names></name> <name><surname>Pitzalis</surname> <given-names>C.</given-names></name> <name><surname>McKeigue</surname> <given-names>P.</given-names></name> <name><surname>Barnes</surname> <given-names>M. R.</given-names></name></person-group> (<year>2023</year>). <article-title>Nestedcv: an R package for fast implementation of nested cross-validation with embedded feature selection designed for transcriptomics and high-dimensional data</article-title>. <source>Bioinform. Adv.</source> <volume>3</volume>, <fpage>1</fpage>&#x2013;<lpage>5</lpage>. doi: <pub-id pub-id-type="doi">10.1093/bioadv/vbad048</pub-id>, PMID: <pub-id pub-id-type="pmid">37113250</pub-id></citation>
</ref>
<ref id="ref28">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Little</surname> <given-names>R. J.</given-names></name> <name><surname>Rubin</surname> <given-names>D. B.</given-names></name></person-group> (<year>2019</year>). <source>Statistical Analysis With Missing Data</source>, vol. <volume>793</volume>. <publisher-loc>Hoboken, NJ, USA</publisher-loc>: <publisher-name>John Wiley &#x0026; Sons</publisher-name>.</citation>
</ref>
<ref id="ref29">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mason</surname> <given-names>E. N.</given-names></name> <name><surname>Benz</surname> <given-names>S. A.</given-names></name> <name><surname>Lembke</surname> <given-names>E. S.</given-names></name> <name><surname>Burns</surname> <given-names>M. K.</given-names></name> <name><surname>Powell</surname> <given-names>S. R.</given-names></name></person-group> (<year>2019</year>). <article-title>From professional development to implementation: a district&#x2019;s experience implementing mathematics tiered systems of support</article-title>. <source>Learn. Disabil. Res. Pract.</source> <volume>34</volume>, <fpage>207</fpage>&#x2013;<lpage>214</lpage>. doi: <pub-id pub-id-type="doi">10.1111/ldrp.12206</pub-id></citation>
</ref>
<ref id="ref30">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>McIntosh</surname> <given-names>K.</given-names></name> <name><surname>Goodman</surname> <given-names>S.</given-names></name></person-group> (<year>2016</year>). <source>Integrated Multi-Tiered Systems of Support: Blending RTI and PBIS</source>. <publisher-loc>New York, NY</publisher-loc>: <publisher-name>The Guilford Press</publisher-name>.</citation>
</ref>
<ref id="ref31">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mellard</surname> <given-names>D.</given-names></name> <name><surname>McKnight</surname> <given-names>M.</given-names></name> <name><surname>Jordan</surname> <given-names>J.</given-names></name></person-group> (<year>2010</year>). <article-title>RTI tier structures and instructional intensity</article-title>. <source>Learn. Disabil. Res. Pract.</source> <volume>25</volume>, <fpage>217</fpage>&#x2013;<lpage>225</lpage>. doi: <pub-id pub-id-type="doi">10.1111/j.1540-5826.2010.00319.x</pub-id></citation>
</ref>
<ref id="ref32">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Moineddin</surname> <given-names>R.</given-names></name> <name><surname>Urquia</surname> <given-names>M. L.</given-names></name></person-group> (<year>2014</year>). <article-title>Regression analysis of aggregate continuous data</article-title>. <source>Epidemiology</source> <volume>25</volume>, <fpage>929</fpage>&#x2013;<lpage>930</lpage>. doi: <pub-id pub-id-type="doi">10.1097/EDE.0000000000000172</pub-id></citation>
</ref>
<ref id="ref33">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>M&#x00FC;nch</surname> <given-names>M. M.</given-names></name> <name><surname>Peeters</surname> <given-names>C. F.</given-names></name> <name><surname>Van Der Vaart</surname> <given-names>A. W.</given-names></name> <name><surname>Van De Wiel</surname> <given-names>M. A.</given-names></name></person-group> (<year>2021</year>). <article-title>Adaptive group-regularized logistic elastic net regression</article-title>. <source>Biostatistics</source> <volume>22</volume>, <fpage>723</fpage>&#x2013;<lpage>737</lpage>. doi: <pub-id pub-id-type="doi">10.1093/biostatistics/kxz062</pub-id>, PMID: <pub-id pub-id-type="pmid">31886488</pub-id></citation>
</ref>
<ref id="ref34">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Musoro</surname> <given-names>J. Z.</given-names></name> <name><surname>Zwinderman</surname> <given-names>A. H.</given-names></name> <name><surname>Puhan</surname> <given-names>M. A.</given-names></name> <name><surname>ter Riet</surname> <given-names>G.</given-names></name> <name><surname>Geskus</surname> <given-names>R. B.</given-names></name></person-group> (<year>2014</year>). <article-title>Validation of prediction models based on lasso regression with multiply imputed data</article-title>. <source>BMC Med. Res. Methodol.</source> <volume>14</volume>, <fpage>1</fpage>&#x2013;<lpage>13</lpage>. doi: <pub-id pub-id-type="doi">10.1186/1471-2288-14-116</pub-id>, PMID: <pub-id pub-id-type="pmid">25323009</pub-id></citation>
</ref>
<ref id="ref35">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Obuchowski</surname> <given-names>N. A.</given-names></name> <name><surname>Bullen</surname> <given-names>J. A.</given-names></name></person-group> (<year>2018</year>). <article-title>Receiver operating characteristic (ROC) curves: review of methods with applications in diagnostic medicine</article-title>. <source>Phys. Med. Biol.</source> <volume>63</volume>:<fpage>07TR01</fpage>. doi: <pub-id pub-id-type="doi">10.1088/1361-6560/aab4b1</pub-id></citation>
</ref>
<ref id="ref36">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Pantanowitz</surname> <given-names>A.</given-names></name> <name><surname>Marwala</surname> <given-names>T.</given-names></name></person-group> (<year>2009</year>). &#x201C;<article-title>Missing data imputation through the use of the random forest algorithm</article-title>&#x201D; in <source>Advances in Computational Intelligence</source>. (eds.) <person-group person-group-type="editor"><name><surname>Yu</surname> <given-names>W.</given-names></name> <name><surname>Sanchez</surname> <given-names>E. N.</given-names></name></person-group> (<publisher-loc>Berlin Heidelberg</publisher-loc>: <publisher-name>Springer</publisher-name>), <fpage>53</fpage>&#x2013;<lpage>62</lpage>.</citation>
</ref>
<ref id="ref38">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Perkins</surname> <given-names>N. J.</given-names></name> <name><surname>Schisterman</surname> <given-names>E. F.</given-names></name></person-group> (<year>2006</year>). <article-title>The inconsistency of &#x201C;optimal&#x201D; cutpoints obtained using two criteria based on the receiver operating characteristic curve</article-title>. <source>Am. J. Epidemiol.</source> <volume>163</volume>, <fpage>670</fpage>&#x2013;<lpage>675</lpage>. doi: <pub-id pub-id-type="doi">10.1093/aje/kwj063</pub-id>, PMID: <pub-id pub-id-type="pmid">16410346</pub-id></citation>
</ref>
<ref id="ref9003">
<citation citation-type="book"><person-group person-group-type="author">
<collab>R Core Team</collab>
</person-group>. (<year>2021</year>). <source>_R: A Language and Environment for Statistical Computing_</source>. <publisher-loc>Vienna, Austria</publisher-loc>: <publisher-name>R Foundation for Statistical Computing</publisher-name>. Available at: <ext-link xlink:href="https://www.R-project.org/" ext-link-type="uri">https://www.R-project.org/</ext-link></citation>
</ref>
<ref id="ref39">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Reynolds</surname> <given-names>C. R.</given-names></name> <name><surname>Shaywitz</surname> <given-names>S. E.</given-names></name></person-group> (<year>2009</year>). <article-title>Response to intervention: ready or not? Or, from wait-to-fail to watch-them-fail</article-title>. <source>Sch. Psychol. Q.</source> <volume>24</volume>, <fpage>130</fpage>&#x2013;<lpage>145</lpage>. doi: <pub-id pub-id-type="doi">10.1037/a0016158</pub-id>, PMID: <pub-id pub-id-type="pmid">20169006</pub-id></citation>
</ref>
<ref id="ref40">
<citation citation-type="book"><person-group person-group-type="author">
<name><surname>Rubin</surname> <given-names>D. B.</given-names></name>
</person-group> (<year>1987</year>). <source>Multiple Imputation for Nonresponse in Surveys</source>. <publisher-loc>New York</publisher-loc>: <publisher-name>Wiley</publisher-name>.</citation>
</ref>
<ref id="ref41">
<citation citation-type="book"><person-group person-group-type="author">
<name><surname>Schulte</surname> <given-names>A. C.</given-names></name>
</person-group> (<year>2016</year>). <article-title>Prevention and response to intervention: past, present, and future</article-title> in <source>Handbook of Response to Intervention: The Science and Practice of Multi-Tiered Systems of Support</source> (<edition>2nd Edn.</edition> (eds.) <person-group person-group-type="editor"><name><surname>Jimerson</surname> <given-names>S. R.</given-names></name> <name><surname>Burns</surname> <given-names>M. K.</given-names></name> <name><surname>DerHeyden</surname> <given-names>A. M.</given-names><prefix>Van</prefix></name></person-group> (<publisher-loc>Boston, MA</publisher-loc>: <publisher-name>Springer</publisher-name>), <fpage>59</fpage>&#x2013;<lpage>72</lpage></citation>
</ref>
<ref id="ref42">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Schumacher</surname> <given-names>R. G.</given-names></name> <name><surname>Edmonds</surname> <given-names>R. Z.</given-names></name> <name><surname>Arden</surname> <given-names>S. V.</given-names></name></person-group> (<year>2017</year>). <article-title>Examining implementation of intensive intervention in mathematics</article-title>. <source>Learn. Disabil. Res. Pract.</source> <volume>32</volume>, <fpage>189</fpage>&#x2013;<lpage>199</lpage>. doi: <pub-id pub-id-type="doi">10.1111/ldrp.12141</pub-id></citation>
</ref>
<ref id="ref43">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Stapleton</surname> <given-names>L. M.</given-names></name> <name><surname>Kang</surname> <given-names>Y.</given-names></name></person-group> (<year>2018</year>). <article-title>Design effects of multilevel estimates from national probability samples</article-title>. <source>Sociol. Methods Res.</source> <volume>47</volume>, <fpage>430</fpage>&#x2013;<lpage>457</lpage>. doi: <pub-id pub-id-type="doi">10.1177/0049124116630563</pub-id></citation>
</ref>
<ref id="ref9004">
<citation citation-type="book"><person-group person-group-type="author">
<collab>StataCorp</collab>
</person-group>. (<year>2021</year>). <article-title>Stata Statistical Software: Release 17</article-title>. <publisher-loc>College Station, TX</publisher-loc>: <publisher-name>StataCorp LLC</publisher-name>.</citation>
</ref>
<ref id="ref44">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Stekhoven</surname> <given-names>D. J.</given-names></name> <name><surname>B&#x00FC;hlmann</surname> <given-names>P.</given-names></name></person-group> (<year>2012</year>). <article-title>MissForest&#x2014;non-parametric missing value imputation for mixed-type data</article-title>. <source>Bioinformatics</source> <volume>28</volume>, <fpage>112</fpage>&#x2013;<lpage>118</lpage>. doi: <pub-id pub-id-type="doi">10.1093/bioinformatics/btr597</pub-id> (Accessed August 20, 2024)., PMID: <pub-id pub-id-type="pmid">22039212</pub-id></citation>
</ref>
<ref id="ref9001">
<citation citation-type="other"><person-group person-group-type="author">
<name><surname>Stekhoven</surname> <given-names>D. J.</given-names></name>
</person-group> (<year>2013</year>). Package &#x2018;missForest&#x2019;. R package version, 1, 21. Available online at: <ext-link xlink:href="https://cran.opencpu.org/web/packages/missForest/missForest.pdf" ext-link-type="uri">https://cran.opencpu.org/web/packages/missForest/missForest.pdf</ext-link> (accessed August 20, 2024).</citation>
</ref>
<ref id="ref46">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sun</surname> <given-names>Y.</given-names></name> <name><surname>Wong</surname> <given-names>A. K.</given-names></name> <name><surname>Kamel</surname> <given-names>M. S.</given-names></name></person-group> (<year>2009</year>). <article-title>Classification of imbalanced data: a review</article-title>. <source>Int. J. Pattern Recognit. Artif. Intell.</source> <volume>23</volume>, <fpage>687</fpage>&#x2013;<lpage>719</lpage>. doi: <pub-id pub-id-type="doi">10.1142/S0218001409007326</pub-id></citation>
</ref>
<ref id="ref47">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Tang</surname> <given-names>F.</given-names></name> <name><surname>Ishwaran</surname> <given-names>H.</given-names></name></person-group> (<year>2017</year>). <article-title>Random forest missing data algorithms</article-title>. <source>Stat. Analy. Data Min. ASA Data Sci. J.</source> <volume>10</volume>, <fpage>363</fpage>&#x2013;<lpage>377</lpage>. doi: <pub-id pub-id-type="doi">10.1002/sam.11348</pub-id>, PMID: <pub-id pub-id-type="pmid">29403567</pub-id></citation>
</ref>
<ref id="ref49">
<citation citation-type="journal"><person-group person-group-type="author">
<name><surname>Unal</surname> <given-names>I.</given-names></name>
</person-group> (<year>2017</year>). <article-title>Defining an optimal cut-point value in ROC analysis: an alternative approach</article-title>. <source>Comput. Math. Methods Med.</source> <volume>2017</volume>, <fpage>1</fpage>&#x2013;<lpage>14</lpage>. doi: <pub-id pub-id-type="doi">10.1155/2017/3762651</pub-id>, PMID: <pub-id pub-id-type="pmid">28642804</pub-id></citation>
</ref>
<ref id="ref50">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Vabalas</surname> <given-names>A.</given-names></name> <name><surname>Gowen</surname> <given-names>E.</given-names></name> <name><surname>Poliakoff</surname> <given-names>E.</given-names></name> <name><surname>Casson</surname> <given-names>A. J.</given-names></name></person-group> (<year>2019</year>). <article-title>Machine learning algorithm validation with a limited sample size</article-title>. <source>PLoS One</source> <volume>14</volume>:<fpage>e0224365</fpage>. doi: <pub-id pub-id-type="doi">10.1371/journal.pone.0224365</pub-id>, PMID: <pub-id pub-id-type="pmid">31697686</pub-id></citation>
</ref>
<ref id="ref52">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Wood</surname> <given-names>A. M.</given-names></name> <name><surname>White</surname> <given-names>I. R.</given-names></name> <name><surname>Royston</surname> <given-names>P.</given-names></name></person-group> (<year>2008</year>). <article-title>How should variable selection be performed with multiply imputed data?</article-title> <source>Stat. Med.</source> <volume>27</volume>, <fpage>3227</fpage>&#x2013;<lpage>3246</lpage>. doi: <pub-id pub-id-type="doi">10.1002/sim.3177</pub-id>, PMID: <pub-id pub-id-type="pmid">18203127</pub-id></citation>
</ref>
<ref id="ref53">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Zahid</surname> <given-names>F. M.</given-names></name> <name><surname>Faisal</surname> <given-names>S.</given-names></name> <name><surname>Heumann</surname> <given-names>C.</given-names></name></person-group> (<year>2020</year>). <article-title>Variable selection techniques after multiple imputation in high-dimensional data</article-title>. <source>Stat. Methods Appl.</source> <volume>29</volume>, <fpage>553</fpage>&#x2013;<lpage>580</lpage>. doi: <pub-id pub-id-type="doi">10.1007/s10260-019-00493-7</pub-id></citation>
</ref>
<ref id="ref54">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Zou</surname> <given-names>H.</given-names></name> <name><surname>Hastie</surname> <given-names>T.</given-names></name></person-group> (<year>2005</year>). <article-title>Regularization and variable selection via the elastic net</article-title>. <source>J. Roy. Stat. Soc. Ser. B. Stat. Methodol.</source> <volume>67</volume>, <fpage>301</fpage>&#x2013;<lpage>320</lpage>. doi: <pub-id pub-id-type="doi">10.1111/j.1467-9868.2005.00503.x</pub-id></citation>
</ref>
</ref-list>
</back>
</article>