<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Archiving and Interchange DTD v2.3 20070202//EN" "archivearticle.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" article-type="methods-article" dtd-version="2.3" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Hum. Neurosci.</journal-id>
<journal-title>Frontiers in Human Neuroscience</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Hum. Neurosci.</abbrev-journal-title>
<issn pub-type="epub">1662-5161</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fnhum.2024.1346050</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Human Neuroscience</subject>
<subj-group>
<subject>Methods</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Development and evaluation of a BCI-neurofeedback system with real-time EEG detection and electrical stimulation assistance during motor attempt for neurorehabilitation of children with cerebral palsy</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name><surname>Behboodi</surname> <given-names>Ahad</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/1648482/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/formal-analysis/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Kline</surname> <given-names>Julia</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Gravunder</surname> <given-names>Andrew</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/2608191/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Phillips</surname> <given-names>Connor</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Parker</surname> <given-names>Sheridan M.</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/2608001/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name><surname>Damiano</surname> <given-names>Diane L.</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="corresp" rid="c001"><sup>&#x002A;</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/233757/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/formal-analysis/"/>
<role content-type="https://credit.niso.org/contributor-roles/funding-acquisition/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/resources/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>Department of Biomechanics, University of Nebraska Omaha</institution>, <addr-line>Omaha, NE</addr-line>, <country>United States</country></aff>
<aff id="aff2"><sup>2</sup><institution>Neurorehabilitation and Biomechanics Research Section, Rehabilitation Medicine Department, Clinical Center, National Institutes of Health</institution>, <addr-line>Bethesda, MD</addr-line>, <country>United States</country></aff>
<author-notes>
<fn fn-type="edited-by" id="fn0001">
<p>Edited by: Jessica Rose, Stanford University, United States</p>
</fn>
<fn fn-type="edited-by" id="fn0002">
<p>Reviewed by: Vacius Jusas, Kaunas University of Technology, Lithuania</p>
<p>Markey Cierra Olson, Barrow Neurological Institute (BNI), United States</p>
</fn>
<corresp id="c001">&#x002A;Correspondence: Diane L. Damiano, <email>damianod@cc.nih.gov</email></corresp>
</author-notes>
<pub-date pub-type="epub">
<day>03</day>
<month>04</month>
<year>2024</year>
</pub-date>
<pub-date pub-type="collection">
<year>2024</year>
</pub-date>
<volume>18</volume>
<elocation-id>1346050</elocation-id>
<history>
<date date-type="received">
<day>28</day>
<month>11</month>
<year>2023</year>
</date>
<date date-type="accepted">
<day>22</day>
<month>03</month>
<year>2024</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x00A9; 2024 Behboodi, Kline, Gravunder, Phillips, Parker and Damiano.</copyright-statement>
<copyright-year>2024</copyright-year>
<copyright-holder>Behboodi, Kline, Gravunder, Phillips, Parker and Damiano</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p>
</license>
</permissions>
<abstract>
<p>In the realm of motor rehabilitation, Brain-Computer Interface Neurofeedback Training (BCI-NFT) emerges as a promising strategy. This aims to utilize an individual&#x2019;s brain activity to stimulate or assist movement, thereby strengthening sensorimotor pathways and promoting motor recovery. Employing various methodologies, BCI-NFT has been shown to be effective for enhancing motor function primarily of the upper limb in stroke, with very few studies reported in cerebral palsy (CP). Our main objective was to develop an electroencephalography (EEG)-based BCI-NFT system, employing an associative learning paradigm, to improve selective control of ankle dorsiflexion in CP and potentially other neurological populations. First, in a cohort of eight healthy volunteers, we successfully implemented a BCI-NFT system based on detection of slow movement-related cortical potentials (MRCP) from EEG generated by attempted dorsiflexion to simultaneously activate Neuromuscular Electrical Stimulation which assisted movement and served to enhance sensory feedback to the sensorimotor cortex. Participants also viewed a computer display that provided real-time visual feedback of ankle range of motion with an individualized target region displayed to encourage maximal effort. After evaluating several potential strategies, we employed a Long short-term memory (LSTM) neural network, a deep learning algorithm, to detect the motor intent prior to movement onset. We then evaluated the system in a 10-session ankle dorsiflexion training protocol on a child with CP. By employing transfer learning across sessions, we could significantly reduce the number of calibration trials from 50 to 20 without compromising detection accuracy, which was 80.8% on average. The participant was able to complete the required calibration trials and the 100 training trials per session for all 10 sessions and post-training demonstrated increased ankle dorsiflexion velocity, walking speed and step length. Based on exceptional system performance, feasibility and preliminary effectiveness in a child with CP, we are now pursuing a clinical trial in a larger cohort of children with CP.</p>
</abstract>
<kwd-group>
<kwd>associative learning</kwd>
<kwd>pediatric</kwd>
<kwd>NMES</kwd>
<kwd>transfer learning</kwd>
<kwd>motor training</kwd>
</kwd-group>
<counts>
<fig-count count="8"/>
<table-count count="3"/>
<equation-count count="0"/>
<ref-count count="46"/>
<page-count count="16"/>
<word-count count="12818"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Motor Neuroscience</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="sec1">
<label>1</label>
<title>Introduction</title>
<p>Brain-Computer Interface (BCI) mediated neurofeedback training (BCI-NFT) involves using an individual&#x2019;s brain signals during a motor task to trigger a robotic device or neuromuscular electrical stimulation (NMES), providing real-time sensory feedback and motor assistance. Thereby, BCI-NFT aims to fortify sensorimotor pathways, potentially fostering motor learning and neuroplasticity (<xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>). BCI-NFT has been used effectively for motor rehabilitation in those with neurological disorders, especially individuals with stroke, as demonstrated in multiple systematic reviews (<xref ref-type="bibr" rid="ref27">Monge-Pereira et al., 2017</xref>; <xref ref-type="bibr" rid="ref9">Cervera et al., 2018</xref>; <xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>; <xref ref-type="bibr" rid="ref24">Kruse et al., 2020</xref>; <xref ref-type="bibr" rid="ref39">Penev et al., 2023</xref>; <xref ref-type="bibr" rid="ref40">Qu et al., 2024</xref>).</p>
<p>BCI-NFT systems typically utilize cortical activity associated with Motor Imagery (MI) or Motor Attempt (MA), as recorded by electroencephalography (EEG), to activate an external device. This device in turn induces stimulation or movement in the targeted body part, with or without additional visual feedback. MI involves mentally rehearsing movements without physical execution, and its neural substrates exhibit substantial overlap with the motor execution network (<xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>). MI has been employed effectively as the method of choice in most BCI-NFT systems. In a randomized controlled trial (RCT), Ang et al. utilized an MI-based BCI with the MIT-Manus robot to train reaching tasks in 24 individuals post-stroke (<xref ref-type="bibr" rid="ref2">Ang et al., 2010</xref>), while Frolov et al. employed a hand exoskeleton to train grasping in 74 individuals post-stroke (<xref ref-type="bibr" rid="ref16">Frolov et al., 2017</xref>). Both studies revealed significant improvements in Upper Extremity (UE) Fugl Meyer Assessment (FMA) scores.</p>
<p>Despite disruptions in the sensorimotor loop in persons with neurological disorders, some motor planning or movement capabilities may be preserved. Thus, these individuals can employ MA, instead of MI, to activate an external device via BCI through attempted movements, even in cases of complete paralysis (<xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>). In a controlled trial, Biasiucci et al. deployed an MA-based BCI-NFT to train wrist and finger extension of 27 participants post-stroke using NMES. They demonstrated significant improvements in UE-FMA scores, which notably lasted 9&#x2009;months post-intervention (<xref ref-type="bibr" rid="ref7">Biasiucci et al., 2018</xref>).</p>
<p>In MI-based rehabilitation, the central nervous system may need to actively suppress limb movement while the participant is visualizing the intended action, which requires learning and sustained concentration (<xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>). Additionally, MI may be particularly challenging for young or cognitively challenged individuals (<xref ref-type="bibr" rid="ref8">Bobrov et al., 2020</xref>). Even among healthy adults, mastering BCI systems through MI requires practice (<xref ref-type="bibr" rid="ref44">Vidaurre and Blankertz, 2010</xref>; <xref ref-type="bibr" rid="ref46">Zhang et al., 2020</xref>). Most participants have difficulty controlling the system initially and many, but not all, are able to gradually improve their BCI accuracy (<xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>). Conversely, MA is considered more natural and verifiable. In neurofeedback training aimed at promoting or restoring motor functioning, choosing MA may improve outcomes by maximizing parallels between the brain-state used for control and the actual task, addressing concerns of task specificity that arise in MI paradigms, where maintaining consistent focus on the motor task is not assured (<xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>). In a recent review, Bai et al. demonstrated that MI may be less effective for motor rehabilitation compared with MA (<xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>).</p>
<p>Regardless of the brain state utilized (MI or MA), BCI-NFT systems typically employ classifiers to establish a mapping between brain activity and activation of external sensory stimuli, in real-time. Brain-state dependent simulation is another type of system for motor rehabilitation that uses data collected during a calibration phase to estimate the mean timing of the peak negative deflection prior to movement execution and employs that value to activate electrical stimulation or robotic assistance during motor training. While effective, these systems cannot be strictly considered BCI-NFT since the brain state, usually MA, is not detected in real-time, i.e., there is not an interface between the brain and the external device during motor training (<xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al., 2019</xref>, <xref ref-type="bibr" rid="ref29">2021</xref>). In two RCTs utilizing brain-state dependent stimulation, including 22 (<xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al., 2016</xref>) and 24 (<xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al., 2019</xref>) participants with stroke, Mrachacz-Kersting et al. showed significant improvements in Lower Extremity (LE) motor function as measured by the LE-FMA, 6-meter Walk Test, and 10-meter Walk Test. The timing estimation was based on the peak negativity (PN), a prominent deflection of low-frequency movement-related cortical potentials (MRCP) (<xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al., 2016</xref>) calculated during calibration, and was used to trigger the NMES system during training in a feedforward manner. PN is associated with motor intent and is usually detectable within a window of 500&#x2009;ms prior to movement execution (<xref ref-type="bibr" rid="ref29">Mrachacz-Kersting et al., 2021</xref>).</p>
<p>The selection of an external device, normally NMES or a robot, may also impact rehabilitation outcomes in BCI-NFT. Bai et al., in a subgroup meta-analysis, demonstrated the statistical superiority of BCI-driven NMES over BCI-driven robots for rehabilitation (<xref ref-type="bibr" rid="ref3">Bai et al., 2020</xref>). Behboodi et al., however, did not observe a difference in effectiveness between the two devices in a MA-based BCI-NFT review in neurological populations (<xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>). NMES is thought to be effective in fortifying the sensorimotor loop during BCI training through a more robust engagement of proprioceptive components, such as muscle spindles, in comparison to robotic devices; thereby increasing movement awareness during motor training and enhancing corticospinal excitability (<xref ref-type="bibr" rid="ref18">Hu et al., 2015</xref>).</p>
<p>To induce motor learning, BCI-NFT studies deploy operant conditioning and associative learning paradigms. Operant conditioning is the most prevalent learning paradigm in BCI-NFT. It centers on inducing alterations in brain activity patterns, providing users with meaningful feedback to modulate their neural responses and maximize a reward (desired movement), thereby learning new motor skills. For instance, in the RCT by Ramos-Murguialdy et al. on 30 participants post-stroke, the experimental group received NMES if their Mu (8&#x2013;13&#x2009;Hz) band power was maintained below a predefined threshold (<xref ref-type="bibr" rid="ref41">Ramos-Murguialday et al., 2019</xref>). In associative rehabilitative NFT paradigms, an endogenous task-specific brain state is temporally linked to sensory feedback, in accordance with the principle of Hebbian association which states that when two neurons repeatedly activate synchronously, they become more likely to fire together over time. The main advantages of this paradigm are its intuitive nature, which requires minimal or no training for learning how to control the system, and its potential to reduce the physical training dose required for meaningful motor improvements (<xref ref-type="bibr" rid="ref29">Mrachacz-Kersting et al., 2021</xref>), as evidenced by the reported enhancements in lower limb functionality after just 20&#x2009;min (30 pairings of brain-state and sensory stimuli) (<xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al., 2016</xref>). To create the requisite association for neuroplasticity, it is imperative that the sensory feedback coincides with the peak cortical activation during MA. Positive correlations between induced plasticity and detection accuracy as demonstrated by <xref ref-type="bibr" rid="ref35">Niazi et al. (2012)</xref>, underscore the importance of classification algorithms that can detect the target EEG features quickly and accurately.</p>
<p>Training a classifier involves an extensive calibration period, and not all systems can achieve sufficient accuracy. The lower the accuracy, the more likely it is that subjects will not achieve the desired result consistently, or they will receive inappropriate feedback even when performing the task correctly (<xref ref-type="bibr" rid="ref20">Irimia et al., 2018</xref>). BCI performance, as quantified by classification accuracy (number of correct classifications/total detection attempts), and the prevalence of false positives, significantly influences operant conditioning and associative learning. False negatives impede reward strategies crucial for patient motivation, while false positives can reinforce unintended actions, potentially leading to incorrect and mal-adaptive associations. The selection of source localization techniques, EEG features, EEG channels, and classification algorithms profoundly impacts detection accuracy. In operant conditioning, simple thresholding algorithms (<xref ref-type="bibr" rid="ref21">Kim and Lee, 2016</xref>), similar to that used by <xref ref-type="bibr" rid="ref41">Ramos-Murguialday et al. (2019)</xref>, or machine learning classifiers like support vector machines (SVM) and linear discriminant analysis (LDA) are used to detect event-related desynchronization (ERD) in the EEG sensorimotor frequency band (~8&#x2013;30&#x2009;Hz), with accuracy ranging from 70 to 85%. Notably, a commercially available MI-based BCI-NFT system, RecoveriX, utilizes a Common Spatial Filter (CSP) and an LDA classifier achieving a mean accuracy of 87.4% across sessions in distinguishing left from right hand MI in stroke (<xref ref-type="bibr" rid="ref20">Irimia et al., 2018</xref>).</p>
<p>Most BCI-NFT systems utilizing associative learning rely on Movement-related cortical potentials (MRCP) as their primary EEG feature. MRCP are low frequency EEG signals that demonstrate a negative shift that starts approximately 1.5&#x2013;2&#x2009;s before movement onset and thus enable one to detect motor intent before muscle activation. The peak in this negative deflection typically occurs within 500&#x2009;msec of movement onset as described in a recent review (<xref ref-type="bibr" rid="ref43">Shakeel et al., 2015</xref>). This makes detection of MRCP particularly applicable to BCI applications as well as rehabilitation ones because these facilitate a more timely (nearly real-time) response from a robotic, prosthetic or electrical stimulation device that will control or assist a movement. These have also been used often for neurofeedback applications because the detection of motor intent before an imagined or attempted movement can be used to assist the limb movement and provide enhanced sensory input to the motor cortex at the time that the brain is most active. The inherent assumption is that the closer this input occurs to the motor imagery or voluntary attempt, the stronger the facilitation of neuroplastic changes.</p>
<p>Depending on whether the motor task is cued as was done here (i.e., participants are warned that they will soon be given the cue to move, and when that is given, they should move as quickly as possible) or self-paced (participant is given an auditory or visual signal letting them know that they can perform the task whenever they want after an at least 2&#x2009;s pause), the timing of the start of the negative deflection may vary given the two paradigms and this pre-movement negative signal may be referred to as Contingent Negative Variation, Bereitschafts or Readiness Potential or the Peak Negativity (<xref ref-type="bibr" rid="ref43">Shakeel et al., 2015</xref>; <xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al., 2019</xref>).</p>
<p>To address the temporal variability of MRCP both within and across subjects which limits its predictability, efforts have been made to detect MRCP features like PN in real-time. Niazi et al. introduced a supervised detection method utilizing a matched filter (<xref ref-type="bibr" rid="ref34">Niazi et al., 2013</xref>); however, the time-variable and uncertain nature of the EEG signals reduces the effectiveness of matched filters (<xref ref-type="bibr" rid="ref42">Ren et al., 2014</xref>). Thus, Niazi et al. then proposed an optimal spatial filter and achieved a detection accuracy of 82.5% in healthy individuals (<xref ref-type="bibr" rid="ref34">Niazi et al., 2013</xref>). Xu et al. implemented a manifold learning algorithm followed by LDA, achieving over 80% detection accuracy (<xref ref-type="bibr" rid="ref42">Ren et al., 2014</xref>). Bhagat et al. used SVM and demonstrated 79&#x2009;&#x00B1;&#x2009;18% accuracy and a 23&#x2009;&#x00B1;&#x2009;20% false positive rate (<xref ref-type="bibr" rid="ref6">Bhagat et al., 2020</xref>). Recent endeavors have explored the use of Deep Learning models like Multilayer Perceptron Neural Networks (MLP-NN) (<xref ref-type="bibr" rid="ref4">Behboodi et al., 2022</xref>) and Convolutional Neural Networks (CNN) (<xref ref-type="bibr" rid="ref25">Lawhern et al., 2018</xref>) in detecting MRCP features in healthy individuals. These models typically employ a larger number of electrodes (21 and 64, respectively) and require minimal pre-processing and spatial filtering to identify EEG features.</p>
<p>The encouraging outcomes of BCI-NFT for restoring motor function in stroke provide the impetus for extending this paradigm to other neurological disorders like cerebral palsy (CP). CP is the most commonly diagnosed child-onset motor disability, characterized by compromised selective motor control. The efficacy of BCI-NFT in individuals with CP remains understudied with only a few studies utilizing various paradigms. For instance, Bobrov et al. employed a MI BCI-driven hand exoskeleton for feedback, conducting training sessions with 14 children diagnosed with CP (<xref ref-type="bibr" rid="ref8">Bobrov et al., 2020</xref>). Remarkable enhancements in hand function, as measured by FMA, were induced after 7&#x2013;10&#x2009;weeks of training. This BCI system also demonstrated effectiveness in stroke (<xref ref-type="bibr" rid="ref23">Kotov et al., 2014</xref>; <xref ref-type="bibr" rid="ref16">Frolov et al., 2017</xref>). Kim et al. conducted an RCT involving 18 children with CP, revealing that MI BCI-driven NMES for hand extension training might evoke a more bilateral sensorimotor rhythm in frontopolar regions compared to NMES alone, after 30 half-hour training sessions (<xref ref-type="bibr" rid="ref21">Kim and Lee, 2016</xref>). However, this study did not report specific motor outcomes. In another study (<xref ref-type="bibr" rid="ref36">Olsen et al., 2021</xref>), participants with CP showed enhanced performance of the non-dominant hand on a serial reaction time task following classic neurofeedback training which involved attempts to modulate alpha power, facilitated with visual feedback. There is a specific need for the integration of MA and associative learning without the necessity of consistent imagination of specific movements or attempts to up- or down-regulate a motor-related EEG frequency band. MA significantly enhances the feasibility and accessibility of BCI-NFT for young children or individuals with cognitive impairments.</p>
<p>Our goal here was to develop and implement a real-time EEG BCI-NFT device that could be utilized in CP and other neurorehabilitation populations. Based on our scoping review of existing BCI-NFT methodologies (<xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>), we focused on two major technological challenges when designing our system. First, we aimed to match or ideally improve detection accuracy beyond what was reported in the literature to reduce the amount of inappropriate feedback delivered during training. Second, we aimed to optimize the timing of EEG detection of motor intent in relation to the activation of sensory feedback delivered via NMES, so that the stimulation assisted the movement and was delivered when the sensorimotor cortex was the most active during the motor task. Here we describe these and other key components of our system, with a summary of the rationale behind the design decisions for each. Finally, to validate that the system performed as intended for a child with CP and to provide preliminary data on its potential effectiveness, we present data for our first clinical study participant. Although the system can be adapted for a range of motor tasks, ours was customized to train ankle dorsiflexion active range and joint angular velocity in those with CP who demonstrated reduced selective ankle control.</p>
</sec>
<sec sec-type="methods" id="sec2">
<label>2</label>
<title>Methods</title>
<p>We developed a BCI-NFT system based on an associative learning paradigm which aims to establish a Hebbian association between task-specific neural activity, during attempts to dorsiflex the ankle, and sensory feedback induced via application of NMES.</p>
<sec id="sec3">
<label>2.1</label>
<title>Description of our BCI-NFT system</title>
<p>Our BCI-NFT system utilized MRCP of the EEG signal (0.05&#x2013;10&#x2009;Hz) to detect the participant&#x2019;s motor intent within a window of 500&#x2009;ms prior to movement onset. The chosen frequency band was based on the MRCP range utilized by <xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al. (2016)</xref> and <xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al. (2019)</xref>; however, other researchers may utilize an even more truncated frequency range for MRCP (<xref ref-type="bibr" rid="ref36">Olsen et al., 2021</xref>). The participant was tasked to dorsiflex the ankle at a specific time, cued by a graphic user interface (GUI), which also provided visual feedback of the participant&#x2019;s ankle angle in real-time. Once the BCI system detected the participant&#x2019;s intention to dorsiflex, the NMES component was activated to stimulate the tibialis anterior (TA) muscle (<xref ref-type="fig" rid="fig1">Figure 1</xref>).</p>
<fig position="float" id="fig1">
<label>Figure 1</label>
<caption>
<p>Diagram of our Brain-computer interface neurofeedback training (BCI-NFT) system which utilizes MRCP of the EEG signal to detect participant motor intent. The participant was tasked to dorsiflex the ankle at a specific time, cued by a graphic user interface (GUI), which also provided visual feedback of the ankle angle in real-time. EEG data were recorded using a 64-channel EEG system and streamed wirelessly to the BCI-NFT software using Stimulation Delivery. Upon detection of motor intent from EEG, the BCI-NFT system triggered a stimulator to apply assistive sensory feedback (NMES of the tibialis anterior muscle). To monitor ankle angle in real-time, two Inertial Measurement Units (IMUs) continuously streamed joint angle data to the visual feedback GUI in Unity.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g001.tif"/>
</fig>
<sec id="sec4">
<label>2.1.1</label>
<title>Hardware</title>
<sec id="sec5">
<label>2.1.1.1</label>
<title>Joint angle</title>
<p>To monitor ankle angle in real-time, we used two Inertial Measurement Units (IMUs), LPMS B2 series (Life performance Research, Tokyo, Japan). These IMUs facilitated the continuous streaming of joint angle data to the visual feedback GUI. The real-time calculation of joint angle was accomplished through a Unity interface developed by the IMU manufacturer.</p>
</sec>
<sec id="sec6">
<label>2.1.1.2</label>
<title>EEG acquisition</title>
<p>EEG data were recorded using a 64-channel active EEG system (Brain Products, Munich, Germany) at a sampling frequency of 500&#x2009;Hz. EEG data were streamed wirelessly to the BCI-NFT software using the BrainAmp amplifier (Brain Products, Munich, Germany).</p>
</sec>
<sec id="sec7">
<label>2.1.1.3</label>
<title>Stimulation delivery</title>
<p>Upon detection of motor intent from EEG, the BCI-NFT system triggered a stimulator, via serial communication, to apply assistive sensory feedback (NMES to the tibialis anterior muscle of the more affected limb). The stimulator was a modified version of a Food &#x0026; Drug Administration (FDA)-approved four-channel LG-8TM Elite Electrical Muscle Stimulator (LG Med Supply).</p>
</sec>
</sec>
<sec id="sec8">
<label>2.1.2</label>
<title>Software</title>
<p>The software consisted of three primary components:<list list-type="order">
<list-item>
<p><italic>Visual feedback GUI in Unity</italic> (Unity Software Inc., San Francisco, CA): This component was responsible for providing task cues and real-time visual feedback of the participant&#x2019;s ankle angle.</p>
</list-item>
<list-item>
<p><italic>Operating GUI in MATLAB</italic> (MathWorks, Natick, MA): The MATLAB-based interface was used for configuring detection parameters and real-time execution of the BCI.</p>
</list-item>
<list-item>
<p><italic>LabStreamingLayer (LSL)</italic>: LSL (<xref ref-type="bibr" rid="ref22">Kothe et al., 2024</xref>) was utilized to synchronize data streams, including joint angle data, triggers from the visual feedback GUI, and EEG signals.</p>
</list-item>
</list></p>
<p><xref ref-type="fig" rid="fig2">Figure 2</xref> is a visual representation of the data flow within the software. The visual feedback GUI transmitted trigger values associated with various task events (provided as visual cues to the participant) and ankle angle data to LSL. EEG was directly streamed into LSL in real-time using an LSL plugin provided by Brain Products (Munich, Germany), the EEG system manufacturer. LSL played a crucial role in synchronization and timestamping all data streams, i.e., EEG, ankle angle, and triggers. Subsequently, it transmitted the EEG and trigger streams to the BCI system.</p>
<fig position="float" id="fig2">
<label>Figure 2</label>
<caption>
<p>Information flow of the BC-NFT software. The visual feedback GUI transmitted trigger values associated with various task events (provided as visual cues to the participant) and ankle angle data to LSL. EEG was directly streamed into the Lab Streaming Layer (LSL) in real-time. LSL played a crucial role in synchronization and timestamping all data streams, i.e., EEG, ankle angle, and triggers. Subsequently, it transmitted the EEG and trigger streams to the BCI system in MATLAB.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g002.tif"/>
</fig>
<p>Certain events during the task triggered the visual feedback GUI to send specific values to the operating GUI, through LSL, to trigger different components of the real-time BCI structure. Notably, trigger value 8 signaled the beginning of each trial, trigger value 6 indicated the initiation of the movement preparation period (as described in section 2.2.2), and trigger value 7 marked the conclusion of each training block (set of five trials). Furthermore, the visual feedback GUI communicated trial-specific information, including the trial name and number, directly to the operating GUI using a Transmission Control Protocol/Internet Protocol (TCPIP) connection.</p>
<sec id="sec9">
<label>2.1.2.1</label>
<title>BCI structure</title>
<p>The BCI structure encompassed three primary processes:<list list-type="order">
<list-item>
<p><italic>Initialization:</italic> Upon initiation of data collection, an initialization function established four essential communication channels, illustrated in <xref ref-type="fig" rid="fig2">Figure 2</xref>. These channels included: (1) a link between BrainVision and LSL for real-time EEG streaming, (2) a pathway from Unity (visual feedback) to LSL for reception of trigger values and ankle angle in real-time, (3) a connection from LSL to MATLAB for the streaming of EEG and trigger values to the operating GUI, and (4) a direct link from Unity to MATLAB for transmission of trial information, such as the trial name. Following setup of these communication channels, LSL commenced recording and time stamping of EEG, trigger values, and ankle angle streams. Subsequently, a timer was initiated in MATLAB to execute the real-time BCI function every 50&#x2009;ms, which retrieved the most recent trigger value and the latest 50&#x2009;ms of the EEG stream from LSL.</p>
</list-item>
<list-item>
<p><italic>Real-time BCI:</italic> During the neurofeedback training sessions, following the receipt of trigger 6, real time detection of Peak Negativity (PN) was initiated: the start of the prep bar (Unity trigger 6) was considered time 0. At 1,000&#x2009;ms, we received inputs consisting of 64 channel &#x00D7; 25 (100&#x2009;ms) packets of 50 EEG samples each. We then classified each sample, so our model outputted a series of 50 &#x201C;0&#x201D;s (rest) or &#x201C;1&#x201D;s (motor intent). If the detection model reached the number of 1&#x2009;s as defined by a threshold value in any packet, it would trigger the NMES. EEG was epoched from 1,000 to 4,000&#x2009;ms for training the LSTM model. The process started with packets 1&#x2013;5, 500&#x2009;ms after the start of the epoch (from time 1,000&#x2013;1,500&#x2009;ms). The actual detection started after packet 5 was received, at 1,500&#x2009;ms after the prep bar started to move. The time between the receipt of packets 5&#x2013;14 (1500&#x2013;2,500&#x2009;ms) should be designated by the detection process as rest. The participant is instructed to dorsiflex 3,000&#x2009;ms after the start of the prep bar. As the PN detection window was considered 500&#x2009;ms around this time (2,500&#x2013;3,500&#x2009;ms), packets 15&#x2013;25 are associated with the PN detection window. If PN was detected and NMES triggered in packets 5&#x2013;14, it was triggered during rest and was a false positive (FESfp variable); if PN was detected and NMES triggered within packets 15&#x2013;25 (2,500&#x2013;3,500&#x2009;ms), it was triggered during motor planning and was a true positive. At that point (3,500&#x2009;ms), we had an automatic NMES trigger if PN had not been detected and detection was therefore ended. We utilized the offline code to calculate false negatives which were defined as the failure to find at least one packet between numbers 15&#x2013;25 with a total of 50 samples indicating motor intent. This process continued until trigger 7 (end of 5th trial in block) was received. Subsequently, the finalization function was triggered.</p>
</list-item>
<list-item>
<p><italic>Finalization</italic>: This function halted the LSL recording and converted the recorded EEG data from the entire training block into EEGLAB&#x2019;s preferred set format to facilitate post-hoc EEG analysis and generation of Event-Related Spectral Perturbation (ERSP) plots via the MATLAB-compatible EEGLAB toolbox (<xref ref-type="bibr" rid="ref14">Delorme and Makeig, 2004</xref>) during data collection.</p>
</list-item>
</list></p>
</sec>
</sec>
</sec>
<sec id="sec10">
<label>2.2</label>
<title>Data collection protocol</title>
<sec id="sec11">
<label>2.2.1</label>
<title>Setup</title>
<p>The participant was seated in front of a monitor in a long sitting position on an examination table with the hip flexed approximately 30&#x00B0;, the knee supported in a small amount of flexion for comfort, and the ankle elevated slightly off the mat to enable unencumbered movement. For real-time tracking of ankle angle, the IMUs were placed on the top of the midfoot directly over the metatarsals and on the lateral side of the shank. The 64 EEG electrodes were positioned using the 5% 10&#x2013;20 international system and actiCap (Brain Products) with FCz designated as the reference point. Conductive gel was inserted between the electrodes and scalp until the acceptable signal quality was achieved, which was indicated by a green LED on the electrodes. When the impedance between the electrode and the scalp was less than 20 kilo ohms, the LED on the EEG electrodes turned green.</p>
</sec>
<sec id="sec12">
<label>2.2.2</label>
<title>Training task and the visual feedback GUI</title>
<p>Each training block consisted of five trials of 10&#x2009;s duration. At the beginning of each trial, the GUI prompted the participant to relax; this cue lasted 3&#x2009;s. Subsequently, a horizontal red preparation bar (as illustrated in <xref ref-type="fig" rid="fig1">Figure 1</xref>) appeared on the screen to cue the subject to prepare to dorsiflex. The red bar was filled from left to right with a green bar which took 3&#x2009;s. The participant was instructed to dorsiflex &#x201C;as fast and far as possible&#x201D; immediately after the green color reached the end of the bar. Once the movement execution phase began, the participant received real-time feedback of their ankle angle, represented as a vertical bar (angle bar). The participant was instructed to reach and attempt to surpass the target region at the top of the angle bar, which indicated 80&#x2009;+&#x2009;% of their maximum dorsiflexion from the previous trial. The participant maintained the dorsiflexed position for 3&#x2009;s until cued to relax again.</p>
</sec>
</sec>
<sec id="sec13">
<label>2.3</label>
<title>Motor intent detection</title>
<p>To identify an optimal detection method for our protocol we evaluated the performance of four different algorithms in detecting the motor preparation phase: (1) Thresholding, where a trial-specific threshold value was set in real-time using the mean of the Cz channel&#x2019;s Event-Related Potentials (ERP) during the rest period of each trial; (2) Average PN, in which the mean timing of PN was determined during the calibration trials and then implemented during training (<xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al., 2019</xref>) (not real-time); and two deep learning models both enabling real-time detection of brain activity; (3) a Multi-layer perceptron neural network (MLP-NN) (<xref ref-type="bibr" rid="ref4">Behboodi et al., 2022</xref>), and (4) a Long-short term memory (LSTM) neural network. Machine Learning (ML) is a type of Artificial Intelligence (AI) that enables the analysis and synthesis of very large datasets, while Deep Learning (DL), a ML methodology, is a type of recurrent neural network especially proficient at extracting meaningful patterns from datasets and which is increasingly being utilized in healthcare settings (<xref ref-type="bibr" rid="ref1">Aminizadeh et al., 2024</xref>). The proposed benefit of LSTM is to appreciably reduce calibration time when used repeatedly across sessions in an individual participant by utilizing and learning from their previous data over time, a feature which could be particularly advantageous when working with a pediatric population.</p>
<p>The parameters of each of these methods were adjusted during a calibration period. All of these algorithms utilized preprocessed EEG signals as input both during the calibration phase and for real-time BCI with the exception of Average PN.</p>
<sec id="sec14">
<label>2.3.1</label>
<title>EEG preprocessing</title>
<p>The 64-channel EEG signals were band pass filtered to retain the MRCP frequency band (0.05 and 10&#x2009;Hz). The signals were then spatially filtered to localize the prominent sources of brain activity and 20 channels situated over the sensorimotor cortex (FC5, FC3, FC1, FC2, FC4, FC6, C5, C3, C1, Cz, C2, C4, C6, CP5, CP3, CP1, CPz, CP2, CP4, CP6), involved with motor planning and execution, were extracted as the inputs for our movement intention detection algorithms.</p>
</sec>
<sec id="sec15">
<label>2.3.2</label>
<title>MRCP</title>
<p>MRCP (<xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al., 2019</xref>) have been recognized previously as reliable signals for predicting movement intention (<xref ref-type="bibr" rid="ref29">Mrachacz-Kersting et al., 2021</xref>). PN, a distinct feature of MRCP which typically occurs within 500&#x2009;ms before movement onset, has been associated with the neurophysiological processes of motor planning and execution (<xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al., 2016</xref>).</p>
</sec>
<sec id="sec16">
<label>2.3.3</label>
<title>Calibration</title>
<p>In each session, the initial five blocks of training, comprising 25 trials, were allocated for calibration of the detection model. For each trial, the preprocessed EEG data were segmented, or &#x201C;epoched,&#x201D; from 3&#x2009;s before the participant&#x2019;s cue to dorsiflex (initiation of preparation phase) until 1&#x2009;s after the cue. To pinpoint the timing of PN for each trial, the minimum (maximum negative) values were computed during the movement preparation period, which extended from 500&#x2009;ms prior to the cue to dorsiflex to 500&#x2009;ms after the cue. This extended period was chosen because it allowed for potential variations in the onset of movement execution during the period when the PN is most likely to occur. Finally, all of the epochs from each channel were consolidated and averaged to generate the calibration ERP or MRCP signal.</p>
</sec>
<sec id="sec17">
<label>2.3.4</label>
<title>Real-time detection/BCI</title>
<sec id="sec18">
<label>2.3.4.1</label>
<title>Threshold algorithm</title>
<sec id="sec19">
<label>2.3.4.1.1</label>
<title>Calibration</title>
<p>The threshold for PN detection was computed using the mean and standard deviation for the first second of the rest portion of the calibration ERP from the central Cz electrode, characterized by a low likelihood of Peak Negativity occurrence. The ERP was then graphed alongside three threshold values represented as horizontal lines. These lines corresponded to 1, 2, and 3&#x2009;standard deviations above the mean of the rest period. The appropriate standard deviation (Std) multiplier (<xref ref-type="bibr" rid="ref27">Monge-Pereira et al., 2017</xref>; <xref ref-type="bibr" rid="ref24">Kruse et al., 2020</xref>; <xref ref-type="bibr" rid="ref5">Behboodi et al., 2022</xref>) for use during real-time BCI processing was visually identified.</p>
</sec>
<sec id="sec20">
<label>2.3.4.1.2</label>
<title>BCI</title>
<p>During neurofeedback training, preprocessed EEG data were fed into the algorithm every 50&#x2009;ms (as detailed in section 2.1.2.1). If the mean of the Cz signal exceeded the threshold value chosen during the calibration phase, NMES was triggered. This algorithm used a Laplacian spatial filter.</p>
</sec>
</sec>
<sec id="sec21">
<label>2.3.4.2</label>
<title>Average PN</title>
<p>In this algorithm, the average PN timing calculated during the calibration trials was utilized to trigger the NMES. During each trial, once the elapsed time calculated by the BCI timer reached the average PN timing, NMES was triggered (i.e., no real-time detection of PN). This open loop algorithm employed a Laplacian spatial filter as in the brain state stimulation system utilized by <xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al. (2016)</xref> and <xref ref-type="bibr" rid="ref31">Mrachacz-Kersting et al. (2019)</xref>.</p>
</sec>
<sec id="sec22">
<label>2.3.4.3</label>
<title>Long-short term memory neural network</title>
<p>LSTM is a prominent type of recurrent neural network (RNN) renowned for its capability to learn both short and long-term temporal dependencies within time series data. Consequently, LSTM has extensive applications in sequence prediction scenarios. To discern minute temporal alterations within our preprocessed EEG data (our feature space) for the classification of participants&#x2019; intention to move, a solution that can memorize and subsequently accumulate these transitions is required. This demands the utilization of the inherent long and short-term memories in this deep learning model (<xref ref-type="bibr" rid="ref45">Zhang et al., 2020</xref>).</p>
<sec id="sec23">
<label>2.3.4.3.1</label>
<title>Calibration</title>
<p>First, the preprocessed EEG were epoched into two classes, rest and motor intent (<xref ref-type="fig" rid="fig3">Figure 3</xref>). These epochs were labeled and used for the supervised training of our LSTM classifier. The classifier was constructed using the MATLAB Deep Learning toolbox and comprised four layers: an input layer with 21 nodes, each corresponding to one of the 20 EEG channels over the sensorimotor cortex; a bilateral LSTM layer consisting of 100 nodes (neurons) responsible for memorizing long and short term dependencies in the feature space; a fully connected layer (dense layer), a typical neural network layer; and a SoftMax layer serving as the output layer, responsible for generating probability distributions of our two classes (as illustrated in <xref ref-type="fig" rid="fig2">Figure 2B</xref>). The training process spanned 100 epochs and utilized the Adam optimizer.</p>
<fig position="float" id="fig3">
<label>Figure 3</label>
<caption>
<p>EEG preprocessing and LSTM training workflow <bold>(A)</bold> and LSTM structure <bold>(B)</bold>.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g003.tif"/>
</fig>
</sec>
<sec id="sec24">
<label>2.3.4.3.2</label>
<title>BCI</title>
<p>During the neurofeedback training, at each iteration of the BCI timer every 50&#x2009;ms, a preprocessed EEG matrix, 64 (number of channels) by 25 (number of samples), was fed into the trained LSTM model. The model labeled each sample and generated an array of 25 binary elements, where 0 signified the &#x201C;rest&#x201D; class and 1 indicated the &#x201C;motor intent&#x201D; class. The sum of the elements within this array was then compared to a predetermined threshold, which in this case was set at 80% of the number of elements. If the sum surpassed this threshold, the EEG data sample was labeled &#x201C;motor intent&#x201D; and this triggered the NMES feedback.</p>
</sec>
</sec>
<sec id="sec25">
<label>2.3.4.4</label>
<title>Multi-layer perceptron neural network</title>
<p>The MLP calibration and BCI procedure are similar to that of the LSTM; its structure, however, is different. Our proposed MLP consisted of two hidden layers, with a dimension of 20 at each layer (20 nodes); each node had ReLu activation function (<xref ref-type="bibr" rid="ref15">Djamal et al., 2020</xref>). Unlike LSTM, the MLP was trained with the Stochastic Gradient Descent (SGD) optimizer (<xref ref-type="bibr" rid="ref13">Craik et al., 2019</xref>) and a learning rate of 1. The preliminary results from the evaluation of this model are presented in <xref ref-type="bibr" rid="ref4">Behboodi et al. (2022)</xref>. The MLP structure was developed through a grid search approach. The number of batches, indicating the number of training time points (rows of the datasets) used per optimization step, and the number of hidden layers had minimal impact on the accuracy. Instead the accuracy was notably influenced by the layer dimensions, with a dimension of 20 for each hidden layer being identified as optimal. Additionally, a comparison of the Adam and SGD optimizer revealed a marginal effect on the accuracy. The learning rate of the optimizers, on the other hand, had a significant influence, with lr&#x2009;=&#x2009;0.001 and lr&#x2009;=&#x2009;0.01 performing optimally for Adam and SGD, respectively. Our final model utilized three hidden layers with 20 dimensions, a batch size of 8, a learning rate of 0.01, and the SGD optimizer.</p>
</sec>
</sec>
</sec>
<sec id="sec26">
<label>2.4</label>
<title>Participants</title>
<p>As a first step, the overall system performance was evaluated on eight healthy adults. Additionally, the detection algorithms were tested using the EEG data collected from this group. Based on the evaluation results, we selected the best performing algorithm for use in our clinical study in children with cerebral palsy (CP).</p>
<p>Subsequently, we present here a case report from the first participant with CP utilizing the finalized BCI-NFT system in a 10-session protocol designed to train better selective control of ankle dorsiflexion, to demonstrate the system&#x2019;s performance and the feasibility of our study protocol and to present some preliminary results in our target population.</p>
<p>Our protocol for both testing and implementing the system was approved by the NIH Institutional Review Board (#13-CC-0110), and all adult participants provided informed consent, with parent consent and child assent given for our participant with CP.</p>
<sec id="sec27">
<label>2.4.1</label>
<title>Healthy adult cohort</title>
<p>Eight healthy subjects (6 females; age: 27.3&#x2009;&#x00B1;&#x2009;7.1&#x2009;years, [22.7&#x2013;39.9]) with no history of neurological disease were enrolled in this study. This group participated in a single session of an ankle dorsiflexion training protocol using the BCI-NFT system to ensure that it functioned as intended and also to accumulate an EEG dataset for algorithm evaluation. At this developmental stage, NMES feedback was administered to the operator&#x2019;s forearm rather than to the participant&#x2019;s TA muscle which allowed the operator to assess whether the feedback activation occurred approximately at the intended time.</p>
<sec id="sec28">
<label>2.4.1.1</label>
<title>Development of the detection algorithm</title>
<p>While the Average PN algorithm had shown promise in previous clinical trials (<xref ref-type="bibr" rid="ref30">Mrachacz-Kersting et al., 2016</xref>, <xref ref-type="bibr" rid="ref31">2019</xref>), it was not considered a BCI system (<xref ref-type="bibr" rid="ref29">Mrachacz-Kersting et al., 2021</xref>), as it employed a predetermined timing in an open-loop manner to trigger stimulation during the neurofeedback sessions rather than real-time detection. Our aim, however, was to develop a real-time detection algorithm capable of handling the individual and trial-by-trial variability in PN timing. Nonetheless, the Average PN algorithm was utilized as a reference for evaluating the performance of two deep learning algorithms, MLP-NN and LSTM models. The detection accuracy of the MLP-NN and LSTM were first compared to each other and then the chosen algorithm was compared to the Average PN algorithm. For each deep learning model a fivefold evaluation was conducted using the first 50% of the trials for training the model and the rest for testing the accuracy.</p>
<p>Finally, to optimize the detection of our deep learning model, we conducted two analyses:<list list-type="order">
<list-item>
<p>Data labeling analysis: To train our deep learning model, it was necessary to label each calibration trial as &#x201C;rest&#x201D; or &#x201C;motor intent.&#x201D; We employed two different data labeling approaches: time-based and PN-based. <italic>Time-based labeling</italic> used the dorsiflexion cue, specifically the time when the preparation bar became fully green, as the reference point for labeling. One second around this reference point was designated as the &#x201C;motor intent&#x201D; class, while the second preceding it was marked as &#x201C;rest.&#x201D; <italic>PN-based labeling</italic> utilized the timing of PN in each trial as the reference point for data labeling. One second around the trial-specific PN time was assigned as the &#x201C;motor intent&#x201D; class, and the second preceding it was categorized as &#x201C;rest.&#x201D;</p>
</list-item>
<list-item>
<p>Spatial filtering analysis: We conducted an analysis to assess the impact of commonly used spatial filters for source localization, including Independent Component Analysis (ICA) and Laplacian on the accuracy of the deep learning models.</p>
</list-item>
</list></p>
</sec>
<sec id="sec31">
<label>2.4.1.2</label>
<title>Outcome measures for the healthy adult cohort</title>
<p>Accuracy is commonly defined as the ratio of correctly labeled EEG data samples to the total number of detections within the PN detection window, reported as a percentage. The commonly used accuracy metric for quantifying the performance of the real-time models, as defined here, is not relevant for the Average PN algorithm. Thus, to assess and compare the chosen real-time model&#x2019;s performance against the well-established Average PN, we utilized a detection error analysis. Error was computed as the temporal difference between the PN of each trial, representing the desired stimulation time, and the instant at which the chosen real-time model activated the NMES (<xref ref-type="fig" rid="fig4">Figure 4</xref>). Trial PN was defined as the absolute minimum of the preprocessed EEG within the detection window, i.e., 500&#x2009;ms before the cue to move and up to 500&#x2009;ms after the cue, in a virtual Cz channel (VCz). The VCz channel was generated by averaging EEG channels C1, C3, and Cz, located medially and centrally above the motor cortex. The detection error was presented as the mean&#x2009;&#x00B1;&#x2009;Std and root mean squared error (RMSE). Furthermore, accuracy of the selected real-time algorithm was optimized under two conditions: data labeling and spatial filtering. To evaluate the statistical significance of the observed differences in the detection performance of the models, a two-sample t-test was performed, comparing the mean accuracies of the selected real-time LSTM model with the established Average PN approach, employing a significance level of <italic>p</italic>&#x2009;&#x003C;&#x2009;0.05.</p>
<fig position="float" id="fig4">
<label>Figure 4</label>
<caption>
<p>Peak negativity (PN) error was computed as the temporal difference between the PN of each trial (red dashed line), representing the desired stimulation time, and the instant at which the model activated the NMES (blue dashed line). The shaded blue area around blue dashed line represents the potential variability in detecting each trial&#x2019;s PN.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g004.tif"/>
</fig>
</sec>
</sec>
<sec id="sec32">
<label>2.4.2</label>
<title>Participant with cerebral palsy</title>
<p>The goal of our BCI-NF training study is to improve selective control of ankle dorsiflexion in children with CP. This is a common impairment in those with either unilateral or bilateral CP. The protocol was to train the most affected ankle for each participant. Our first participant enrolled and completed the study in the summer of 2023, and we report the results here to demonstrate the robustness of our system as designed, illustrate the feasibility of the protocol for children with CP, and provide preliminary data on each of the functional and neural outcome measures for this participant.</p>
<p>The first participant was a 11.5-year-old boy with right unilateral cerebral palsy, classified as Gross Motor Function Classification System (GMFCS) level I. The etiology of his brain injury is not certain. His MRI indicated that he had diffuse white matter injury in the periventricular region typically consistent with preterm birth, even though he was born without complications at 38&#x2009;weeks plus 5&#x2009;days, and interestingly showed a focal injury in the right frontal region. However, his motor symptoms are mild and primarily isolated to his left upper and lower limbs. His main functional complaint relevant to this study was that it was difficult for him to run quickly because he tended to drag his right toes/forefoot. He was playing in a community baseball league and wanted to be able to run the bases faster.</p>
<sec id="sec33">
<label>2.4.2.1</label>
<title>Training protocol</title>
<p>He participated in a 10-session training regimen (<xref ref-type="fig" rid="fig5">Figure 5</xref>) during which he performed a minimum of 20 blocks of 5 dorsiflexion attempts, or 100 dorsiflexion trials, per session.</p>
<fig position="float" id="fig5">
<label>Figure 5</label>
<caption>
<p>Training regimen for the participant with CP. We conducted two assessment sessions, pre- and post-training, and 10 training sessions.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g005.tif"/>
</fig>
<p>Because our selected deep learning model hinges on an adequate amount of training data, the initial calibration session required data from 50 trials, or 10 blocks of 5 repetitions each. Subsequently, the model was trained with data from the 50 calibration trials from the initial calibration or 100 trials from the previous training session, and an additional 25 trials from each day&#x2019;s training session.</p>
<p>The task performed during each training session closely resembled the one described in section 2.2.2. with the notable additions that the participant received a performance score, in addition to ankle angle visual feedback during each trial and would receive electrical stimulation to the right TA activated by the EEG PN prior to each movement&#x2019;s onset. For the training, we set a target goal of at least 80% of his mean maximum active ankle dorsiflexion angle during the calibration trials and asked the participant to try to exceed that on each trial. A vertical bar provided real-time feedback on ankle dorsiflexion angle recorded from the IMUs with the target range indicated and participant performance presented in real-time. The stimulation electrodes were positioned on the participant&#x2019;s TA, and the stimulation intensity was adjusted in accordance with his comfort and tolerance levels. The stimulation pulse width and frequency remained constant at 150&#x2009;&#x03BC;s and 40&#x2009;Hz, respectively. However, we modulated the stimulation amplitude from 0 and incrementally increased it until reaching the maximum level that the subject found comfortable. Once determined, this amplitude was kept constant for the remainder of the session. This process was repeated for each session. The chosen frequency range and the pulse width fall within the standard parameters used in NMES research targeting the lower limb muscles of children with CP (<xref ref-type="bibr" rid="ref21">Kim and Lee, 2016</xref>; <xref ref-type="bibr" rid="ref28">Mooney and Rose, 2019</xref>).</p>
</sec>
<sec id="sec34">
<label>2.4.2.2</label>
<title>Assessment sessions</title>
<p>We conducted two assessment sessions, pre- and post-training. For these assessments, reflective markers were attached to anatomical landmarks on the participant&#x2019;s lower limbs using a modified Helen Hayes marker set. Active ankle dorsiflexion range of motion when seated as well as over ground gait kinematics when walking at freely-selected and fast speeds were recorded using a 12 camera 3D motion capture system (Vicon, Denver CO). The goal was to assess whether any potential improvements in ankle motion from the training would transfer to improvements in ankle motion during gait. While the motion capture markers were used to record ankle angle, the visual feedback for the ankle joint angle was based on IMU recordings.</p>
</sec>
<sec id="sec35">
<label>2.4.2.3</label>
<title>Outcome measures</title>
<p>The primary motor outcome measures were ankle dorsiflexion angle and joint angular velocity during task execution and during over ground free and fast speed walking. We also computed temporal&#x2013;spatial gait measures (velocity, cadence and step length). Additionally, system detection accuracy for EEG PN was assessed for the 10 training sessions to compare performance in CP to that of healthy adults. The primary neural outcome measure was the magnitude of event-related desynchronization (ERD) in the alpha (8&#x2013;12&#x2009;Hz) and beta (13&#x2013;30&#x2009;Hz) bands in comparable motor-related brain regions before and after training. We also assessed a laterality index for brain regions with ICs in both hemispheres for both timepoints. This was calculated as: (ERD on hemisphere contralateral to trained side &#x2013; ERD on ipsilateral side)/ (ERD on hemisphere contralateral to trained side + ERD on ipsilateral side).</p>
</sec>
<sec id="sec36">
<label>2.4.2.4</label>
<title>Transfer learning</title>
<p>To enhance user compliance and maximize the number of neurofeedback trials, we aimed to reduce the duration of each training session, eventually achieving a timeframe of less than 2&#x2009;h, including the setup time. The entire training protocol including set-up time, calibration trials, up to 100 motor attempts during training and ample rest time to minimize muscle fatigue or to take snack breaks never exceeded 3&#x2009;h. For our pediatric population with CP which is more likely to experience muscle fatigue and become bored with performing the same task repetitively, we decided to utilize a machine (transfer) learning algorithm here which can appreciably reduce the number of calibration trials once the initial model is built. The primary goal was to pre-train a model using all of the trials of the previous session, and subsequently fine-tune it during the calibration phase of the current session, as opposed to training an entirely new model for each session. This enables them to devote more of their effort and attention on the motor training itself. The session-by-session accuracy of the transferred LSTM model was calculated and compared with the accuracy of the non-transferred model trained on the 50 initial trials of each session.</p>
</sec>
<sec id="sec37">
<label>2.4.2.5</label>
<title>EEG data analysis</title>
<p>EEGLAB open-source software was utilized, with a processing stream detailed in an earlier publication (<xref ref-type="bibr" rid="ref17">Hinchberger et al., 2023</xref>). Briefly, steps included attenuation of power line noise using the EEGLAB cleanline function, automatic removal of bad channels using the clean_rawdata function, creation of a merged file with all rest and experimental trials, visual removal of noisy periods, down-sampling to 250&#x2009;Hz, application of Artifact Subspace Reconstruction (ASR) (<xref ref-type="bibr" rid="ref33">Mullen et al., 2013</xref>), re-referencing to a common-average, and 1&#x2009;Hz high-pass filtering. Adaptive Mixture Independent Component Analysis (AMICA) was used to parse mixed scalp signals into maximally temporally independent component (IC) signals. The EEGLAB DIPFIT algorithm was used to generate a best-fit equivalent current dipole for each IC, removing those with residual variance &#x003E;20% or topographical sparseness &#x003E;5. Scalp topographies, power spectra, and dipole locations were evaluated, and those deemed of non-cortical origin were removed. Selected epochs were from 1&#x2009;s before to 1&#x2009;s after movement onset. Rest trials from each assessment were used as the baseline which was subtracted from the movement trials for that assessment. To demonstrate significant differences across assessment sessions, we created significance masked ERSPs using the EEGLAB &#x201C;condstat&#x201D; function, 200 bootstrapped surrogate datasets, and a significance value of less than 0.05. Non-significant values were set to 0 (green) in the difference ERSPs (row 3, <xref ref-type="fig" rid="fig6">Figure 6</xref>).</p>
<fig position="float" id="fig6">
<label>Figure 6</label>
<caption>
<p>Event-related spectral perturbation (ERSP) plots of the motor regions pre and post training (top and middle row, respectively) and their significance masked differences (bottom row). All rows share a common time and frequency axis, with time zero corresponding to dorsiflexion onset of the trained leg (vertical line). The top two rows (pre and post) share a common colorbar (change in power in decibels), with blue representing a decrease in power relative to that session&#x2019;s baseline and red representing an increase in power. For the bottom row, the significance masked ERSPs (green means a voxel was not significant at an alpha value of 0.05) are scaled symmetrically from &#x2212;4 db to 4 db&#x002A;, with the exception of the central motor region, which is scaled from &#x2212;8 to 8 db, to adequately represent the larger relative changes seen in this region.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g006.tif"/>
</fig>
</sec>
</sec>
</sec>
</sec>
<sec sec-type="results" id="sec38">
<label>3</label>
<title>Results</title>
<sec id="sec39">
<label>3.1</label>
<title>Healthy adult cohort</title>
<p>In this study, we assessed the performance of various detection algorithms using EEG datasets collected from healthy participants. The thresholding algorithm was excluded from further analysis due to its frequent misdetections, often stemming from threshold values set during calibration that were too high for real-time training trials, hindering effective triggering of NMES.</p>
<sec id="sec40">
<label>3.1.1</label>
<title>Comparing the deep learning models: MLP vs. LSTM NN</title>
<p>The mean detection accuracy of LSTM across healthy participants (88% &#x00B1;10) was significantly higher than that of MLP-NN (85%&#x2009;&#x00B1;&#x2009;8), <italic>p</italic>&#x2009;&#x003C;&#x2009;0.001. The modest difference in mean accuracy led to the selection of the LSTM model. More importantly, LSTM was selected due to its ability to handle subject heterogeneity in individuals with CP by leveraging its capacity to process long-term and short term interdependencies in a time-series, as its name implies.</p>
</sec>
<sec id="sec41">
<label>3.1.2</label>
<title>Comparing the selected deep learning LSTM model to the average PN algorithm</title>
<p>Although the RMSE value of 220&#x2009;ms for LSTM was not significantly different from that of the Average PN algorithm (214&#x2009;ms, <italic>p</italic>&#x2009;=&#x2009;0.343), LSTM mean error was negative (&#x2212;70&#x2009;ms&#x2009;&#x00B1;&#x2009;150), whereas the Average PN algorithm&#x2019;s mean error was positive (128&#x2009;ms&#x2009;&#x00B1;&#x2009;180) meaning that on average LSTM detected the PN sooner than the average PN. This analysis demonstrated that LSTM has comparable performance to the Average PN algorithm, however, LSTM can be incorporated in a BCI system and it can be implemented with greater flexibility in parameters optimization increasing accuracy in those with neurological injuries who may have greater individual variability in PN occurrence and timing.</p>
</sec>
<sec id="sec42">
<label>3.1.3</label>
<title>Data labeling</title>
<p>The model was significantly more accurate under time-based labeling (88%&#x2009;&#x00B1;&#x2009;10) than PN-based labeling (83%&#x2009;&#x00B1;&#x2009;11), <italic>p</italic>&#x2009;&#x003C;&#x2009;0.001.</p>
</sec>
<sec id="sec43">
<label>3.1.4</label>
<title>Spatial filtering</title>
<p>Despite being significantly different, the accuracy of the LSTM with a Laplacian spatial filter was not appreciably higher than that of an ICA spatial filter (88% &#x00B1;10 vs. 85% &#x00B1;11, <italic>p</italic>&#x2009;&#x003C;&#x2009;0.048). The main reason we opted for the Laplacian filter over ICA was its faster operation, as ICA could take up to 10&#x2009;min depending on its parameters. Moreover, ICA has the potential to alter the polarity of MRCP, which may impact detection reliability. Additionally, the selection of the appropriate component in ICA requires visual inspection which is not needed with Laplacian.</p>
</sec>
</sec>
<sec id="sec44">
<label>3.2</label>
<title>Participant with CP</title>
<p>The system performed well throughout all assessments and training without any major issues. The participant was able to complete all needed calibration trials and perform the desired 100 trials per training session. During the first training session, we noted that our participant was not responding in a timely manner to the cue to move, which would delay the EEG signal needed to activate the NMES beyond the time window we had allotted. Therefore, for several of the subsequent blocks during the first session, we asked him to focus on reducing his reaction time, i.e., to attempt to move as soon as he was cued to do so. He quickly improved on this aspect and retained good timing across all training sessions. It was also noted on reviewing his performance during his first training session that he had a functional range (greater than 10&#x00B0;) of dorsiflexion. Therefore, for the rest of training sessions, we instructed him to focus more on velocity while still reaching his target dorsiflexion angle for each session.</p>
<sec id="sec45">
<label>3.2.1</label>
<title>Motor outcome</title>
<p>Motor outcomes data are summarized in <xref ref-type="table" rid="tab1">Table 1</xref>, including the mean active ankle dorsiflexion angle while seated, peak dorsiflexion angle and joint angular velocity during initial contact and the swing phase of gait across 20 strides in the free and fast walking trials with associated <italic>p</italic>-values, and temporal&#x2013;spatial gait measures. Maximum dorsiflexion did not increase in any condition, and where significant, a small decrease was shown. However, there was a very large increase in dorsiflexion velocity after BCI-NFT training which, while significant, did not transfer to walking. Interestingly though, gait speed, and its components, cadence and step length, all increased after the training, with a significant difference in right step length for the freely-selected speed condition.</p>
<table-wrap position="float" id="tab1">
<label>Table 1</label>
<caption>
<p>Temporal&#x2013;spatial gait measures and ankle dorsiflexion (DF) measurements pre- and post-training for the participant.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th align="left" valign="top">Outcomes</th>
<th align="center" valign="bottom">Right pre</th>
<th align="center" valign="bottom">Left pre</th>
<th align="center" valign="bottom">Right post</th>
<th align="center" valign="bottom">Left post</th>
<th align="center" valign="bottom">Right diff</th>
<th align="center" valign="bottom">Left diff</th>
<th align="center" valign="bottom">Right p val</th>
<th align="center" valign="bottom">Left p val</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top">Gait speed free (m/s)</td>
<td align="center" valign="top">1.19</td>
<td/>
<td align="center" valign="top">1.34</td>
<td/>
<td align="center" valign="top">0.15</td>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="top">Cadence (steps/min)</td>
<td align="center" valign="top">120.1</td>
<td align="center" valign="top">132.9</td>
<td align="center" valign="top">125.6</td>
<td align="center" valign="top">139.3</td>
<td align="center" valign="top">5.54</td>
<td align="center" valign="top">6.40</td>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="top">Step length (m)</td>
<td align="center" valign="top">0.56</td>
<td align="center" valign="top">0.57</td>
<td align="center" valign="top">0.61</td>
<td align="center" valign="top">0.61</td>
<td align="center" valign="top">0.05</td>
<td align="center" valign="top">0.04</td>
<td align="center" valign="top"><bold>0.001</bold></td>
<td align="center" valign="top">0.054</td>
</tr>
<tr>
<td align="left" valign="top">Gait speed fast (m/s)</td>
<td align="center" valign="top">1.70</td>
<td/>
<td align="center" valign="top">1.84</td>
<td/>
<td align="center" valign="top">0.14</td>
<td/>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="top">Cadence fast (steps/min)</td>
<td align="center" valign="top">138.3</td>
<td align="center" valign="top">153.1</td>
<td align="center" valign="top">148.0</td>
<td align="center" valign="top">157.4</td>
<td align="center" valign="top">9.68</td>
<td align="center" valign="top">4.34</td>
<td/>
<td/>
</tr>
<tr>
<td align="left" valign="top">Step length fast (m)</td>
<td align="center" valign="top">0.72</td>
<td align="center" valign="top">0.69</td>
<td align="center" valign="top">0.72</td>
<td align="center" valign="top">0.73</td>
<td align="center" valign="top">0.001</td>
<td align="center" valign="top">0.04</td>
<td align="center" valign="top">0.19</td>
<td align="center" valign="top">0.08</td>
</tr>
<tr>
<td align="left" valign="top">MAX DF angle swing (&#x00B0;)</td>
<td align="center" valign="top">4.58</td>
<td align="center" valign="top">9.90</td>
<td align="center" valign="top">1.43</td>
<td align="center" valign="top">6.90</td>
<td align="center" valign="top">&#x2212;3.15</td>
<td align="center" valign="top">&#x2212;3.00</td>
<td align="center" valign="top"><bold>0.003</bold></td>
<td align="center" valign="top"><bold>0.001</bold></td>
</tr>
<tr>
<td align="left" valign="top">DF angle initial contact (&#x00B0;)</td>
<td align="center" valign="top">&#x2212;2.70</td>
<td align="center" valign="top">0.88</td>
<td align="center" valign="top">&#x2212;2.86</td>
<td align="center" valign="top">&#x2212;1.15</td>
<td align="center" valign="top">&#x2212;0.16</td>
<td align="center" valign="top">&#x2212;2.03</td>
<td align="center" valign="top">0.48</td>
<td align="center" valign="top"><bold>0.032</bold></td>
</tr>
<tr>
<td align="left" valign="top">MAX DF velocity stance (&#x00B0;/s)</td>
<td align="center" valign="top">141.3</td>
<td align="center" valign="top">110.3</td>
<td align="center" valign="top">146.9</td>
<td align="center" valign="top">151.9</td>
<td align="center" valign="top">5.69</td>
<td align="center" valign="top">41.6</td>
<td align="center" valign="top">0.48</td>
<td align="center" valign="top"><bold>0.001</bold></td>
</tr>
<tr>
<td align="left" valign="top">MAX DF velocity swing (&#x00B0;/s)</td>
<td align="center" valign="top">121.6</td>
<td align="center" valign="top">199.4</td>
<td align="center" valign="top">115.0</td>
<td align="center" valign="top">191.3</td>
<td align="center" valign="top">&#x2212;6.67</td>
<td align="center" valign="top">&#x2212;8.03</td>
<td align="center" valign="top">0.62</td>
<td align="center" valign="top">0.36</td>
</tr>
<tr>
<td align="left" valign="top">MAX DF angle in sitting (&#x00B0;)</td>
<td align="center" valign="top">10.8</td>
<td/>
<td align="center" valign="top">8.46</td>
<td/>
<td align="center" valign="top">&#x2212;2.37</td>
<td/>
<td align="center" valign="top"><bold>0.001</bold></td>
<td/>
</tr>
<tr>
<td align="left" valign="top">MAX DF velocity in sitting (&#x00B0;/s)</td>
<td align="center" valign="top">133.4</td>
<td/>
<td align="center" valign="top">234.0</td>
<td/>
<td align="center" valign="top">100.6</td>
<td/>
<td align="center" valign="top"><bold>0.001</bold></td>
<td/>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>The right side was trained, so the greatest differences were anticipated on that side. BOLD&#x2009;=&#x2009;<italic>p</italic>&#x2009;&#x003C;&#x2009;0.05.</p>
</table-wrap-foot>
</table-wrap>
</sec>
<sec id="sec46">
<label>3.2.2</label>
<title>EEG analysis</title>
<p>Event-related spectral perturbation (ERSP) plots pre- and post-training are shown in <xref ref-type="fig" rid="fig6">Figure 6</xref> for the right and left pre-motor/supplementary motor regions and for the central motor region that is localized just to the right of midline. These plots demonstrate that the greatest change in activation was in the central motor region, with beta differences being more prominent. Additionally, mean event-related desynchronization (ERD) values are presented in <xref ref-type="table" rid="tab2">Table 2</xref>. To calculate ERD, EEG was epoched from 1&#x2009;s before to 1&#x2009;s after dorsiflexion movement onset. EEG was recorded during active dorsiflexion trials while seated and included data from motor-related brain regions. These ERDs indicate the amount of power reduction from rest in the alpha and beta bands pre- and post-training.</p>
<table-wrap position="float" id="tab2">
<label>Table 2</label>
<caption>
<p>Mean event-related desynchronization (ERD) values indicating the amount of power reduction from rest in the alpha and beta bands in motor-related brain regions pre-and post-training, recorded during active dorsiflexion trials while seated.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th align="left" valign="top">Outcome</th>
<th align="center" valign="top">Right pre</th>
<th align="center" valign="top">Left pre</th>
<th align="center" valign="top">Right post</th>
<th align="center" valign="top">Left post</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top">Prefrontal alpha</td>
<td align="center" valign="top">1.61</td>
<td/>
<td align="center" valign="top">3.67</td>
<td/>
</tr>
<tr>
<td align="left" valign="top">Prefrontal beta</td>
<td align="center" valign="top">2.60</td>
<td/>
<td align="center" valign="top">3.35</td>
<td/>
</tr>
<tr>
<td align="left" valign="top">Premotor region alpha</td>
<td align="center" valign="top">5.44</td>
<td align="center" valign="top">6.01</td>
<td align="center" valign="top">6.29</td>
<td align="center" valign="top">6.05</td>
</tr>
<tr>
<td align="left" valign="top">Premotor region beta</td>
<td align="center" valign="top">3.56</td>
<td align="center" valign="top">3.68</td>
<td align="center" valign="top">4.15</td>
<td align="center" valign="top">5.11</td>
</tr>
<tr>
<td align="left" valign="top">Central motor alpha</td>
<td align="center" valign="top">4.83</td>
<td/>
<td align="center" valign="top">7.63</td>
<td/>
</tr>
<tr>
<td align="left" valign="top">Central motor beta</td>
<td align="center" valign="top">3.18</td>
<td/>
<td align="center" valign="top">5.45</td>
<td/>
</tr>
</tbody>
</table>
</table-wrap>
</sec>
<sec id="sec47">
<label>3.2.3</label>
<title>Transfer learning</title>
<p>Given that the performance of any deep learning model hinges on an adequate amount of training data, this strategy ensured a more efficient utilization of available data while minimizing calibration time. Consequently, the LSTM model was trained with the trials collected during the previous session (over100 trials) and then transferred to the current session using the first 25 trials. To this end, the weightings of the first two layers of this LSTM model were held constant, or &#x201C;frozen,&#x201D; then a new bilateral LSTM layer with 100 nodes was added to the structure (<xref ref-type="fig" rid="fig7">Figure 7</xref>). Due to the frozen state of the first two layers, only the weights of the new bilateral LSTM and the fully connected layer were updated during transfer learning. To minimize the impact of session-by-session variability in our feature space (preprocessed EEG) we employed Z-score normalization at the input layer, i.e., the input EEG was scaled by subtracting its mean and dividing by its standard deviation. In the initial session, the model underwent training using data from 50 trials. <xref ref-type="fig" rid="fig8">Figure 8</xref> demonstrates the session-by-session accuracy of the transferred LSTM compared with that of the LSTM model if it was not transferred, i.e., if it was trained on only the 50 initial trials of each session. The mean detection accuracy of the transferred model was 80.8% vs. 78.9% without transfer learning (<italic>p</italic>&#x2009;=&#x2009;0.064). Detection of the wrong signal as the PN occurred in about 17% of trials (false positives) with false negatives being rare. The session-by-session accuracy results are in <xref ref-type="table" rid="tab3">Table 3</xref>. Please note that the accuracy results were processed post-hoc using the model that was trained at each session of the data collection.</p>
<fig position="float" id="fig7">
<label>Figure 7</label>
<caption>
<p>Transfer learning. The LSTM model was trained with trials collected during the previous session (100 trials) and then transferred to the current session using the first 25 trials. The weightings of the first two layers of this LSTM model were held constant, or &#x201C;frozen,&#x201D; then a new bilateral LSTM layer was inserted to model. Due to the frozen state of the first two layers only the weights of the new bilateral LSTM and the fully connected layer were updated during the transfer learning.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g007.tif"/>
</fig>
<fig position="float" id="fig8">
<label>Figure 8</label>
<caption>
<p>The session-by-session accuracy of the transferred LSTM, trained on 100 trials from the previous session and transferred to 25 trials of the current session (red circles), compared with the accuracy of the LSTM model if it was not transferred and instead was trained on the first 50 trials of each session (blue squares). Here we only included the sessions that used transferred LSTM as the detection model.</p>
</caption>
<graphic xlink:href="fnhum-18-1346050-g008.tif"/>
</fig>
<table-wrap position="float" id="tab3">
<label>Table 3</label>
<caption>
<p>Detection accuracy showing that on average 80% of trials detected the peak negativity in the proper window and activated the stimulator.</p>
</caption>
<table frame="hsides" rules="groups">
<thead>
<tr>
<th align="left" valign="top">Session</th>
<th align="center" valign="top">False positives (%)</th>
<th align="center" valign="top">False negatives (%)</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top">2</td>
<td align="center" valign="top">19.1</td>
<td align="center" valign="top">0</td>
</tr>
<tr>
<td align="left" valign="top">3</td>
<td align="center" valign="top">10.8</td>
<td align="center" valign="top">1.5</td>
</tr>
<tr>
<td align="left" valign="top">4</td>
<td align="center" valign="top">26.2</td>
<td align="center" valign="top">0</td>
</tr>
<tr>
<td align="left" valign="top">5</td>
<td align="center" valign="top">15.4</td>
<td align="center" valign="top">1.5</td>
</tr>
<tr>
<td align="left" valign="top">6</td>
<td align="center" valign="top">17.5</td>
<td align="center" valign="top">3.2</td>
</tr>
<tr>
<td align="left" valign="top">7</td>
<td align="center" valign="top">10.3</td>
<td align="center" valign="top">1.7</td>
</tr>
<tr>
<td align="left" valign="top">9</td>
<td align="center" valign="top">19.1</td>
<td align="center" valign="top">1.5</td>
</tr>
<tr>
<td align="left" valign="top">10</td>
<td align="center" valign="top">19.1</td>
<td align="center" valign="top">6.4</td>
</tr>
<tr>
<td align="left" valign="top">Mean</td>
<td align="center" valign="top">17.2</td>
<td align="center" valign="top">2.0</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>Only the sessions that used transferred LSTM (sessions 2&#x2013;10) as the detection model were included.</p>
</table-wrap-foot>
</table-wrap>
</sec>
</sec>
</sec>
<sec sec-type="discussion" id="sec48">
<label>4</label>
<title>Discussion</title>
<p>To extend the application of BCI-NFT from the stroke population to a pediatric population with CP, we designed a MA-based BCI-NFT to train ankle dorsiflexion in children with CP. The BCI-NFT system employed LSTM to detect PN in MRCP, an EEG feature generated by an attempt to dorsiflex the ankle in real-time, and thereby activate enriched sensory feedback to the TA muscle by application of NMES, in an associative learning paradigm. The system was finalized using a cohort of eight healthy individuals and evaluated on a child with CP in a 10-session dorsiflexion training protocol. Considering the heterogeneity of cortical activity in the CP population and the location of the ankle on the somatotopic motor map of the primary motor cortex (M1), the system demonstrated good to high reliability in applying NMES feedback, as demonstrated by the reported false positives and false negatives that may have also been affected by some expected fluctuations in attentiveness to the task given the multiple repetitions required.</p>
<sec id="sec49">
<label>4.1</label>
<title>BCI-NFT system accuracy</title>
<p>Similar to other BCI systems, one of our major considerations in designing our system was to achieve as high a detection accuracy of motor intent as possible, such that the task is assisted and sensory feedback enhanced only during focused efforts to perform the task, and not when the participant may not be attending as well to the task. Detection accuracy is especially important in BCI-NFT systems as false positives may lead to mal-adaptive neuroplasticity by potentially enhancing inappropriate associations or failing to promote appropriate ones. Previous MA-based BCI-NFT systems predominantly used machine learning classifiers such as linear LDA (<xref ref-type="bibr" rid="ref37">Ono et al., 2013</xref>; <xref ref-type="bibr" rid="ref26">McCrimmon et al., 2014</xref>; <xref ref-type="bibr" rid="ref32">Mukaino et al., 2014</xref>; <xref ref-type="bibr" rid="ref38">Osuagwu et al., 2016</xref>; <xref ref-type="bibr" rid="ref10">Chen et al., 2020</xref>), SVM (<xref ref-type="bibr" rid="ref12">Chowdhury et al., 2018</xref>; <xref ref-type="bibr" rid="ref6">Bhagat et al., 2020</xref>; <xref ref-type="bibr" rid="ref11">Chowdhury et al., 2020</xref>), Gaussian (<xref ref-type="bibr" rid="ref7">Biasiucci et al., 2018</xref>), and logistic regression (<xref ref-type="bibr" rid="ref19">Ib&#x00E1;&#x00F1;ez et al., 2017</xref>). Our study uniquely employed LSTM (a deep learning algorithm often utilized in BCI applications), for detecting motor intent, demonstrating the potential for enhanced accuracy in MRCP detection. The performance of motor attempt BCI-NFT systems designed for motor rehabilitation in neurological populations has been primarily characterized by detection accuracies ranging from 70 to 80% in the stroke population. A requisite for effective BCI-NFT, as established in MI BCI-NFT systems, is the attainment of a detection accuracy exceeding 70% (<xref ref-type="bibr" rid="ref44">Vidaurre and Blankertz, 2010</xref>). The implementation of a Bayesian classifier in a MI BCI-NFT protocol for children with CP by Bobrov et al. demonstrated a maximum 70% detection accuracy (<xref ref-type="bibr" rid="ref8">Bobrov et al., 2020</xref>). In our study, the BCI-NFT system showcased a robust detection accuracy of over 88% in the healthy cohort. However, our participant with CP exhibited a slightly reduced detection accuracy of 79%, but still surpassing the 70% threshold and similar to studies in other populations with and without neurological conditions.</p>
<p>Among notable studies employing real-time MRCP feature detection, Bhagat et al. reported a 79% detection rate in 10 participants post-stroke, alongside a 23% false positive rate, utilizing 15 electrodes over the sensorimotor cortex (SMC) and an SVM classifier (<xref ref-type="bibr" rid="ref6">Bhagat et al., 2020</xref>). Xu et al. introduced a manifold learning algorithm followed by LDA, demonstrating a detection delay of 200&#x2013;400&#x2009;ms and a detection accuracy exceeding 80% for the classification of motor intent in healthy individuals (<xref ref-type="bibr" rid="ref42">Ren et al., 2014</xref>). Additionally, Niazi et al. introduced an optimal spatial filter, reducing detection timing by 66.6&#x2009;&#x00B1;&#x2009;121&#x2009;ms, accompanied by a detection accuracy of 82.5% (<xref ref-type="bibr" rid="ref34">Niazi et al., 2013</xref>). Lawhern et al. contributed significantly to the utilization of deep learning for detecting MRCP features, evaluating EEGNet, a convolutional neural network (CNN), in detecting motor intent during index finger tapping in 13 healthy individuals, with a within-subject accuracy of approximately 80% (<xref ref-type="bibr" rid="ref25">Lawhern et al., 2018</xref>). It is important to note that all of these studies reporting BCI performance focused on upper limb movement attempts, which involve a larger more superficial area of the sensorimotor cortex, which is more easily accessible with EEG.</p>
</sec>
<sec id="sec50">
<label>4.2</label>
<title>Case discussion in a participant with CP</title>
<p>Our neurofeedback system worked exactly as intended for all assessments and training with our first participant with CP. He was initially not as attentive to the cued movement timing as desired, but he responded well once prompted to react in a more timely manner. The ability to train dorsiflexion range of motion and/or velocity is an advantage of our design that was helpful for this particular child who needed to practice keeping his foot more dorsiflexed at key points in the gait cycle but who could also potentially also benefit from improvements in dorsiflexion velocity. Changes were predominanetly task specific, i.e., they were the greatest for dorsiflexion velocity, which was the primary focus of his training sessions, but only during the training task. Improvements in temporal&#x2013;spatial gait parameters, while encouraging, could not be explained by positive changes in ankle dorsiflexion range or velocity during gait, which were not found.</p>
<p>The differences in detection accuracy between healthy adults and a child with CP are not surprising. Maintaining attention and motivation was more challenging in this younger child than it was in adults, and we believe that inattention may account at least in part for his lower detection accuracy. For future participants, we do not plan to provide stimulation if the PN is not detected in the detection time window, so that they are not reinforced in a similar manner when they are not as engaged in the task, because inappropriate stimulation may have a negative effect on neuroplasticity by associating inappropriate behavior with enhanced sensory feedback.</p>
<p>The EEG results showed minimal changes post-training in brain regions involved in motor planning, with the exception of small changes mainly in the beta band in the untrained hemisphere, which may indicate a need to recruit ipsilateral pathways to improve performance. The most dramatic improvement was in the central motor region in the beta band. While this IC was localized slightly to the right of midline, the position of the ankle area within the central sulcus makes determining hemispheric location difficult at best, although it is also possible that this could reflect reorganization due to his left brain injury.</p>
<p>While far more data are needed regarding the immediate and long-term effects of this training in CP before any generalized conclusions can be made, the functional changes reported here after only 10 sessions are encouraging. Surprisingly, fatigue did not appear to be a limiting factor during the training. This participant&#x2019;s only complaint was of boredom due to the number of repetitions requested. Future plans are to gamify the training so that it is more motivating and enjoyable. Data collection on additional participants is currently underway.</p>
</sec>
<sec id="sec51">
<label>4.3</label>
<title>Future plans</title>
<p>In our BCI-NFT system, we successfully employed LSTM as a classifier for motor intent. In the future, we aim to utilize LSTM&#x2019;s predictive capabilities, specifically focusing on predicting future states within EEG time series to forecast PN occurrences, thereby enhancing the system&#x2019;s adaptability to the heterogeneity of cortical activity in children with CP. Additionally, our strategy involves the integration of reinforcement learning techniques to enhance the performance of our BCI in accurately predicting or detecting motor intent states. Our first participant with CP exhibited fairly good motor control in that he could dorsiflex repeatedly on command. Children with CP can have a wide range of impairments, and future participants may have greater difficulty reliably performing the task, may not be able to move the ankle at all, or may be prone to fatigue with repeated attempts. These situations would likely present additional challenges to our current system and its performance and effectiveness that we have not yet addressed here. We also plan to collaborate with groups who have commercialized these systems so that they can be utilized in clinics and even in patients&#x2019; homes, assuming that BCI-NFT demonstrates efficacy and efficiency in improving motor function in CP.</p>
</sec>
</sec>
<sec sec-type="conclusions" id="sec52">
<label>5</label>
<title>Conclusion</title>
<p>BCI-NFT systems utilizing motor attempt and newer algorithms that can reduce calibration time while maintaining accuracy provide a feasible method for motor training in children with CP with the possibility of accelerating improvements in motor performance as well as neuroplasticity with stronger evidence of effectiveness needed before implementation in clinical settings.</p>
</sec>
<sec sec-type="data-availability" id="sec53">
<title>Data availability statement</title>
<p>The raw data supporting the conclusions of this article will be made available by the authors, without undue reservation.</p>
</sec>
<sec sec-type="ethics-statement" id="sec54">
<title>Ethics statement</title>
<p>The studies involving humans were approved by the National Institutes of Health Institutional Review Board. The studies were conducted in accordance with the local legislation and institutional requirements. Written informed consent for participation in this study was provided by the participants&#x2019; legal guardians/next of kin. Written informed consent was obtained from the individual(s), and minor (s)&#x2019; legal guardian/next of kin, for the publication of any potentially identifiable images or data included in this article.</p>
</sec>
<sec sec-type="author-contributions" id="sec55">
<title>Author contributions</title>
<p>AB: Conceptualization, Data curation, Formal analysis, Investigation, Methodology, Project administration, Software, Writing &#x2013; original draft. JK: Data curation, Investigation, Methodology, Writing &#x2013; review &#x0026; editing. AG: Conceptualization, Methodology, Software, Writing &#x2013; review &#x0026; editing. CP: Investigation, Methodology, Software, Writing &#x2013; review &#x0026; editing. SP: Investigation, Project administration, Writing &#x2013; review &#x0026; editing. DD: Conceptualization, Formal analysis, Funding acquisition, Investigation, Methodology, Project administration, Resources, Supervision, Writing &#x2013; original draft, Writing &#x2013; review &#x0026; editing.</p>
</sec>
</body>
<back>
<sec sec-type="funding-information" id="sec56">
<title>Funding</title>
<p>The author(s) declare financial support was received for the research, authorship, and/or publication of this article. This work was funded by the Intramural Research Program of the NIH Clinical Center (Protocol #13-CC-0110).</p>
</sec>
<sec sec-type="COI-statement" id="sec57">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec id="sec100" sec-type="disclaimer">
<title>Publisher&#x2019;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<ref-list>
<title>References</title>
<ref id="ref1">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Aminizadeh</surname> <given-names>S.</given-names></name> <name><surname>Heidari</surname> <given-names>A.</given-names></name> <name><surname>Dehghan</surname> <given-names>M.</given-names></name> <name><surname>Toumaj</surname> <given-names>S.</given-names></name> <name><surname>Rezaei</surname> <given-names>M.</given-names></name> <name><surname>Jafari Navimipour</surname> <given-names>N.</given-names></name> <etal/></person-group>. (<year>2024</year>). <article-title>Opportunities and challenges of artificial intelligence and distributed systems to improve the quality of healthcare service</article-title>. <source>Artif. Intell. Med.</source> <volume>149</volume>:<fpage>102779</fpage>. doi: <pub-id pub-id-type="doi">10.1016/j.artmed.2024.102779</pub-id>, PMID: <pub-id pub-id-type="pmid">38462281</pub-id></citation>
</ref>
<ref id="ref2">
<citation citation-type="confproc"><person-group person-group-type="author"><name><surname>Ang</surname> <given-names>K. K.</given-names></name> <name><surname>Guan</surname> <given-names>C.</given-names></name> <name><surname>Chua</surname> <given-names>K. S. G.</given-names></name> <name><surname>Ang</surname> <given-names>B. T.</given-names></name> <name><surname>Kuah</surname> <given-names>C.</given-names></name> <name><surname>Wang</surname> <given-names>C.</given-names></name> <etal/></person-group>. <article-title>Clinical study of neurorehabilitation in stroke using EEG-based motor imagery brain-computer interface with robotic feedback</article-title>. In: <conf-name>2010 Annual International Conference of the IEEE Engineering in Medicine and Biology</conf-name>. <publisher-loc>Buenos Aires</publisher-loc>: <publisher-name>IEEE</publisher-name> (<year>2010</year>), <fpage>5549</fpage>&#x2013;<lpage>5552</lpage>. <comment>Available at:</comment> <ext-link xlink:href="http://ieeexplore.ieee.org/document/5626782/" ext-link-type="uri">http://ieeexplore.ieee.org/document/5626782/</ext-link></citation>
</ref>
<ref id="ref3">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bai</surname> <given-names>Z.</given-names></name> <name><surname>Fong</surname> <given-names>K. N. K.</given-names></name> <name><surname>Zhang</surname> <given-names>J. J.</given-names></name> <name><surname>Chan</surname> <given-names>J.</given-names></name> <name><surname>Ting</surname> <given-names>K. H.</given-names></name></person-group> (<year>2020</year>). <article-title>Immediate and long-term effects of BCI-based rehabilitation of the upper extremity after stroke: a systematic review and meta-analysis</article-title>. <source>J. Neuro Eng. Rehabil.</source> <volume>17</volume>:<fpage>57</fpage>. doi: <pub-id pub-id-type="doi">10.1186/s12984-020-00686-2</pub-id>, PMID: <pub-id pub-id-type="pmid">32334608</pub-id></citation>
</ref>
<ref id="ref4">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Behboodi</surname> <given-names>A.</given-names></name> <name><surname>Lee</surname> <given-names>W. A.</given-names></name> <name><surname>Bulea</surname> <given-names>T. C.</given-names></name> <name><surname>Damiano</surname> <given-names>D. L.</given-names></name></person-group> (<year>2022</year>). &#x201C;<article-title>Evaluation of multi-layer perceptron neural networks in predicting ankle dorsiflexion in healthy adults using movement-related cortical potentials for BCI-neurofeedback applications</article-title>&#x201D; in <source>2022 International Conference on Rehabilitation Robotics (ICORR)</source> (<publisher-loc>Rotterdam</publisher-loc>: <publisher-name>IEEE</publisher-name>), <fpage>1</fpage>&#x2013;<lpage>5</lpage>. <comment>Available at:</comment> <ext-link xlink:href="https://ieeexplore.ieee.org/document/9896584/" ext-link-type="uri">https://ieeexplore.ieee.org/document/9896584/</ext-link></citation>
</ref>
<ref id="ref5">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Behboodi</surname> <given-names>A.</given-names></name> <name><surname>Lee</surname> <given-names>W. A.</given-names></name> <name><surname>Hinchberger</surname> <given-names>V. S.</given-names></name> <name><surname>Damiano</surname> <given-names>D. L.</given-names></name></person-group> (<year>2022</year>). <article-title>Determining optimal mobile neurofeedback methods for motor neurorehabilitation in children and adults with non-progressive neurological disorders: a scoping review</article-title>. <source>J Neuro Eng. Rehabil.</source> <volume>19</volume>:<fpage>104</fpage>. doi: <pub-id pub-id-type="doi">10.1186/s12984-022-01081-9</pub-id>, PMID: <pub-id pub-id-type="pmid">36171602</pub-id></citation>
</ref>
<ref id="ref6">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bhagat</surname> <given-names>N. A.</given-names></name> <name><surname>Yozbatiran</surname> <given-names>N.</given-names></name> <name><surname>Sullivan</surname> <given-names>J. L.</given-names></name> <name><surname>Paranjape</surname> <given-names>R.</given-names></name> <name><surname>Losey</surname> <given-names>C.</given-names></name> <name><surname>Hernandez</surname> <given-names>Z.</given-names></name> <etal/></person-group>. (<year>2020</year>). <article-title>Neural activity modulations and motor recovery following brain-exoskeleton interface mediated stroke rehabilitation</article-title>. <source>Neuroimage</source> <volume>28</volume>:<fpage>102502</fpage>. doi: <pub-id pub-id-type="doi">10.1016/j.nicl.2020.102502</pub-id></citation>
</ref>
<ref id="ref7">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Biasiucci</surname> <given-names>A.</given-names></name> <name><surname>Leeb</surname> <given-names>R.</given-names></name> <name><surname>Iturrate</surname> <given-names>I.</given-names></name> <name><surname>Perdikis</surname> <given-names>S.</given-names></name> <name><surname>Al-Khodairy</surname> <given-names>A.</given-names></name> <name><surname>Corbet</surname> <given-names>T.</given-names></name> <etal/></person-group>. (<year>2018</year>). <article-title>Brain-actuated functional electrical stimulation elicits lasting arm motor recovery after stroke</article-title>. <source>Nat. Commun.</source> <volume>9</volume>:<fpage>2421</fpage>. doi: <pub-id pub-id-type="doi">10.1038/s41467-018-04673-z</pub-id>, PMID: <pub-id pub-id-type="pmid">29925890</pub-id></citation>
</ref>
<ref id="ref8">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bobrov</surname> <given-names>P.</given-names></name> <name><surname>Biryukova</surname> <given-names>E.</given-names></name> <name><surname>Polyaev</surname> <given-names>B.</given-names></name> <name><surname>Lajsheva</surname> <given-names>O.</given-names></name> <name><surname>Usachjova</surname> <given-names>E.</given-names></name> <name><surname>Sokolova</surname> <given-names>A.</given-names></name> <etal/></person-group>. (<year>2020</year>). <article-title>Rehabilitation of patients with cerebral palsy using hand exoskeleton controlled by brain-computer interface</article-title>. <source>BRSMU</source> <volume>2020(4)</volume>, <fpage>33</fpage>&#x2013;<lpage>40</lpage>. doi: <pub-id pub-id-type="doi">10.24075/brsmu.2020.047</pub-id></citation>
</ref>
<ref id="ref9">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Cervera</surname> <given-names>M. A.</given-names></name> <name><surname>Soekadar</surname> <given-names>S. R.</given-names></name> <name><surname>Ushiba</surname> <given-names>J.</given-names></name> <name><surname>Mill&#x00E1;n</surname> <given-names>J. D. R.</given-names></name> <name><surname>Liu</surname> <given-names>M.</given-names></name> <name><surname>Birbaumer</surname> <given-names>N.</given-names></name> <etal/></person-group>. (<year>2018</year>). <article-title>Brain-computer interfaces for post-stroke motor rehabilitation: a meta-analysis</article-title>. <source>Ann. Clin. Transl. Neurol.</source> <volume>5</volume>, <fpage>651</fpage>&#x2013;<lpage>663</lpage>. doi: <pub-id pub-id-type="doi">10.1002/acn3.544</pub-id>, PMID: <pub-id pub-id-type="pmid">29761128</pub-id></citation>
</ref>
<ref id="ref10">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chen</surname> <given-names>S.</given-names></name> <name><surname>Cao</surname> <given-names>L.</given-names></name> <name><surname>Shu</surname> <given-names>X.</given-names></name> <name><surname>Wang</surname> <given-names>H.</given-names></name> <name><surname>Ding</surname> <given-names>L.</given-names></name> <name><surname>Wang</surname> <given-names>S. H.</given-names></name> <etal/></person-group>. (<year>2020</year>). <article-title>Longitudinal electroencephalography analysis in subacute stroke patients during intervention of brain&#x2013;computer Interface with exoskeleton feedback</article-title>. <source>Front. Neurosci.</source> <volume>14</volume>:<fpage>809</fpage>. doi: <pub-id pub-id-type="doi">10.3389/fnins.2020.00809</pub-id>, PMID: <pub-id pub-id-type="pmid">32922254</pub-id></citation>
</ref>
<ref id="ref11">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chowdhury</surname> <given-names>A.</given-names></name> <name><surname>Dutta</surname> <given-names>A.</given-names></name> <name><surname>Prasad</surname> <given-names>G.</given-names></name></person-group> (<year>2020</year>). <article-title>Corticomuscular co-activation based hybrid brain-computer interface for motor recovery monitoring</article-title>. <source>IEEE Access.</source> <volume>8</volume>, <fpage>174542</fpage>&#x2013;<lpage>174557</lpage>. doi: <pub-id pub-id-type="doi">10.1109/ACCESS.2020.3026037</pub-id></citation>
</ref>
<ref id="ref12">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chowdhury</surname> <given-names>A.</given-names></name> <name><surname>Meena</surname> <given-names>Y. K.</given-names></name> <name><surname>Raza</surname> <given-names>H.</given-names></name> <name><surname>Bhushan</surname> <given-names>B.</given-names></name> <name><surname>Uttam</surname> <given-names>A. K.</given-names></name> <name><surname>Pandey</surname> <given-names>N.</given-names></name> <etal/></person-group>. (<year>2018</year>). <article-title>Active physical practice followed by mental practice using BCI-driven hand exoskeleton: a pilot trial for clinical effectiveness and usability</article-title>. <source>IEEE J. Biomed. Health Inform.</source> <volume>22</volume>, <fpage>1786</fpage>&#x2013;<lpage>1795</lpage>. doi: <pub-id pub-id-type="doi">10.1109/JBHI.2018.2863212</pub-id>, PMID: <pub-id pub-id-type="pmid">30080152</pub-id></citation>
</ref>
<ref id="ref13">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Craik</surname> <given-names>A.</given-names></name> <name><surname>He</surname> <given-names>Y.</given-names></name> <name><surname>Contreras-Vidal</surname> <given-names>J. L.</given-names></name></person-group> (<year>2019</year>). <article-title>Deep learning for electroencephalogram (EEG) classification tasks: a review</article-title>. <source>J. Neural Eng.</source> <volume>16</volume>:<fpage>031001</fpage>. doi: <pub-id pub-id-type="doi">10.1088/1741-2552/ab0ab5</pub-id>, PMID: <pub-id pub-id-type="pmid">30808014</pub-id></citation>
</ref>
<ref id="ref14">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Delorme</surname> <given-names>A.</given-names></name> <name><surname>Makeig</surname> <given-names>S.</given-names></name></person-group> (<year>2004</year>). <article-title>EEGLAB: an open source toolbox for analysis of single-trial EEG dynamics including independent component analysis</article-title>. <source>J. Neurosci. Methods</source> <volume>134</volume>, <fpage>9</fpage>&#x2013;<lpage>21</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.jneumeth.2003.10.009</pub-id>, PMID: <pub-id pub-id-type="pmid">15102499</pub-id></citation>
</ref>
<ref id="ref15">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Djamal</surname> <given-names>E. C.</given-names></name> <name><surname>Ramadhan</surname> <given-names>R. I.</given-names></name> <name><surname>Mandasari</surname> <given-names>M. I.</given-names></name> <name><surname>Djajasasmita</surname> <given-names>D.</given-names></name></person-group> (<year>2020</year>). <article-title>Identification of post-stroke EEG signal using wavelet and convolutional neural networks</article-title>. <source>Bulletin EEI.</source> <volume>9</volume>, <fpage>1890</fpage>&#x2013;<lpage>1898</lpage>. doi: <pub-id pub-id-type="doi">10.11591/eei.v9i5.2005</pub-id></citation>
</ref>
<ref id="ref16">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Frolov</surname> <given-names>A. A.</given-names></name> <name><surname>Mokienko</surname> <given-names>O.</given-names></name> <name><surname>Lyukmanov</surname> <given-names>R.</given-names></name> <name><surname>Biryukova</surname> <given-names>E.</given-names></name> <name><surname>Kotov</surname> <given-names>S.</given-names></name> <name><surname>Turbina</surname> <given-names>L.</given-names></name> <etal/></person-group>. (<year>2017</year>). <article-title>Post-stroke rehabilitation training with a motor-imagery-based brain-computer Interface (BCI)-controlled hand exoskeleton: a randomized controlled multicenter trial</article-title>. <source>Front. Neurosci.</source> <volume>11</volume>:<fpage>400</fpage>. doi: <pub-id pub-id-type="doi">10.3389/fnins.2017.00400</pub-id>, PMID: <pub-id pub-id-type="pmid">28775677</pub-id></citation>
</ref>
<ref id="ref17">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hinchberger</surname> <given-names>V.</given-names></name> <name><surname>Kang</surname> <given-names>S. H.</given-names></name> <name><surname>Kline</surname> <given-names>J.</given-names></name> <name><surname>Stanley</surname> <given-names>C. J.</given-names></name> <name><surname>Bulea</surname> <given-names>T. C.</given-names></name> <name><surname>Damiano</surname> <given-names>D. L.</given-names></name></person-group> (<year>2023</year>). <article-title>Investigation of brain mechanisms underlying upper limb function in bilateral cerebral palsy using EEG</article-title>. <source>Clin. Neurophysiol.</source> <volume>151</volume>, <fpage>116</fpage>&#x2013;<lpage>127</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.clinph.2023.04.006</pub-id>, PMID: <pub-id pub-id-type="pmid">37245498</pub-id></citation>
</ref>
<ref id="ref18">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hu</surname> <given-names>X. L.</given-names></name> <name><surname>Tong</surname> <given-names>R. K.</given-names></name> <name><surname>Ho</surname> <given-names>N. S. K.</given-names></name> <name><surname>Xue</surname> <given-names>J. J.</given-names></name> <name><surname>Rong</surname> <given-names>W.</given-names></name> <name><surname>Li</surname> <given-names>L. S. W.</given-names></name></person-group> (<year>2015</year>). <article-title>Wrist rehabilitation assisted by an electromyography-driven neuromuscular electrical stimulation robot after stroke</article-title>. <source>Neurorehabil. Neural Repair</source> <volume>29</volume>, <fpage>767</fpage>&#x2013;<lpage>776</lpage>. doi: <pub-id pub-id-type="doi">10.1177/1545968314565510</pub-id>, PMID: <pub-id pub-id-type="pmid">25549656</pub-id></citation>
</ref>
<ref id="ref19">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Ib&#x00E1;&#x00F1;ez</surname> <given-names>J.</given-names></name> <name><surname>Monge-Pereira</surname> <given-names>E.</given-names></name> <name><surname>Molina-Rueda</surname> <given-names>F.</given-names></name> <name><surname>Serrano</surname> <given-names>J. I.</given-names></name> <name><surname>Del Castillo</surname> <given-names>M. D.</given-names></name> <name><surname>Cuesta-G&#x00F3;mez</surname> <given-names>A.</given-names></name> <etal/></person-group>. (<year>2017</year>). <article-title>Low latency estimation of motor intentions to assist reaching movements along multiple sessions in chronic stroke patients: a feasibility study</article-title>. <source>Front. Neurosci.</source> <volume>11</volume>:<fpage>126</fpage>. doi: <pub-id pub-id-type="doi">10.3389/fnins.2017.00126</pub-id></citation>
</ref>
<ref id="ref20">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Irimia</surname> <given-names>D. C.</given-names></name> <name><surname>Ortner</surname> <given-names>R.</given-names></name> <name><surname>Poboroniuc</surname> <given-names>M. S.</given-names></name> <name><surname>Ignat</surname> <given-names>B. E.</given-names></name> <name><surname>Guger</surname> <given-names>C.</given-names></name></person-group> (<year>2018</year>). <article-title>High classification accuracy of a motor imagery based brain-computer Interface for stroke rehabilitation training</article-title>. <source>Front Robot AI.</source> <volume>5</volume>:<fpage>130</fpage>. doi: <pub-id pub-id-type="doi">10.3389/frobt.2018.00130</pub-id>, PMID: <pub-id pub-id-type="pmid">33501008</pub-id></citation>
</ref>
<ref id="ref21">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kim</surname> <given-names>T. W.</given-names></name> <name><surname>Lee</surname> <given-names>B. H.</given-names></name></person-group> (<year>2016</year>). <article-title>Clinical usefulness of brain-computer interface-controlled functional electrical stimulation for improving brain activity in children with spastic cerebral palsy: a pilot randomized controlled trial</article-title>. <source>J. Phys. Ther. Sci.</source> <volume>28</volume>, <fpage>2491</fpage>&#x2013;<lpage>2494</lpage>. doi: <pub-id pub-id-type="doi">10.1589/jpts.28.2491</pub-id>, PMID: <pub-id pub-id-type="pmid">27799677</pub-id></citation>
</ref>
<ref id="ref22">
<citation citation-type="other"><person-group person-group-type="author"><name><surname>Kothe</surname> <given-names>C.</given-names></name> <name><surname>Medine</surname> <given-names>D.</given-names></name> <name><surname>Boulay</surname> <given-names>C.</given-names></name> <name><surname>Grivich</surname> <given-names>M.</given-names></name> <name><surname>Stenner</surname> <given-names>T.</given-names></name></person-group>. (<year>2024</year>) <source>Lab streaming layer (LSL)</source>. <comment>Available at:</comment> <ext-link xlink:href="https://github.com/sccn/labstreaminglayer" ext-link-type="uri">https://github.com/sccn/labstreaminglayer</ext-link></citation>
</ref>
<ref id="ref23">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kotov</surname> <given-names>S. V.</given-names></name> <name><surname>Turbina</surname> <given-names>L. G.</given-names></name> <name><surname>Bobrov</surname> <given-names>P. D.</given-names></name> <name><surname>Frolov</surname> <given-names>A. A.</given-names></name> <name><surname>Pavlova</surname> <given-names>O. G.</given-names></name> <name><surname>Kurganskaia</surname> <given-names>M. E.</given-names></name> <etal/></person-group>. (<year>2014</year>). <article-title>Rehabilitation of post stroke patients using a bioengineering system &#x201C;brain-computer interface + exoskeleton.&#x201D;</article-title>. <source>Z Nevrol Psikhiatr Im SS Korsakova</source> <volume>114</volume>:<fpage>66</fpage>. doi: <pub-id pub-id-type="doi">10.17116/jnevro201411412266-71</pub-id></citation>
</ref>
<ref id="ref24">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kruse</surname> <given-names>A.</given-names></name> <name><surname>Suica</surname> <given-names>Z.</given-names></name> <name><surname>Taeymans</surname> <given-names>J.</given-names></name> <name><surname>Schuster-Amft</surname> <given-names>C.</given-names></name></person-group> (<year>2020</year>). <article-title>Effect of brain-computer interface training based on non-invasive electroencephalography using motor imagery on functional recovery after stroke - a systematic review and meta-analysis</article-title>. <source>BMC Neurol.</source> <volume>20</volume>:<fpage>385</fpage>. doi: <pub-id pub-id-type="doi">10.1186/s12883-020-01960-5</pub-id>, PMID: <pub-id pub-id-type="pmid">33092554</pub-id></citation>
</ref>
<ref id="ref25">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lawhern</surname> <given-names>V. J.</given-names></name> <name><surname>Solon</surname> <given-names>A. J.</given-names></name> <name><surname>Waytowich</surname> <given-names>N. R.</given-names></name> <name><surname>Gordon</surname> <given-names>S. M.</given-names></name> <name><surname>Hung</surname> <given-names>C. P.</given-names></name> <name><surname>Lance</surname> <given-names>B. J.</given-names></name></person-group> (<year>2018</year>). <article-title>EEGNet: a compact convolutional neural network for EEG-based brain&#x2013;computer interfaces</article-title>. <source>J. Neural Eng.</source> <volume>15</volume>:<fpage>056013</fpage>. doi: <pub-id pub-id-type="doi">10.1088/1741-2552/aace8c</pub-id>, PMID: <pub-id pub-id-type="pmid">29932424</pub-id></citation>
</ref>
<ref id="ref26">
<citation citation-type="confproc"><person-group person-group-type="author"><name><surname>McCrimmon</surname> <given-names>C. M.</given-names></name> <name><surname>King</surname> <given-names>C. E.</given-names></name> <name><surname>Wang</surname> <given-names>P. T.</given-names></name> <name><surname>Cramer</surname> <given-names>S. C.</given-names></name> <name><surname>Nenadic</surname> <given-names>Z.</given-names></name> <name><surname>Do</surname> <given-names>A. H.</given-names></name></person-group> <article-title>Brain-controlled functional electrical stimulation for lower-limb motor recovery in stroke survivors</article-title>. In: <conf-name>2014 36th Annual International Conference of the IEEE Engineering in Medicine and Biology Society</conf-name>. <publisher-loc>Chicago, IL</publisher-loc>: <publisher-name>IEEE</publisher-name> (<year>2014</year>), <fpage>1247</fpage>&#x2013;<lpage>1250</lpage>. <comment>Available at:</comment> <ext-link xlink:href="http://ieeexplore.ieee.org/document/6943823/" ext-link-type="uri">http://ieeexplore.ieee.org/document/6943823/</ext-link></citation>
</ref>
<ref id="ref27">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Monge-Pereira</surname> <given-names>E.</given-names></name> <name><surname>Iba&#x00F1;ez-Pereda</surname> <given-names>J.</given-names></name> <name><surname>Alguacil-Diego</surname> <given-names>I. M.</given-names></name> <name><surname>Serrano</surname> <given-names>J. I.</given-names></name> <name><surname>Spottorno-Rubio</surname> <given-names>M. P.</given-names></name> <name><surname>Molina-Rueda</surname> <given-names>F.</given-names></name></person-group> (<year>2017</year>). <article-title>Use of electroencephalography brain-computer Interface systems as a rehabilitative approach for upper limb function after a stroke: a systematic review</article-title>. <source>PM R.</source> <volume>9</volume>, <fpage>918</fpage>&#x2013;<lpage>932</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.pmrj.2017.04.016</pub-id>, PMID: <pub-id pub-id-type="pmid">28512066</pub-id></citation>
</ref>
<ref id="ref28">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mooney</surname> <given-names>J. A.</given-names></name> <name><surname>Rose</surname> <given-names>J.</given-names></name></person-group> (<year>2019</year>). <article-title>A scoping review of neuromuscular electrical stimulation to improve gait in cerebral palsy: the arc of Progress and future strategies</article-title>. <source>Front. Neurol.</source> <volume>10</volume>:<fpage>887</fpage>. doi: <pub-id pub-id-type="doi">10.3389/fneur.2019.00887</pub-id>, PMID: <pub-id pub-id-type="pmid">31496986</pub-id></citation>
</ref>
<ref id="ref29">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mrachacz-Kersting</surname> <given-names>N.</given-names></name> <name><surname>Ib&#x00E1;&#x00F1;ez</surname> <given-names>J.</given-names></name> <name><surname>Farina</surname> <given-names>D.</given-names></name></person-group> (<year>2021</year>). <article-title>Towards a mechanistic approach for the development of non-invasive brain-computer interfaces for motor rehabilitation</article-title>. <source>J. Physiol.</source> <volume>599</volume>, <fpage>2361</fpage>&#x2013;<lpage>2374</lpage>. doi: <pub-id pub-id-type="doi">10.1113/JP281314</pub-id></citation>
</ref>
<ref id="ref30">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mrachacz-Kersting</surname> <given-names>N.</given-names></name> <name><surname>Jiang</surname> <given-names>N.</given-names></name> <name><surname>Stevenson</surname> <given-names>A. J. T.</given-names></name> <name><surname>Niazi</surname> <given-names>I. K.</given-names></name> <name><surname>Kostic</surname> <given-names>V.</given-names></name> <name><surname>Pavlovic</surname> <given-names>A.</given-names></name> <etal/></person-group>. (<year>2016</year>). <article-title>Efficient neuroplasticity induction in chronic stroke patients by an associative brain-computer interface</article-title>. <source>J. Neurophysiol.</source> <volume>115</volume>, <fpage>1410</fpage>&#x2013;<lpage>1421</lpage>. doi: <pub-id pub-id-type="doi">10.1152/jn.00918.2015</pub-id>, PMID: <pub-id pub-id-type="pmid">26719088</pub-id></citation>
</ref>
<ref id="ref31">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mrachacz-Kersting</surname> <given-names>N.</given-names></name> <name><surname>Stevenson</surname> <given-names>A. J. T.</given-names></name> <name><surname>J&#x00F8;rgensen</surname> <given-names>H. R. M.</given-names></name> <name><surname>Severinsen</surname> <given-names>K. E.</given-names></name> <name><surname>Aliakbaryhosseinabadi</surname> <given-names>S.</given-names></name> <name><surname>Jiang</surname> <given-names>N.</given-names></name> <etal/></person-group>. (<year>2019</year>). <article-title>Brain state&#x2013;dependent stimulation boosts functional recovery following stroke</article-title>. <source>Ann. Neurol.</source> <volume>85</volume>, <fpage>84</fpage>&#x2013;<lpage>95</lpage>. doi: <pub-id pub-id-type="doi">10.1002/ana.25375</pub-id>, PMID: <pub-id pub-id-type="pmid">30408227</pub-id></citation>
</ref>
<ref id="ref32">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Mukaino</surname> <given-names>M.</given-names></name> <name><surname>Ono</surname> <given-names>T.</given-names></name> <name><surname>Shindo</surname> <given-names>K.</given-names></name> <name><surname>Fujiwara</surname> <given-names>T.</given-names></name> <name><surname>Ota</surname> <given-names>T.</given-names></name> <name><surname>Kimura</surname> <given-names>A.</given-names></name> <etal/></person-group>. (<year>2014</year>). <article-title>Efficacy of brain-computer interface-driven neuromuscular electrical stimulation for chronic paresis after stroke</article-title>. <source>J. Rehabil. Med.</source> <volume>46</volume>, <fpage>378</fpage>&#x2013;<lpage>382</lpage>. doi: <pub-id pub-id-type="doi">10.2340/16501977-1785</pub-id>, PMID: <pub-id pub-id-type="pmid">24590225</pub-id></citation>
</ref>
<ref id="ref33">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Mullen</surname> <given-names>T.</given-names></name> <name><surname>Kothe</surname> <given-names>C.</given-names></name> <name><surname>Chi</surname> <given-names>Y. M.</given-names></name> <name><surname>Ojeda</surname> <given-names>A.</given-names></name> <name><surname>Kerth</surname> <given-names>T.</given-names></name> <name><surname>Makeig</surname> <given-names>S.</given-names></name> <etal/></person-group>. (<year>2013</year>). &#x201C;<article-title>Real-time modeling and 3D visualization of source dynamics and connectivity using wearable EEG</article-title>&#x201D; in <source>2013 35th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)</source> (<publisher-loc>Osaka</publisher-loc>: <publisher-name>IEEE</publisher-name>), <fpage>2184</fpage>&#x2013;<lpage>2187</lpage>. <comment>Available at:</comment> <ext-link xlink:href="http://ieeexplore.ieee.org/document/6609968/" ext-link-type="uri">http://ieeexplore.ieee.org/document/6609968/</ext-link></citation>
</ref>
<ref id="ref34">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Niazi</surname> <given-names>I. K.</given-names></name> <name><surname>Jiang</surname> <given-names>N.</given-names></name> <name><surname>Jochumsen</surname> <given-names>M.</given-names></name> <name><surname>Nielsen</surname> <given-names>J. F.</given-names></name> <name><surname>Dremstrup</surname> <given-names>K.</given-names></name> <name><surname>Farina</surname> <given-names>D.</given-names></name></person-group> (<year>2013</year>). <article-title>Detection of movement-related cortical potentials based on subject-independent training</article-title>. <source>Med. Biol. Eng. Comput.</source> <volume>51</volume>, <fpage>507</fpage>&#x2013;<lpage>512</lpage>. doi: <pub-id pub-id-type="doi">10.1007/s11517-012-1018-1</pub-id>, PMID: <pub-id pub-id-type="pmid">23283643</pub-id></citation>
</ref>
<ref id="ref35">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Niazi</surname> <given-names>I. K.</given-names></name> <name><surname>Mrachacz-Kersting</surname> <given-names>N.</given-names></name> <name><surname>Jiang</surname> <given-names>N.</given-names></name> <name><surname>Dremstrup</surname> <given-names>K.</given-names></name> <name><surname>Farina</surname> <given-names>D.</given-names></name></person-group> (<year>2012</year>). <article-title>Peripheral electrical stimulation triggered by self-paced detection of motor intention enhances motor evoked potentials</article-title>. <source>IEEE Trans. Neural Syst. Rehabil. Eng.</source> <volume>20</volume>, <fpage>595</fpage>&#x2013;<lpage>604</lpage>. doi: <pub-id pub-id-type="doi">10.1109/TNSRE.2012.2194309</pub-id>, PMID: <pub-id pub-id-type="pmid">22547461</pub-id></citation>
</ref>
<ref id="ref36">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Olsen</surname> <given-names>S.</given-names></name> <name><surname>Alder</surname> <given-names>G.</given-names></name> <name><surname>Williams</surname> <given-names>M.</given-names></name> <name><surname>Chambers</surname> <given-names>S.</given-names></name> <name><surname>Jochumsen</surname> <given-names>M.</given-names></name> <name><surname>Signal</surname> <given-names>N.</given-names></name> <etal/></person-group>. (<year>2021</year>). <article-title>Electroencephalographic recording of the movement-related cortical potential in ecologically valid movements: a scoping review</article-title>. <source>Front. Neurosci.</source> <volume>15</volume>:<fpage>721387</fpage>. doi: <pub-id pub-id-type="doi">10.3389/fnins.2021.721387</pub-id>, PMID: <pub-id pub-id-type="pmid">34650399</pub-id></citation>
</ref>
<ref id="ref37">
<citation citation-type="confproc"><person-group person-group-type="author"><name><surname>Ono</surname> <given-names>T.</given-names></name> <name><surname>Mukaino</surname> <given-names>M.</given-names></name> <name><surname>Ushiba</surname> <given-names>J.</given-names></name></person-group> <article-title>Functional recovery in upper limb function in stroke survivors by using brain-computer interface a single case A-B-A-B design</article-title>. In: <conf-name>2013 35th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)</conf-name>. <publisher-loc>Osaka</publisher-loc>: <publisher-name>IEEE</publisher-name> (<year>2013</year>), <fpage>265</fpage>&#x2013;<lpage>268</lpage>. <comment>Available at:</comment> <ext-link xlink:href="http://ieeexplore.ieee.org/document/6609488/" ext-link-type="uri">http://ieeexplore.ieee.org/document/6609488/</ext-link></citation>
</ref>
<ref id="ref38">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Osuagwu</surname> <given-names>B. C. A.</given-names></name> <name><surname>Wallace</surname> <given-names>L.</given-names></name> <name><surname>Fraser</surname> <given-names>M.</given-names></name> <name><surname>Vuckovic</surname> <given-names>A.</given-names></name></person-group> (<year>2016</year>). <article-title>Rehabilitation of hand in subacute tetraplegic patients based on brain computer interface and functional electrical stimulation: a randomised pilot study</article-title>. <source>J. Neural Eng.</source> <volume>13</volume>:<fpage>065002</fpage>. doi: <pub-id pub-id-type="doi">10.1088/1741-2560/13/6/065002</pub-id>, PMID: <pub-id pub-id-type="pmid">27739405</pub-id></citation>
</ref>
<ref id="ref39">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Penev</surname> <given-names>Y. P.</given-names></name> <name><surname>Beneke</surname> <given-names>A.</given-names></name> <name><surname>Root</surname> <given-names>K. T.</given-names></name> <name><surname>Meisel</surname> <given-names>E.</given-names></name> <name><surname>Kwak</surname> <given-names>S.</given-names></name> <name><surname>Diaz</surname> <given-names>M.</given-names></name> <etal/></person-group>. (<year>2023</year>). <article-title>Therapeutic effectiveness of brain computer interfaces in stroke patients: a systematic review</article-title>. <source>J. Exp. Neurol.</source> <volume>4</volume>, <fpage>87</fpage>&#x2013;<lpage>93</lpage>. doi: <pub-id pub-id-type="doi">10.33696/Neurol.4.077</pub-id></citation>
</ref>
<ref id="ref40">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Qu</surname> <given-names>H.</given-names></name> <name><surname>Zeng</surname> <given-names>F.</given-names></name> <name><surname>Tang</surname> <given-names>Y.</given-names></name> <name><surname>Shi</surname> <given-names>B.</given-names></name> <name><surname>Wang</surname> <given-names>Z.</given-names></name> <name><surname>Chen</surname> <given-names>X.</given-names></name> <etal/></person-group>. (<year>2024</year>). <article-title>The clinical effects of brain&#x2013;computer interface with robot on upper-limb function for post-stroke rehabilitation: a meta-analysis and systematic review</article-title>. <source>Disabil. Rehabil. Assist. Technol.</source> <volume>19</volume>, <fpage>30</fpage>&#x2013;<lpage>41</lpage>. doi: <pub-id pub-id-type="doi">10.1080/17483107.2022.2060354</pub-id></citation>
</ref>
<ref id="ref41">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Ramos-Murguialday</surname> <given-names>A.</given-names></name> <name><surname>Curado</surname> <given-names>M. R.</given-names></name> <name><surname>Broetz</surname> <given-names>D.</given-names></name> <name><surname>Yilmaz</surname> <given-names>&#x00D6;.</given-names></name> <name><surname>Brasil</surname> <given-names>F. L.</given-names></name> <name><surname>Liberati</surname> <given-names>G.</given-names></name> <etal/></person-group>. (<year>2019</year>). <article-title>Brain-machine Interface in chronic stroke: randomized trial Long-term follow-up</article-title>. <source>Neurorehabil. Neural Repair</source> <volume>33</volume>, <fpage>188</fpage>&#x2013;<lpage>198</lpage>. doi: <pub-id pub-id-type="doi">10.1177/1545968319827573</pub-id>, PMID: <pub-id pub-id-type="pmid">30722727</pub-id></citation>
</ref>
<ref id="ref42">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Ren</surname> <given-names>X.</given-names></name> <name><surname>Jiang</surname> <given-names>N.</given-names></name> <name><surname>Lin</surname> <given-names>C.</given-names></name> <name><surname>Mrachacz-Kersting</surname> <given-names>N.</given-names></name> <name><surname>Dremstrup</surname> <given-names>K.</given-names></name> <name><surname>Farina</surname> <given-names>D.</given-names></name></person-group> (<year>2014</year>). <article-title>Enhanced low-latency detection of motor intention from EEG for closed-loop brain-computer Interface applications</article-title>. <source>I.E.E.E. Trans. Biomed. Eng.</source> <volume>61</volume>, <fpage>288</fpage>&#x2013;<lpage>296</lpage>. doi: <pub-id pub-id-type="doi">10.1109/TBME.2013.2294203</pub-id>, PMID: <pub-id pub-id-type="pmid">24448593</pub-id></citation>
</ref>
<ref id="ref43">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Shakeel</surname> <given-names>A.</given-names></name> <name><surname>Navid</surname> <given-names>M. S.</given-names></name> <name><surname>Anwar</surname> <given-names>M. N.</given-names></name> <name><surname>Mazhar</surname> <given-names>S.</given-names></name> <name><surname>Jochumsen</surname> <given-names>M.</given-names></name> <name><surname>Niazi</surname> <given-names>I. K.</given-names></name></person-group> (<year>2015</year>). <article-title>A review of techniques for detection of movement intention using movement-related cortical potentials</article-title>. <source>Comput. Math. Methods Med.</source> <volume>2015</volume>, <fpage>1</fpage>&#x2013;<lpage>13</lpage>. doi: <pub-id pub-id-type="doi">10.1155/2015/346217</pub-id>, PMID: <pub-id pub-id-type="pmid">26881008</pub-id></citation>
</ref>
<ref id="ref44">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Vidaurre</surname> <given-names>C.</given-names></name> <name><surname>Blankertz</surname> <given-names>B.</given-names></name></person-group> (<year>2010</year>). <article-title>Towards a cure for BCI illiteracy</article-title>. <source>Brain Topogr.</source> <volume>23</volume>, <fpage>194</fpage>&#x2013;<lpage>198</lpage>. doi: <pub-id pub-id-type="doi">10.1007/s10548-009-0121-6</pub-id>, PMID: <pub-id pub-id-type="pmid">19946737</pub-id></citation>
</ref>
<ref id="ref45">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Zhang</surname> <given-names>G.</given-names></name> <name><surname>Davoodnia</surname> <given-names>V.</given-names></name> <name><surname>Sepas-Moghaddam</surname> <given-names>A.</given-names></name> <name><surname>Zhang</surname> <given-names>Y.</given-names></name> <name><surname>Etemad</surname> <given-names>A.</given-names></name></person-group> (<year>2020</year>). <article-title>Classification of hand movements from EEG using a deep attention-based LSTM network</article-title>. <source>IEEE Sensors J.</source> <volume>20</volume>, <fpage>3113</fpage>&#x2013;<lpage>3122</lpage>. doi: <pub-id pub-id-type="doi">10.1109/JSEN.2019.2956998</pub-id></citation>
</ref>
<ref id="ref46">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Zhang</surname> <given-names>X.</given-names></name> <name><surname>Guo</surname> <given-names>Y.</given-names></name> <name><surname>Gao</surname> <given-names>B.</given-names></name> <name><surname>Long</surname> <given-names>J.</given-names></name></person-group> (<year>2020</year>). <article-title>Alpha frequency intervention by electrical stimulation to improve performance in mu-based BCI</article-title>. <source>IEEE Trans. Neural Syst. Rehabil. Eng.</source> <volume>28</volume>, <fpage>1262</fpage>&#x2013;<lpage>1270</lpage>. doi: <pub-id pub-id-type="doi">10.1109/TNSRE.2020.2987529</pub-id>, PMID: <pub-id pub-id-type="pmid">32305926</pub-id></citation>
</ref>
</ref-list>
</back>
</article>