<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article xml:lang="EN" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Med.</journal-id>
<journal-title>Frontiers in Medicine</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Med.</abbrev-journal-title>
<issn pub-type="epub">2296-858X</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fmed.2023.1195451</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Medicine</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>A prospective multicenter clinical research study validating the effectiveness and safety of a chest X-ray-based pulmonary tuberculosis screening software JF CXR-1 built on a convolutional neural network algorithm</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" equal-contrib="yes">
<name><surname>Yang</surname> <given-names>Yang</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="author-notes" rid="fn001"><sup>&#x02020;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1831225/overview"/>
</contrib>
<contrib contrib-type="author" equal-contrib="yes">
<name><surname>Xia</surname> <given-names>Lu</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="author-notes" rid="fn001"><sup>&#x02020;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1779687/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Liu</surname> <given-names>Ping</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1062827/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Yang</surname> <given-names>Fuping</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Wu</surname> <given-names>Yuqing</given-names></name>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Pan</surname> <given-names>Hongqiu</given-names></name>
<xref ref-type="aff" rid="aff5"><sup>5</sup></xref>
</contrib>
<contrib contrib-type="author">
<name><surname>Hou</surname> <given-names>Dailun</given-names></name>
<xref ref-type="aff" rid="aff6"><sup>6</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/1586215/overview"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Liu</surname> <given-names>Ning</given-names></name>
<xref ref-type="aff" rid="aff7"><sup>7</sup></xref>
</contrib>
<contrib contrib-type="author" corresp="yes">
<name><surname>Lu</surname> <given-names>Shuihua</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="corresp" rid="c001"><sup>&#x0002A;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/458728/overview"/>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>Department of Tuberculosis, Shanghai Public Health Clinical Center Affiliated to Fudan University</institution>, <addr-line>Shanghai</addr-line>, <country>China</country></aff>
<aff id="aff2"><sup>2</sup><institution>Department of Pulmonary Medicine, National Clinical Research Center for Infectious Disease, Shenzhen Third People&#x00027;s Hospital/The Second Affiliated Hospital, School of Medicine, Southern University of Science and Technology, Shenzhen</institution>, <addr-line>Guangdong</addr-line>, <country>China</country></aff>
<aff id="aff3"><sup>3</sup><institution>Department of Tuberculosis, Chongqing Public Health Medical Center, Southwest University</institution>, <addr-line>Chongqing</addr-line>, <country>China</country></aff>
<aff id="aff4"><sup>4</sup><institution>Department of Tuberculosis, Jiangxi Chest Hospital, Nanchang</institution>, <addr-line>Jiangxi</addr-line>, <country>China</country></aff>
<aff id="aff5"><sup>5</sup><institution>Department of Tuberculosis, The Third Hospital of Zhenjiang, Zhenjiang</institution>, <addr-line>Jiangsu</addr-line>, <country>China</country></aff>
<aff id="aff6"><sup>6</sup><institution>Department of Radiology, Beijing Chest Hospital, Capital Medical University</institution>, <addr-line>Beijing</addr-line>, <country>China</country></aff>
<aff id="aff7"><sup>7</sup><institution>Department of Tuberculosis, Hebei Chest Hospital, Shijiangzhuang</institution>, <addr-line>Hebei</addr-line>, <country>China</country></aff>
<author-notes>
<fn fn-type="edited-by"><p>Edited by: Ramalingam Bethunaickan, National Institute of Research in Tuberculosis (ICMR), India</p></fn>
<fn fn-type="edited-by"><p>Reviewed by: Chinnaiyan Ponnuraja, National Institute of Research in Tuberculosis (ICMR), India; Umashankar Vetrivel, Indian Council of Medical Research (ICMR), India; Alangudi Palaniappan Natarajan, National Institute of Research in Tuberculosis (ICMR), India</p></fn>
<corresp id="c001">&#x0002A;Correspondence: Shuihua Lu <email>lushuihua66&#x00040;126.com</email></corresp>
<fn fn-type="equal" id="fn001"><p>&#x02020;These authors share first authorship</p></fn></author-notes>
<pub-date pub-type="epub">
<day>15</day>
<month>08</month>
<year>2023</year>
</pub-date>
<pub-date pub-type="collection">
<year>2023</year>
</pub-date>
<volume>10</volume>
<elocation-id>1195451</elocation-id>
<history>
<date date-type="received">
<day>28</day>
<month>03</month>
<year>2023</year>
</date>
<date date-type="accepted">
<day>24</day>
<month>07</month>
<year>2023</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x000A9; 2023 Yang, Xia, Liu, Yang, Wu, Pan, Hou, Liu and Lu.</copyright-statement>
<copyright-year>2023</copyright-year>
<copyright-holder>Yang, Xia, Liu, Yang, Wu, Pan, Hou, Liu and Lu</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p></license> </permissions>
<abstract>
<sec>
<title>Background</title>
<p>Chest radiography (chest X-ray or CXR) plays an important role in the early detection of active pulmonary tuberculosis (TB). In areas with a high TB burden that require urgent screening, there is often a shortage of radiologists available to interpret the X-ray results. Computer-aided detection (CAD) software employed with artificial intelligence (AI) systems may have the potential to solve this problem.</p></sec>
<sec>
<title>Objective</title>
<p>We validated the effectiveness and safety of pulmonary tuberculosis imaging screening software that is based on a convolutional neural network algorithm.</p></sec>
<sec>
<title>Methods</title>
<p>We conducted prospective multicenter clinical research to validate the performance of pulmonary tuberculosis imaging screening software (JF CXR-1). Volunteers under the age of 15 years, both with or without suspicion of pulmonary tuberculosis, were recruited for CXR photography. The software reported a probability score of TB for each participant. The results were compared with those reported by radiologists. We measured sensitivity, specificity, consistency rate, and the area under the receiver operating characteristic curves (AUC) for the diagnosis of tuberculosis. Besides, adverse events (AE) and severe adverse events (SAE) were also evaluated.</p></sec>
<sec>
<title>Results</title>
<p>The clinical research was conducted in six general infectious disease hospitals across China. A total of 1,165 participants were enrolled, and 1,161 were enrolled in the full analysis set (FAS). Men accounted for 60.0% (697/1,161). Compared to the results from radiologists on the board, the software showed a sensitivity of 94.2% (95% CI: 92.0&#x02013;95.8%) and a specificity of 91.2% (95% CI: 88.5&#x02013;93.2%). The consistency rate was 92.7% (91.1&#x02013;94.1%), with a Kappa value of 0.854 (<italic>P</italic> = 0.000). The AUC was 0.98. In the safety set (SS), which consisted of 1,161 participants, 0.3% (3/1,161) had AEs that were not related to the software, and no severe AEs were observed.</p></sec>
<sec>
<title>Conclusion</title>
<p>The software for tuberculosis screening based on a convolutional neural network algorithm is effective and safe. It is a potential candidate for solving tuberculosis screening problems in areas lacking radiologists with a high TB burden.</p></sec></abstract>
<kwd-group>
<kwd>tuberculosis</kwd>
<kwd>screening</kwd>
<kwd>radiography</kwd>
<kwd>deep learning</kwd>
<kwd>convolutional neural network algorithm</kwd>
<kwd>computer-aided detection (CAD)</kwd>
<kwd>diagnosis</kwd>
</kwd-group>
<counts>
<fig-count count="3"/>
<table-count count="5"/>
<equation-count count="0"/>
<ref-count count="33"/>
<page-count count="9"/>
<word-count count="6030"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Pulmonary Medicine</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="s1">
<title>Introduction</title>
<p>Tuberculosis (TB) remains one of the leading causes of death worldwide, killing 1.4 million people per year (<xref ref-type="bibr" rid="B1">1</xref>). Although the disease is largely curable and preventable, an estimated 2.9 million per 10 million people falling sick with TB were not diagnosed or reported to the World Health Organization (WHO) (<xref ref-type="bibr" rid="B2">2</xref>). Therefore, there is a pressing need to improve the early diagnosis of TB disease, thus initiating treatment promptly and reducing the transmission of <italic>Mycobacterium tuberculosis</italic>. One effective strategy is systemic screening, which should distinguish people with a high possibility of TB from those without. Chest radiography (chest X-ray or CXR) is a widely used and cost-effective screening tool for TB detection, achieving a sensitivity and specificity of 85.0 and 96.0% for TB-related abnormalities, respectively (<xref ref-type="bibr" rid="B2">2</xref>&#x02013;<xref ref-type="bibr" rid="B5">5</xref>). While in areas with low resources and a high TB burden requiring mass screening urgently, the interpretation of CXRs is labor-intensive and time-consuming under the circumstances that trained health personnel interpreting the CXRs are very lacking (<xref ref-type="bibr" rid="B6">6</xref>&#x02013;<xref ref-type="bibr" rid="B11">11</xref>).</p>
<p>Computer-aided detection (CAD) technologies, especially those with AI algorithms, vastly increase the capacity of image reading and have similar or even better diagnostic accuracy performance compared to human readers (<xref ref-type="bibr" rid="B12">12</xref>&#x02013;<xref ref-type="bibr" rid="B14">14</xref>). The technologies make it possible to perform accurate mass screening with fewer resources. Thus, the WHO has recommended CAD as an alternative to human interpretation of CXR for screening and triaging pulmonary TB in individuals aged 15 years or older (<xref ref-type="bibr" rid="B2">2</xref>). The edge of CAD is artificial intelligence, especially in the field of deep learning, in which convolutional neural networks (CNNs) are the most promising algorithms for dealing with visual tasks (<xref ref-type="bibr" rid="B15">15</xref>&#x02013;<xref ref-type="bibr" rid="B17">17</xref>). JF CXR-1 (version 2) (JF Healthcare, Jiangxi, China) is a simultaneous CXR detection CAD software based on CNNs that detects multiple thorax diseases, such as TB, lung mass, and lung nodules. The software was trained on 14,160 CXRs from township-level hospitals across China. In the testing phase, 13122 CXRs were provided for JF CXR-1 to detect TB. Among them, 31.5% (4127/13122) were pulmonary tuberculosis, 16.3% (2143/13122) were other pulmonary diseases such as pneumonia, pulmonary abscess, lung cancers, etc., and 52.2% (6852/13122) were normal. And JF CXR-1 has achieved an AUC of 0.94, a sensitivity of 0.91 (3,755/4,127), and a specificity of 0.81 (7,286/8,995), which meet the WHO&#x00027;s criteria for the target product profile. To validate its effectiveness and safety in clinical use, we conducted prospective multicenter clinical research in six general infectious disease hospitals in mainland China.</p></sec>
<sec sec-type="methods" id="s2">
<title>Methods</title>
<sec>
<title>Study setting and population</title>
<p>The study was conducted at Shanghai Public Health Clinical Center, Beijing Chest Hospital, the Third Hospital of Zhenjiang, Chongqing Public Health Medical Center, Jiangxi Province Chest Hospital, and Hebei Chest Hospital. All of them are designated hospitals for tuberculosis in China, and there are also healthy individuals in the medical examination centers of each hospital. Participants were recruited from the visitors from tuberculosis clinics and medical examination centers of the above hospitals since June 2020. The inclusion criteria were (1) being aged 15 or over, (2) being willing to receive the CXR examination or could provide the image (DICOM format) of the posterior-anterior CXR taken in the late 40 days, and (3) voluntarily participating and providing informed written consent. The exclusion criteria included (1) a history of obsolete pulmonary tuberculosis, (2) neutropenia, (3) infection with the human immunodeficiency virus (HIV), (4) subjects whose CXR images do not meet the diagnostic requirements, (5) those with a history of hematological disorders, (6) those with a history of pulmonary lobectomy, (7) those with a history of mental illness or cognitive disorder, (8) those who are pregnant or breastfeeding, (9) those who had participated in pharmaceutical clinical research within 30 days, and (10) those who had other conditions that investigators consider inappropriate for research participation. All enrolled participants provided informed written consent. This study was approved by the ethics committees of Shanghai Public Health Clinical Center, Beijing Chest Hospital, The Third Hospital of Zhenjiang, Chongqing Public Health Medical Center, Jiangxi Province Chest Hospital, and Hebei Chest Hospital, respectively.</p></sec>
<sec>
<title>Procedures</title>
<p>Each participant received a complete blood count (CBC) test and an HIV antibody test. Participants with neutropenia or/and HIV infection were excluded according to the results of the blood tests. Then, each of the rest of the participants received a digital posterior-anterior CXR. Even though not all the centers have the same X-ray machines, efforts were made to set the parameters as similar as possible. The CXR images were saved in DICOM format and imported into the AI-based CAD software (JF CXR-1 v2, produced by Jiangxi Zhongke Jiufeng Smart Medical Technology Co., Ltd.). JF-CXR-1 and the radiologist group read every single CXR image independently and were blinded to all the information, including age and sex. The AI algorithm would produce a probability score for each anonymized image to predict its likelihood of being TB-positive.</p>
<p>A score &#x0003E;0.35 indicated a high possibility of tuberculosis, prompting the need for further diagnostic measures such as microbiological tests and a chest CT scan (When the threshold score = 0.35, the combination of sensitivity, specificity, and Kappa value is the best). The radiologist group consisted of eight certified senior radiologists from a third-party organization. They utilized the Diagnosis for Pulmonary Tuberculosis of Chinese Health Industry Standards as the fundamental criteria for TB x-ray screening. All radiologists have at least 10 years of experience in Grade-A tertiary hospitals. The CXR images were read independently by five senior radiologists. Diagnosis suggestions in CXR reports with &#x0201C;suspect TB&#x0201D; or &#x0201C;TB&#x0201D; were considered positive for TB. &#x0201C;Normal&#x0201D; and &#x0201C;other abnormal&#x0201D; CXR were considered negative for TB. The final decision among the five radiologists was determined based on the principle that the minority is subordinate to the majority. If cases where three radiologists shared the same opinion but the other two disagreed, the image would be sent to three other radiologists for arbitration. The final decision of the radiologist group was determined by the three &#x0201C;arbitrators&#x0201D; in a written report. Two months after the CXR examination, the clinical diagnosis information of every participant was collected. TB diagnosis followed China&#x00027;s National TB Diagnosis Guideline (WS288-2017).</p></sec>
<sec>
<title>Data analysis</title>
<p>All the data were statistically described, including baseline information, effectiveness data, and safety data. The final decision of the radiologist group was set as the reference standard for the software to compare. For effectiveness, the main evaluation indicators were sensitivity and specificity, referring to the results of the radiologists&#x00027; board. The secondary evaluation indicators were the area under the receiver operating characteristic (ROC) curve (AUC) and the consistency rate with the final diagnosis (including bacteriologically confirmed and clinical diagnoses). If an image was diagnosed as TB or non-TB by both the radiologist group and the AI software, it was defined as true positive (TP) or true negative (TN), respectively. If an image was diagnosed as TB by the radiologist group but non-TB by the software, it was defined as a false negative (FN). Moreover, if an image was diagnosed as non-TB by the radiologist group but TB by the software, it was defined as a false positive (FP). Sensitivity = TP/(TP &#x0002B; FN), specificity = TN/(TN &#x0002B; FP) and the consistency rate = (TP &#x0002B; TN)/(TP &#x0002B; TN &#x0002B; FP &#x0002B; FN). The ROC curve was acquired when the sensitivity was set as the <italic>Y</italic>-axis, and 1-specificity was set as the <italic>X</italic>-axis. And the AUC was the area under the ROC.</p>
<p>Sensitivity is the proportion of true positive tests in all patients with a condition. A test or instrument can yield a positive result for a subject with that condition. In our study, it means the test ability of AI software screening out TB on the CXRs compared with the radiologist group.</p>
<p>Specificity is the percentage of people without the disease who are correctly excluded by the test. It is important to exclude people with diseases during screening. In our study, it refers to the ability of AI software to rule out non-TB participants. Ideally, a test should provide high sensitivity and specificity. For safety, adverse events (AE) and severe adverse events (SAE) were evaluated for every participant since their CXR was imported into the AI software and stopped 2 weeks later.</p></sec>
<sec>
<title>Role of the AI developers</title>
<p>The AI developer had no role in study design, data collection, analysis, or manuscript writing, but they provided us with a free account to use the software and free technical support.</p></sec></sec>
<sec sec-type="results" id="s3">
<title>Results</title>
<p>Between 06 Nov 2020, and 04 Jun 2021, 1,218 participants were screened, and 53 failed screening. Thus, 1,165 were enrolled, 1,161 were included in the Full Analysis Set (FAS) and Safety Analysis Set (SS) due to one dropout and three eliminations, and 1,150 were included in the Per Protocol Set (PPS) due to 11 protocol deviations (<xref ref-type="table" rid="T1">Table 1</xref>; <xref ref-type="fig" rid="F1">Figure 1</xref>). The participants&#x00027; ages ranged from 15 to 86 years, and the average was 44.4 &#x000B1; 16.4 years. Men accounted for 60.0% (697/1,161), while women accounted for 40.0% (464/1,161).</p>
<table-wrap position="float" id="T1">
<label>Table 1</label>
<caption><p>Participants distribution.</p></caption> 
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="left"><bold>Description</bold></th>
<th valign="top" align="center"><bold>Center 01</bold></th>
<th valign="top" align="center"><bold>Center 02</bold></th>
<th valign="top" align="center"><bold>Center 03</bold></th>
<th valign="top" align="center"><bold>Center 04</bold></th>
<th valign="top" align="center"><bold>Center 05</bold></th>
<th valign="top" align="center"><bold>Center 06</bold></th>
<th valign="top" align="center"><bold>Total</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">Screening</td>
<td valign="top" align="center">199</td>
<td valign="top" align="center">70</td>
<td valign="top" align="center">154</td>
<td valign="top" align="center">375</td>
<td valign="top" align="center">360</td>
<td valign="top" align="center">60</td>
<td valign="top" align="center">1,218</td>
</tr> <tr>
<td valign="top" align="left">Enrollment</td>
<td valign="top" align="center">186</td>
<td valign="top" align="center">68</td>
<td valign="top" align="center">150</td>
<td valign="top" align="center">358</td>
<td valign="top" align="center">343</td>
<td valign="top" align="center">60</td>
<td valign="top" align="center">1,165</td>
</tr> <tr>
<td valign="top" align="left">Drop out</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">1</td>
</tr> <tr>
<td valign="top" align="left">Elimination</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">3</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">3</td>
</tr> <tr>
<td valign="top" align="left">Protocol deviation</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">6</td>
<td valign="top" align="center">5</td>
<td valign="top" align="center">0</td>
<td valign="top" align="center">11</td>
</tr> <tr>
<td valign="top" align="left">FAS</td>
<td valign="top" align="center">186</td>
<td valign="top" align="center">67</td>
<td valign="top" align="center">150</td>
<td valign="top" align="center">355</td>
<td valign="top" align="center">343</td>
<td valign="top" align="center">60</td>
<td valign="top" align="center">1,161<xref ref-type="table-fn" rid="TN1"><sup>&#x0002A;</sup></xref></td>
</tr> <tr>
<td valign="top" align="left">PPS</td>
<td valign="top" align="center">186</td>
<td valign="top" align="center">67</td>
<td valign="top" align="center">150</td>
<td valign="top" align="center">349</td>
<td valign="top" align="center">338</td>
<td valign="top" align="center">60</td>
<td valign="top" align="center">1,150</td>
</tr>
<tr>
<td valign="top" align="left">SS</td>
<td valign="top" align="center">186</td>
<td valign="top" align="center">67</td>
<td valign="top" align="center">150</td>
<td valign="top" align="center">355</td>
<td valign="top" align="center">343</td>
<td valign="top" align="center">60</td>
<td valign="top" align="center">1,161</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<p>Center 01, Shanghai Public Health Clinical Center; Center 02, Beijing Chest Hospital; Center 03, The Third Hospital of Zhenjiang; Center 04, Chongqing Public Health Medical Center; Center 05, Jiangxi Province Chest Hospital; Center 06, Hebei Chest Hospital. FAS, full analysis set; PPS, per protocol set; SS, safety set.</p>
<fn id="TN1"><label>&#x0002A;</label><p>The reports of ten subjects were lost during an office movement due to the COVID-19.</p></fn>
</table-wrap-foot>
</table-wrap>
<fig id="F1" position="float">
<label>Figure 1</label>
<caption><p>Participant selection process and the diagnostic workflow. FAS, analysis set; SS, safety analysis set; PPS, per protocol set; AUC, area under the receiver operating characteristic curves. &#x0002A;, details are in the methods section.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fmed-10-1195451-g0001.tif"/>
</fig>
<p>In the FAS, the reports of 10 subjects were lost. Thus, the results of 1,151 subjects were analyzed. According to the radiologist group, 601 images were positive for TB, while 550 were negative for TB. Compared with the reference standard, the sensitivity of the software was 94.2% (95% CI: 92.0&#x02013;95.8%), while the specificity was 91.2% (95% CI: 88.5&#x02013;93.2%). Therefore, the consistency rate was 92.7% (95% CI: 91.1&#x02013;94.1%) with a Kappa value of 0.854 (<italic>P</italic> = 0.000) (<xref ref-type="table" rid="T2">Table 2</xref>). The AUC was 0.98 (<xref ref-type="fig" rid="F2">Figure 2</xref>). After 2 months of clinical evaluation since enrollment, the diagnosis remained unclear for 29 out of 1,161 participants. Among the rest of the 1,132 participants, 687 were diagnosed with tuberculosis, while the others were not. Therefore, taking the final diagnosis (including bacteriologically confirmed and clinically diagnosed) as the ground truth, the sensitivity of the software was 78.9% (75.7&#x02013;81.8%), the specificity was 89.9% (86.7&#x02013;92.4%), and the consistency rate was 83.2% (80.9&#x02013;85.3%) with a Kappa value of 0.662 (<italic>p</italic> = 0.000) (<xref ref-type="table" rid="T3">Table 3</xref>).</p>
<table-wrap position="float" id="T2">
<label>Table 2</label>
<caption><p>Performance of JF CXR-1 against radiologists (FAS).<xref ref-type="table-fn" rid="TN2"><sup>&#x0002A;</sup></xref></p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="left" rowspan="2"><bold>Results from JF CXR-1</bold></th>
<th valign="top" align="center" colspan="2"><bold>Results from radiologists</bold></th>
<th valign="top" align="center" rowspan="2"><bold>Total sum</bold></th>
</tr>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="center"><bold>TB (&#x0002B;)</bold></th>
<th valign="top" align="center"><bold>TB (&#x02013;)</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">TB (&#x0002B;)</td>
<td valign="top" align="center">551</td>
<td valign="top" align="center">50</td>
<td valign="top" align="center">601</td>
</tr> <tr>
<td valign="top" align="left">TB (<bold>&#x02013;</bold>)</td>
<td valign="top" align="center">34</td>
<td valign="top" align="center">516</td>
<td valign="top" align="center">550</td>
</tr> <tr>
<td valign="top" align="left">Total sum</td>
<td valign="top" align="center">585</td>
<td valign="top" align="center">566</td>
<td valign="top" align="center">1,151</td>
</tr> <tr>
<td valign="top" align="left">Sensitivity (95%CI)</td>
<td valign="top" align="center">94.2% (92.0&#x02013;95.8%)</td>
</tr> <tr>
<td valign="top" align="left">Specificity (95%CI)</td>
<td valign="top" align="center">91.2% (88.5&#x02013;93.2%)</td>
</tr> <tr>
<td valign="top" align="left">Consistency rate (95%CI)</td>
<td valign="top" align="center">92.7% (91.1&#x02013;94.1%)</td>
</tr>
<tr>
<td valign="top" align="left">Kappa</td>
<td valign="top" align="center">0.854 (<italic>P</italic> = 0.000)</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<fn id="TN2"><label>&#x0002A;</label><p>In the FAS, 10 subjects have no reports of CXR images because they met the exclusion criteria but were mistakenly enrolled.</p></fn>
</table-wrap-foot>
</table-wrap>
<fig id="F2" position="float">
<label>Figure 2</label>
<caption><p>ROC graph for JF CXR-1 of FAS.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fmed-10-1195451-g0002.tif"/>
</fig>
<table-wrap position="float" id="T3">
<label>Table 3</label>
<caption><p>Performance of JF CXR-1 against final diagnosis (FAS).<xref ref-type="table-fn" rid="TN3"><sup>&#x0002A;</sup></xref></p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="left" rowspan="2"><bold>Results from JF CXR-1</bold></th>
<th valign="top" align="center" colspan="2"><bold>Results from clinical diagnosis</bold></th>
<th valign="top" align="center" rowspan="2"><bold>Total sum</bold></th>
</tr>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="center"><bold>TB (&#x0002B;)</bold></th>
<th valign="top" align="center"><bold>TB (&#x02013;)</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">TB (&#x0002B;)</td>
<td valign="top" align="center">542</td>
<td valign="top" align="center">45</td>
<td valign="top" align="center">587</td>
</tr> <tr>
<td valign="top" align="left">TB (<bold>&#x02013;</bold>)</td>
<td valign="top" align="center">145</td>
<td valign="top" align="center">400</td>
<td valign="top" align="center">545</td>
</tr> <tr>
<td valign="top" align="left">Total sum</td>
<td valign="top" align="center">687</td>
<td valign="top" align="center">445</td>
<td valign="top" align="center">1,132</td>
</tr> <tr>
<td valign="top" align="left">Sensitivity (95%CI)</td>
<td valign="top" align="center">78.9% (75.7&#x02013;81.8%)</td>
</tr> <tr>
<td valign="top" align="left">Specificity (95%CI)</td>
<td valign="top" align="center">89.9% (86.7&#x02013;92.4%)</td>
</tr> <tr>
<td valign="top" align="left">consistency rate (95%CI)</td>
<td valign="top" align="center">83.2% (81.0&#x02013;85.3%)</td>
</tr>
<tr>
<td valign="top" align="left">Kappa</td>
<td valign="top" align="center">0.662 (<italic>P</italic> = 0.000)</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<fn id="TN3"><label>&#x0002A;</label><p>Final diagnosis included bacteriologically confirmed and clinical diagnosed cases.</p></fn>
</table-wrap-foot>
</table-wrap>
<p>In the PPS, the results of 1,150 subjects were analyzed. We found that 600 images were positive for TB, while 550 were negative for TB. Compared with the reference standard, the sensitivity of the software was 94.2% (92.0%&#x02212;95.8%), while the specificity was 91.2% (88.5&#x02013;93.2%). Therefore, the consistency rate was 92.7% (91.1&#x02013;94.1%) with a Kappa value of 0.854 (<italic>P</italic> = 0.000) (<xref ref-type="table" rid="T4">Table 4</xref>). The AUC was 0.98 (<xref ref-type="fig" rid="F3">Figure 3</xref>). The diagnosis remained unclear for 29 out of 1,161 participants after 2 months of clinical evaluation since enrollment. Among the rest of the 1,121 participants, 582 were diagnosed with tuberculosis, while the other 539 were not. Therefore, taking the final diagnosis (including bacteriologically confirmed and clinically diagnosed) as the ground truth, the sensitivity of the software was 79.3% (76.1&#x02013;82.2%), the specificity was 90.5% (87.4&#x02013;92.9%), and the consistency rate was 83.7% (81.4&#x02013;85.7%) with a Kappa value of 0.671 (<italic>p</italic> = 0.000) (<xref ref-type="table" rid="T5">Table 5</xref>). Since it has already been reported that the JF CXR-1 v2 algorithm performed worse among older age groups (&#x0003E;60 years) (<xref ref-type="bibr" rid="B13">13</xref>), performance analysis of the tool with both &#x0003E;60 and &#x0003C; 60 years was checked parallelly. In the FAS, 20.2% (233/1,151) of cases were &#x0003E;60 years old, and 79.8% (918/1,151) were &#x02264; 60 years old. When the results of CXRs were set as the reference standard, the sensitivity and specificity in the older group were 0.95(125/132) and 0.85(86/101), respectively, and 0.94(426/453) and 0.93(432/465) in the younger group, respectively (<xref ref-type="supplementary-material" rid="SM1">Supplementary Table S1</xref>). Nevertheless, the sample size was only 233, which was too small to draw a powerful conclusion.</p>
<table-wrap position="float" id="T4">
<label>Table 4</label>
<caption><p>Performance of JF CXR-1 against radiologists (PPS).<sup>&#x0002A;</sup></p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="left" rowspan="2"><bold>Results from JF-CXR1</bold></th>
<th valign="top" align="center" colspan="2"><bold>Results from radiologists</bold></th>
<th valign="top" align="center" rowspan="2"><bold>Total sum</bold></th>
</tr>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="center"><bold>TB (&#x0002B;)</bold></th>
<th valign="top" align="center"><bold>TB (&#x02013;)</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">TB (&#x0002B;)</td>
<td valign="top" align="center">550</td>
<td valign="top" align="center">50</td>
<td valign="top" align="center">600</td>
</tr> <tr>
<td valign="top" align="left">TB (<bold>&#x02013;</bold>)</td>
<td valign="top" align="center">34</td>
<td valign="top" align="center">516</td>
<td valign="top" align="center">550</td>
</tr> <tr>
<td valign="top" align="left">Total sum</td>
<td valign="top" align="center">584</td>
<td valign="top" align="center">566</td>
<td valign="top" align="center">1,150</td>
</tr> <tr>
<td valign="top" align="left">Sensitivity (95%CI)</td>
<td valign="top" align="center">94.2% (92.0&#x02013;95.8%)</td>
</tr> <tr>
<td valign="top" align="left">Specificity (95%CI)</td>
<td valign="top" align="center">91.2% (88.5&#x02013;93.2%)</td>
</tr> <tr>
<td valign="top" align="left">consistency rate (95%CI)</td>
<td valign="top" align="center">92.7% (91.1&#x02013;94.1%)</td>
</tr>
<tr>
<td valign="top" align="left">Kappa</td>
<td valign="top" align="center">0.854 (<italic>P</italic> = 0.000)</td>
</tr>
</tbody>
</table>
</table-wrap>
<fig id="F3" position="float">
<label>Figure 3</label>
<caption><p>ROC graph for JF CXR-1 of PPS.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fmed-10-1195451-g0003.tif"/>
</fig>
<table-wrap position="float" id="T5">
<label>Table 5</label>
<caption><p>Performance of JF CXR-1 against final diagnosis (PPS).<xref ref-type="table-fn" rid="TN4"><sup>&#x0002A;</sup></xref></p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="left" rowspan="2"><bold>Results from JF CXR-1</bold></th>
<th valign="top" align="center" colspan="2"><bold>Results from clinical diagnosis</bold></th>
<th valign="top" align="center" rowspan="2"><bold>Total sum</bold></th>
</tr>
<tr style="background-color:#919497;color:#ffffff">
<th valign="top" align="center"><bold>TB (&#x0002B;)</bold></th>
<th valign="top" align="center"><bold>TB (&#x02013;)</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">TB (&#x0002B;)</td>
<td valign="top" align="center">540</td>
<td valign="top" align="center">42</td>
<td valign="top" align="center">582</td>
</tr> <tr>
<td valign="top" align="left">TB (<bold>&#x02013;</bold>)</td>
<td valign="top" align="center">141</td>
<td valign="top" align="center">398</td>
<td valign="top" align="center">539</td>
</tr> <tr>
<td valign="top" align="left">Total sum</td>
<td valign="top" align="center">681</td>
<td valign="top" align="center">440</td>
<td valign="top" align="center">1,121</td>
</tr> <tr>
<td valign="top" align="left">Sensitivity (95%CI)</td>
<td valign="top" align="center">79.3% (76.1&#x02013;82.2%)</td>
</tr> <tr>
<td valign="top" align="left">Specificity (95%CI)</td>
<td valign="top" align="center">90.5% (87.4&#x02013;92.9%)</td>
</tr> <tr>
<td valign="top" align="left">consistency rate (95%CI)</td>
<td valign="top" align="center">83.7% (81.4&#x02013;85.7%)</td>
</tr>
<tr>
<td valign="top" align="left">Kappa</td>
<td valign="top" align="center">0.671 (<italic>P</italic> = 0.000)</td>
</tr>
</tbody>
</table>
<table-wrap-foot>
<fn id="TN4"><label>&#x0002A;</label><p>Final diagnosis included bacteriologically confirmed and clinical diagnosed cases.</p></fn>
</table-wrap-foot>
</table-wrap>
<p>In the safety set (SS), which included 1,161 participants, 0.3% (3/1,161) had AEs, and no SAE was reported. One had his ankle twisted while running, and two had drug-induced dermatitis after initiating anti-tuberculosis treatment. The AEs were mild, generally not bothersome, and unrelated to the software. No SAE was observed.</p></sec>
<sec sec-type="discussion" id="s4">
<title>Discussion</title>
<p>In the era of the application of AI, the state-of-the-art in image recognition is CNNs, which have attracted a number of researchers to develop algorithms to replace human readers to identify tuberculosis in chest CXR images (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B15">15</xref>, <xref ref-type="bibr" rid="B18">18</xref>&#x02013;<xref ref-type="bibr" rid="B24">24</xref>). However, the majority of the software is only tested with retrospective analysis (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B13">13</xref>, <xref ref-type="bibr" rid="B14">14</xref>, <xref ref-type="bibr" rid="B17">17</xref>, <xref ref-type="bibr" rid="B19">19</xref>, <xref ref-type="bibr" rid="B25">25</xref>&#x02013;<xref ref-type="bibr" rid="B27">27</xref>) or only with datasets (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B18">18</xref>, <xref ref-type="bibr" rid="B22">22</xref>, <xref ref-type="bibr" rid="B23">23</xref>). Few studies focus on software evaluation in prospective clinical contexts (<xref ref-type="bibr" rid="B28">28</xref>, <xref ref-type="bibr" rid="B29">29</xref>). JF CXR-1 v2 had been certified by the National Medical Products Administration of China for the screening and auxiliary diagnosis of active pulmonary tuberculosis in individuals no younger than 15 years and without immunodeficiency. We conducted this prospective multicenter clinical research to evaluate the performance of JF CXR-1 v2 to recognize tuberculosis in persons without immunodeficiency and aged 15 years or older. As described above, 1,151 subjects in the FAS were analyzed. Compared to the results from radiologists on the board (considered the reference standard), the software showed a sensitivity of 94.2% (95% CI: 92.0&#x02013;95.8%) and a specificity of 91.2% (95% CI: 88.5&#x02013;93.2%), and the consistency rate was 92.7% (91.1&#x02013;94.1%) with a Kappa value of 0.854 (<italic>P</italic> = 0.000). The AUC was 0.98 (<xref ref-type="fig" rid="F3">Figure 3</xref>). The results were very close in the PPS. The study of Nijiati et al. (<xref ref-type="bibr" rid="B17">17</xref>), which evaluated the performance of a trained AI model with CNNs screening TB in chest CXRs in an underdeveloped area, demonstrated its sensitivity, specificity, consistency rate, and AUC of 85.7%, 94.1%, 91.0%, and 0.910, respectively. Noteworthy, Nijiati et al. (<xref ref-type="bibr" rid="B17">17</xref>) also took the results of radiologists as a reference standard because their purpose was screening rather than triage. In a prospective study of a pilot active TB onsite screening project (<xref ref-type="bibr" rid="B4">4</xref>), where the reference standard in the project was bacteriologically confirmed, and clinically diagnosed TB, JF CXR-1 had sensitivity, specificity, and AUC of 100.0%, 95.7%, and 0.978 at threshold 30, and of 75.0%, 96.8%, and 0.859 at threshold 50. When the same reference standard was used in our study, the sensitivity, specificity, and consistency rates were 78.9% (75.7&#x02013;81.8%), 89.9% (86.7&#x02013;92.4%), and 83.2% (80.9&#x02013;85.3%) with a Kappa value of 0.66 (<italic>p</italic> = 0.000) in the FAS, and 79.3% (76.1&#x02013;82.2%), 90.5% (87.4&#x02013;92.9%), and 83.7% (81.4&#x02013;85.7%) with a Kappa value of 0.67 (<italic>p</italic> = 0.000) (<xref ref-type="table" rid="T5">Table 5</xref>) in the PPS.</p>
<p>The first open evaluation of JF CXR-1 was reported by Qin et al. in 2021 (<xref ref-type="bibr" rid="B13">13</xref>). JF CXR-1 was one of the five commercial AI algorithms evaluated for TB triaging in Dhaka, Bangladesh, a high-burden setting. Chest CXRs from 23,954 individuals were included in the analysis, and Xpert was set as the reference standard. However, JF CXR-1 has not met the WHO&#x00027;s Target Product Profile (TPP) of a triage test of at least 90.0% sensitivity and at least 70.0% specificity (<xref ref-type="bibr" rid="B30">30</xref>), similar to that of InferRead DR and Lunit INSIGHT. It has been proven that JF-CXR-1 significantly outperformed radiologists. When the sensitivity was fixed at 90.0%, the specificity was 61.1% (60.4&#x02013;61.8%), while when the specificity was fixed at 70.0%, the sensitivity was 85.0% (83.8&#x02013;86.2%). Moreover, the AUC was 0.849 (0.843&#x02013;0.855). It was found to reduce half of the required Xpert tests while maintaining a sensitivity above 90.0%. JF CXR-1 had higher sensitivity for most of the decision thresholds (above &#x0007E;0.15), which confers it more competence to be a better screening tool. Codlin et al. (<xref ref-type="bibr" rid="B25">25</xref>) conducted an independent evaluation of CAD software for TB screening; 12 types of software were included. The performance of each software was compared against both an expert and an intermediate human reader. Xpert results were the reference standard, and half of the 12 software programs, including JF CXR-1, achieved similar results on par with the expert reader. The AUC of JF CXR-1 was 0.77 (0.73&#x02013;0.81), ranking third among the 12 evaluated CAD software.</p>
<p>Currently, there are 17 available or upcoming AI-CAD products for TB detection, including JF CXR-1. Nevertheless, only two are in the catalog of the Global Drug Facility of the Stop TB Partnership: CAD4TB software (The Netherlands) and InferRead DR Chest (Japan). JF CXR-1 is still under evaluation by the WHO (2020 report). CAD4TB is an example in Qin et al.&#x00027;s retrospective research (<xref ref-type="bibr" rid="B13">13</xref>) of comparing the competence of 5 AI-CAD products for TB triaging, including CAD4TB, InferRead DR, Lunit INSIGHT, JF CXR-1, and qXR. CAD4TB was the top performer with sensitivity, specificity, and AUC of 90.0% (89.0&#x02013;91.0%), 72.9% (72.3&#x02013;73.5%), and 0.903, respectively. Besides, 49.0% of the people triaged by CAD4TB would be recalled for confirmatory tests in a program focused on capturing almost all people with tuberculosis, while the percentage was 57.0% by JF CXR-1 (<xref ref-type="bibr" rid="B13">13</xref>). In another prospective research, (<xref ref-type="bibr" rid="B31">31</xref>) which recruited symptomatic adults in a Pakistani hospital and used a reference mycobacterial culture of two sputa, CAD4TB had a sensitivity, specificity, and AUC of 93.0% (90.0&#x02013;96.0%), 69.0% (67.0&#x02013;71.0%), and 0.87 (0.85&#x02013;0.86), respectively. In brief, the AUC of CAD4TB ranges from 0.71 to 0.94 (<xref ref-type="bibr" rid="B14">14</xref>, <xref ref-type="bibr" rid="B32">32</xref>), and nearly all the reference standards were set according to the bacteriological results. The sensitivity, specificity, consistency rate, and AUC of JF CXR-1 in our study were non-inferior to most results from other AI-based software. However, since the reference standard in our study was from the human reader instead of bacteriological results, the application of JF CXR-1 is limited to clinical use. While it could play an important role in screening programs, especially in resource-constrained areas (<xref ref-type="bibr" rid="B4">4</xref>), the proportion of bacterially confirmed TB only accounts for 63.0% (<xref ref-type="bibr" rid="B1">1</xref>). Moreover, the purpose of screening is to pick out people suspected of having TB and those who need further diagnostic evaluation.</p>
<p>Our study has the advantages of a prospective nature and clinical validation, where the CXR data has not been used for software training and testing. There are also limitations. First, participants with obsolete tuberculosis were not included. Considering the confusing presentation of CXRs between obsolete tuberculosis and active tuberculosis, the efficiency of AI software might be lowered once obsolete tuberculosis is included. Second, because our purpose was to evaluate the screening performance of JF CXR-1, the main reference standard was the results drawn from the radiologist rather than the bacteriological results; the bacteriological screen has not been feasible due to financial reasons in a less developed, high-TB-burden country like China. Third, when it comes to real-world use, whether the software performs better than human readers remains uncertain because the comparison to bacteria was not conducted. Fourth, since the tool has shown poor performance in the age group &#x0003E;60 years (<xref ref-type="bibr" rid="B13">13</xref>), future studies are needed to fine-tune the algorithm in this area. The technology of AI is used not only for human resource optimization but also for better output. Therefore, we plan to conduct clinical research covering obsolete pulmonary tuberculosis participants and further evaluate the triage performance (with bacterial results) of JF CXR-1 in the future.</p>
<p>Furthermore, since the application of AI in tuberculosis screening is still under exploration, there are some practical considerations, especially in resource-constrained settings. First, some people might have little trust in AI and worry about data safety. Therefore, people undergoing TB screening should be aware that the screening tool utilizes AI software, and they should be given informed consent before proceeding with the screening process. Second, safety should be guaranteed for AI in healthcare (<xref ref-type="bibr" rid="B33">33</xref>). Unlike AI-based clinical decision support (CDS) software, screening software might be safer because of diagnosis procedures after screening. However, the reliability, validity, and stability of the software still need to be checked from time to time. Third, the network infrastructure might still be insufficient in resource-constrained settings, and there is the cost of managing network establishment, maintenance, and repair. There are limitations to CNNs: (1) CNNs have high computational requirements, and (2) since the CNNs have multiple layers, the training process takes a particularly long time if the computer does not have a powerful graphics processing unit (GPU). Even if TB screening with AI could save much money, whether resource-constrained areas could afford other costs or not remains problematic.</p>
<p>In our study, the software for tuberculosis screening based on a convolutional neural network algorithm was effective and safe, with satisfying diagnosis performance. It is a potential candidate for solving tuberculosis screening problems in areas lacking radiologists with a high TB burden.</p></sec>
<sec sec-type="data-availability" id="s5">
<title>Data availability statement</title>
<p>The raw data supporting the conclusions of this article will be made available by the authors, without undue reservation.</p></sec>
<sec sec-type="ethics-statement" id="s6">
<title>Ethics statement</title>
<p>The studies involving humans were approved by The Ethics Committee of Shanghai Public Health Clinical Center, The Ethics Committee of Beijing Chest Hospital, The Ethics Committee of the Third Hospital of Zhenjiang, The Ethics Committee of Chongqing Public Health Medical Center, The Ethics Committee of Jiangxi Province Chest Hospital, The Ethics Committee of Hebei Chest Hospital. The studies were conducted in accordance with the local legislation and institutional requirements. Written informed consent for participation in this study was provided by the participants&#x00027; legal guardians/next of kin. No potentially identifiable images or data are presented in this study.</p></sec>
<sec sec-type="author-contributions" id="s7">
<title>Author contributions</title>
<p>SL designed the study. YY, LX, and PL were responsible for conducting clinical research at Shanghai Public Health Clinical Center. FY, YW, HP, DH, and NL were responsible for conducting research at Chongqing Public Health Medical Center, Jiangxi Chest Hospital, the Third Hospital of Zhenjiang, Beijing Chest Hospital, Hebei Chest Hospital, and respectively. YY and LX gathered the data from all six centers, did the statistical analysis, and wrote the manuscript. All authors contributed to the article and approved the submitted version.</p></sec>
</body>
<back>
<sec sec-type="funding-information" id="s8">
<title>Funding</title>
<p>This study was supported by grants from the Shanghai Public Health Clinical Center (No. KY-GW-2021-21), the Shenzhen Key Medical Discipline Construction Fund (SZGSP010), the National Clinical Research Center for Infectious Diseases (Tuberculosis) (No. 2020B1111170014), the Shenzhen Clinical Research Center for Tuberculosis [No. (2021) 287 by the Shenzhen Scientific and Technological Innovation], the National Key Research and Development Program of China (2021YFC2301503), and the National Natural and Science Foundation of China (81873884, 82171739, 81770011, and 81900005).</p>
</sec>
<ack><p>We gratefully acknowledge the work of the Shanghai Public Health Clinical Center, Beijing Chest Hospital, The Third Hospital of Zhenjiang, Chongqing Public Health Medical Center, Jiangxi Province Chest Hospital, and Hebei Chest Hospital, all the colleagues, participants, and others who contributed to revealing the scientific truth.</p>
</ack>
<sec sec-type="COI-statement" id="conf1">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec sec-type="disclaimer" id="s9">
<title>Publisher&#x00027;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<sec sec-type="supplementary-material" id="s10">
<title>Supplementary material</title>
<p>The Supplementary Material for this article can be found online at: <ext-link ext-link-type="uri" xlink:href="https://www.frontiersin.org/articles/10.3389/fmed.2023.1195451/full#supplementary-material">https://www.frontiersin.org/articles/10.3389/fmed.2023.1195451/full#supplementary-material</ext-link></p>
<supplementary-material xlink:href="Data_Sheet_1.docx" id="SM1" mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document" xmlns:xlink="http://www.w3.org/1999/xlink"/></sec>
<ref-list>
<title>References</title>
<ref id="B1">
<label>1.</label>
<citation citation-type="book"><person-group person-group-type="author"><collab>WHO</collab></person-group>. <source>Global Tuberculosis Report 2022</source>. <publisher-loc>Geneva</publisher-loc>: <publisher-name>World Health Organization</publisher-name> (<year>2022</year>).</citation>
</ref>
<ref id="B2">
<label>2.</label>
<citation citation-type="book"><person-group person-group-type="author"><name><surname>WHO</surname> <given-names>Operational Handbook on Tuberculosis</given-names></name></person-group>. <source>Module 2: Screening - Systematic Screening for Tuberculosis Disease</source>. <publisher-loc>Geneva</publisher-loc>: <publisher-name>World Health Organizatio.</publisher-name> (<year>2021</year>).<pub-id pub-id-type="pmid">33822560</pub-id></citation></ref>
<ref id="B3">
<label>3.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hwang</surname> <given-names>EJ</given-names></name> <name><surname>Park</surname> <given-names>S</given-names></name> <name><surname>Jin</surname> <given-names>K-N</given-names></name> <name><surname>Kim</surname> <given-names>JI</given-names></name> <name><surname>Choi</surname> <given-names>SY</given-names></name> <name><surname>Lee</surname> <given-names>JH</given-names></name> <etal/></person-group>. <article-title>Development and validation of a deep learning-based automatic detection algorithm for active pulmonary tuberculosis on chest radiographs</article-title>. <source>Clin Infect Dis.</source> (<year>2019</year>) <volume>69</volume>:<fpage>739</fpage>&#x02013;<lpage>47</lpage>. <pub-id pub-id-type="doi">10.1093/cid/ciy967</pub-id><pub-id pub-id-type="pmid">30418527</pub-id></citation></ref>
<ref id="B4">
<label>4.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Liao</surname> <given-names>Q</given-names></name> <name><surname>Feng</surname> <given-names>H</given-names></name> <name><surname>Li</surname> <given-names>Y</given-names></name> <name><surname>Lai</surname> <given-names>X</given-names></name> <name><surname>Pan</surname> <given-names>J</given-names></name> <name><surname>Zhou</surname> <given-names>F</given-names></name> <etal/></person-group>. <article-title>Evaluation of an artificial intelligence (AI) system to detect tuberculosis on chest X-ray at a pilot active screening project in Guangdong, China in 2019</article-title>. <source>J Xray Sci Technol.</source> (<year>2022</year>) <volume>30</volume>:<fpage>221</fpage>&#x02013;<lpage>30</lpage>. <pub-id pub-id-type="doi">10.3233/XST-211019</pub-id><pub-id pub-id-type="pmid">34924433</pub-id></citation></ref>
<ref id="B5">
<label>5.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>WHO</collab></person-group>. <source>Chest Radiography in Tuberculosis Detection: Summary of Current WHO Recommendations and Guidance on Programmatic Approaches</source>. Geneva: World Health Organization (<year>2016</year>). Available online at: <ext-link ext-link-type="uri" xlink:href="https://www.who.int/publications/i/item/9789241511506">https://www.who.int/publications/i/item/9789241511506</ext-link> (accessed January 3, 2023).</citation>
</ref>
<ref id="B6">
<label>6.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hoog</surname> <given-names>AHV</given-names></name> <name><surname>Meme</surname> <given-names>HK</given-names></name> <name><surname>van Deutekom</surname> <given-names>H</given-names></name> <name><surname>Mithika</surname> <given-names>AM</given-names></name> <name><surname>Olunga</surname> <given-names>C</given-names></name> <name><surname>Onyino</surname> <given-names>F</given-names></name> <etal/></person-group>. <article-title>High sensitivity of chest radiograph reading by clinical officers in a tuberculosis prevalence survey</article-title>. <source>Int J Tuberc Lung Dis.</source> (<year>2011</year>) <volume>15</volume>:<fpage>1308</fpage>&#x02013;<lpage>14</lpage>. <pub-id pub-id-type="doi">10.5588/ijtld.11.0004</pub-id><pub-id pub-id-type="pmid">22283886</pub-id></citation></ref>
<ref id="B7">
<label>7.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Melendez</surname> <given-names>J</given-names></name> <name><surname>S&#x000E1;nchez</surname> <given-names>CI</given-names></name> <name><surname>Philipsen</surname> <given-names>RHHM</given-names></name> <name><surname>Maduskar</surname> <given-names>P</given-names></name> <name><surname>Dawson</surname> <given-names>R</given-names></name> <name><surname>Theron</surname> <given-names>G</given-names></name> <etal/></person-group>. <article-title>An automated tuberculosis screening strategy combining X-ray-based computer-aided detection and clinical information</article-title>. <source>Sci Rep.</source> (<year>2016</year>) <volume>6</volume>:<fpage>25265</fpage>. <pub-id pub-id-type="doi">10.1038/srep25265</pub-id><pub-id pub-id-type="pmid">27126741</pub-id></citation></ref>
<ref id="B8">
<label>8.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Pande</surname> <given-names>T</given-names></name> <name><surname>Pai</surname> <given-names>M</given-names></name> <name><surname>Khan</surname> <given-names>FA</given-names></name> <name><surname>Denkinger</surname> <given-names>CM</given-names></name></person-group>. <article-title>Use of chest radiography in the 22 highest tuberculosis burden countries</article-title>. <source>Eur Respir J.</source> (<year>2015</year>) <volume>46</volume>:<fpage>1816</fpage>&#x02013;<lpage>9</lpage>. <pub-id pub-id-type="doi">10.1183/13993003.01064-2015</pub-id><pub-id pub-id-type="pmid">26405288</pub-id></citation></ref>
<ref id="B9">
<label>9.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Piccazzo</surname> <given-names>R</given-names></name> <name><surname>Paparo</surname> <given-names>F</given-names></name> <name><surname>Garlaschi</surname> <given-names>G</given-names></name></person-group>. <article-title>Diagnostic accuracy of chest radiography for the diagnosis of tuberculosis (TB) and its role in the detection of latent TB infection: a systematic review</article-title>. <source>J Rheumatol Suppl.</source> (<year>2014</year>) <volume>91</volume>:<fpage>32</fpage>&#x02013;<lpage>40</lpage>. <pub-id pub-id-type="doi">10.3899/jrheum.140100</pub-id><pub-id pub-id-type="pmid">24788998</pub-id></citation></ref>
<ref id="B10">
<label>10.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Pinto</surname> <given-names>LM</given-names></name> <name><surname>Pai</surname> <given-names>M</given-names></name> <name><surname>Dheda</surname> <given-names>K</given-names></name> <name><surname>Schwartzman</surname> <given-names>K</given-names></name> <name><surname>Menzies</surname> <given-names>D</given-names></name> <name><surname>Steingart</surname> <given-names>KR</given-names></name></person-group>. <article-title>Scoring systems using chest radiographic features for the diagnosis of pulmonary tuberculosis in adults: a systematic review</article-title>. <source>Eur Respir J.</source> (<year>2013</year>) <volume>42</volume>:<fpage>480</fpage>&#x02013;<lpage>94</lpage>. <pub-id pub-id-type="doi">10.1183/09031936.00107412</pub-id><pub-id pub-id-type="pmid">23222871</pub-id></citation></ref>
<ref id="B11">
<label>11.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Van&#x00027;t Hoog</surname> <given-names>A</given-names></name> <name><surname>Viney</surname> <given-names>K</given-names></name> <name><surname>Biermann</surname> <given-names>O</given-names></name> <name><surname>Yang</surname> <given-names>B</given-names></name> <name><surname>Leeflang</surname> <given-names>MM</given-names></name> <name><surname>Langendam</surname> <given-names>MW</given-names></name></person-group>. <article-title>Symptom- and chest-radiography screening for active pulmonary tuberculosis in HIV-negative adults and adults with unknown HIV status</article-title>. <source>Cochr Database Syst Rev</source>. (<year>2022</year>) <volume>3</volume>:<fpage>CD010890</fpage>. <pub-id pub-id-type="doi">10.1002/14651858.CD010890.pub2</pub-id><pub-id pub-id-type="pmid">35320584</pub-id></citation></ref>
<ref id="B12">
<label>12.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Cao XF Li</surname> <given-names>Y</given-names></name> <name><surname>Xin</surname> <given-names>HN</given-names></name> <name><surname>Zhang</surname> <given-names>HR</given-names></name> <name><surname>Pai</surname> <given-names>M</given-names></name> <name><surname>Gao</surname> <given-names>L</given-names></name></person-group>. <article-title>Application of artificial intelligence in digital chest radiography reading for pulmonary tuberculosis screening</article-title>. <source>Chronic Dis Transl Med.</source> (<year>2021</year>) <volume>7</volume>:<fpage>35</fpage>&#x02013;<lpage>40</lpage>. <pub-id pub-id-type="doi">10.1016/j.cdtm.2021.02.001</pub-id><pub-id pub-id-type="pmid">34013178</pub-id></citation></ref>
<ref id="B13">
<label>13.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Qin</surname> <given-names>ZZ</given-names></name> <name><surname>Ahmed</surname> <given-names>S</given-names></name> <name><surname>Sarker</surname> <given-names>MS</given-names></name> <name><surname>Paul</surname> <given-names>K</given-names></name> <name><surname>Adel</surname> <given-names>ASS</given-names></name> <name><surname>Naheyan</surname> <given-names>T</given-names></name> <etal/></person-group>. <article-title>Tuberculosis detection from chest x-rays for triaging in a high tuberculosis-burden setting: an evaluation of five artificial intelligence algorithms</article-title>. <source>Lancet Digit Health.</source> (<year>2021</year>) <volume>3</volume>:<fpage>e543</fpage>&#x02013;<lpage>54</lpage>. <pub-id pub-id-type="doi">10.1016/S2589-7500(21)00116-3</pub-id><pub-id pub-id-type="pmid">34446265</pub-id></citation></ref>
<ref id="B14">
<label>14.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Qin</surname> <given-names>ZZ</given-names></name> <name><surname>Sander</surname> <given-names>MS</given-names></name> <name><surname>Rai</surname> <given-names>B</given-names></name> <name><surname>Titahong</surname> <given-names>CN</given-names></name> <name><surname>Sudrungrot</surname> <given-names>S</given-names></name> <name><surname>Laah</surname> <given-names>SN</given-names></name> <etal/></person-group>. <article-title>Using artificial intelligence to read chest radiographs for tuberculosis detection: a multi-site evaluation of the diagnostic accuracy of three deep learning systems</article-title>. <source>Sci Rep.</source> (<year>2019</year>) <volume>9</volume>:<fpage>15000</fpage>. <pub-id pub-id-type="doi">10.1038/s41598-019-51503-3</pub-id><pub-id pub-id-type="pmid">31628424</pub-id></citation></ref>
<ref id="B15">
<label>15.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lakhani</surname> <given-names>P</given-names></name> <name><surname>Sundaram</surname> <given-names>B</given-names></name></person-group>. <article-title>Deep learning at chest radiography: automated classification of pulmonary tuberculosis by using convolutional neural networks</article-title>. <source>Radiology.</source> (<year>2017</year>) <volume>284</volume>:<fpage>574</fpage>&#x02013;<lpage>82</lpage>. <pub-id pub-id-type="doi">10.1148/radiol.2017162326</pub-id><pub-id pub-id-type="pmid">28436741</pub-id></citation></ref>
<ref id="B16">
<label>16.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Ma</surname> <given-names>L</given-names></name> <name><surname>Wang</surname> <given-names>Y</given-names></name> <name><surname>Guo</surname> <given-names>L</given-names></name> <name><surname>Zhang</surname> <given-names>Y</given-names></name> <name><surname>Wang</surname> <given-names>P</given-names></name> <name><surname>Pei</surname> <given-names>X</given-names></name> <etal/></person-group>. <article-title>Developing and verifying automatic detection of active pulmonary tuberculosis from multi-slice spiral CT images based on deep learning</article-title>. <source>J Xray Sci Technol.</source> (<year>2020</year>) <volume>28</volume>:<fpage>939</fpage>&#x02013;<lpage>51</lpage>. <pub-id pub-id-type="doi">10.3233/XST-200662</pub-id><pub-id pub-id-type="pmid">32651351</pub-id></citation></ref>
<ref id="B17">
<label>17.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nijiati</surname> <given-names>M</given-names></name> <name><surname>Zhang</surname> <given-names>Z</given-names></name> <name><surname>Abulizi</surname> <given-names>A</given-names></name> <name><surname>Miao</surname> <given-names>H</given-names></name> <name><surname>Tuluhong</surname> <given-names>A</given-names></name> <name><surname>Quan</surname> <given-names>S</given-names></name> <etal/></person-group>. <article-title>Deep learning assistance for tuberculosis diagnosis with chest radiography in low-resource settings</article-title>. <source>J Xray Sci Technol.</source> (<year>2021</year>) <volume>29</volume>:<fpage>785</fpage>&#x02013;<lpage>96</lpage>. <pub-id pub-id-type="doi">10.3233/XST-210894</pub-id><pub-id pub-id-type="pmid">34219703</pub-id></citation></ref>
<ref id="B18">
<label>18.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nijiati</surname> <given-names>M</given-names></name> <name><surname>Ma</surname> <given-names>J</given-names></name> <name><surname>Hu</surname> <given-names>C</given-names></name> <name><surname>Tuersun</surname> <given-names>A</given-names></name> <name><surname>Abulizi</surname> <given-names>A</given-names></name> <name><surname>Kelimu</surname> <given-names>A</given-names></name> <etal/></person-group>. <article-title>Artificial intelligence assisting the early detection of active pulmonary tuberculosis from chest X-rays: a population-based study</article-title>. <source>Front Mol Biosci.</source> (<year>2022</year>) <volume>9</volume>:<fpage>874475</fpage>. <pub-id pub-id-type="doi">10.3389/fmolb.2022.874475</pub-id><pub-id pub-id-type="pmid">35463963</pub-id></citation></ref>
<ref id="B19">
<label>19.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lee</surname> <given-names>JH</given-names></name> <name><surname>Park</surname> <given-names>S</given-names></name> <name><surname>Hwang</surname> <given-names>EJ</given-names></name> <name><surname>Goo</surname> <given-names>JM</given-names></name> <name><surname>Lee</surname> <given-names>WY</given-names></name> <name><surname>Lee</surname> <given-names>S</given-names></name> <etal/></person-group>. <article-title>Deep learning-based automated detection algorithm for active pulmonary tuberculosis on chest radiographs: diagnostic performance in systematic screening of asymptomatic individuals</article-title>. <source>Eur Radiol.</source> (<year>2021</year>) <volume>31</volume>:<fpage>1069</fpage>&#x02013;<lpage>80</lpage>. <pub-id pub-id-type="doi">10.1007/s00330-020-07219-4</pub-id><pub-id pub-id-type="pmid">32857202</pub-id></citation></ref>
<ref id="B20">
<label>20.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Heo</surname> <given-names>S-J</given-names></name> <name><surname>Kim</surname> <given-names>Y</given-names></name> <name><surname>Yun</surname> <given-names>S</given-names></name> <name><surname>Lim</surname> <given-names>S-S</given-names></name> <name><surname>Kim</surname> <given-names>J</given-names></name> <name><surname>Nam</surname> <given-names>C-M</given-names></name> <etal/></person-group>. <article-title>Deep learning algorithms with demographic information help to detect tuberculosis in chest radiographs in annual workers&#x00027; health examination data</article-title>. <source>Int J Environ Res Public Health</source>. (<year>2019</year>) <fpage>16</fpage>. <pub-id pub-id-type="doi">10.3390/ijerph16020250</pub-id><pub-id pub-id-type="pmid">30654560</pub-id></citation></ref>
<ref id="B21">
<label>21.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Pasa</surname> <given-names>F</given-names></name> <name><surname>Golkov</surname> <given-names>V</given-names></name> <name><surname>Pfeiffer</surname> <given-names>F</given-names></name> <name><surname>Cremers</surname> <given-names>D</given-names></name> <name><surname>Pfeiffer</surname> <given-names>D</given-names></name></person-group>. <article-title>Efficient deep network architectures for fast chest X-ray tuberculosis screening and visualization</article-title>. <source>Sci Rep.</source> (<year>2019</year>) <volume>9</volume>:<fpage>6268</fpage>. <pub-id pub-id-type="doi">10.1038/s41598-019-42557-4</pub-id><pub-id pub-id-type="pmid">31000728</pub-id></citation></ref>
<ref id="B22">
<label>22.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Rajaraman</surname> <given-names>S</given-names></name> <name><surname>Zamzmi</surname> <given-names>G</given-names></name> <name><surname>Folio</surname> <given-names>LR</given-names></name> <name><surname>Antani</surname> <given-names>S</given-names></name></person-group>. <article-title>Detecting tuberculosis-consistent findings in lateral chest X-rays using an ensemble of CNNs and vision transformers</article-title>. <source>Front Genet.</source> (<year>2022</year>) <volume>13</volume>:<fpage>864724</fpage>. <pub-id pub-id-type="doi">10.3389/fgene.2022.864724</pub-id><pub-id pub-id-type="pmid">35281798</pub-id></citation></ref>
<ref id="B23">
<label>23.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nafisah</surname> <given-names>SI</given-names></name> <name><surname>Muhammad</surname> <given-names>G</given-names></name></person-group>. <article-title>Tuberculosis detection in chest radiograph using convolutional neural network architecture and explainable artificial intelligence</article-title>. <source>Neural Comput Appl</source>. (<year>2022</year>) <volume>19</volume>:<fpage>1</fpage>&#x02013;<lpage>21</lpage>. <pub-id pub-id-type="doi">10.1007/s00521-022-07258-6</pub-id><pub-id pub-id-type="pmid">35462630</pub-id></citation></ref>
<ref id="B24">
<label>24.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Lee</surname> <given-names>S</given-names></name> <name><surname>Yim</surname> <given-names>J-J</given-names></name> <name><surname>Kwak</surname> <given-names>N</given-names></name> <name><surname>Lee</surname> <given-names>YJ</given-names></name> <name><surname>Lee</surname> <given-names>J-K</given-names></name> <name><surname>Lee</surname> <given-names>JY</given-names></name> <etal/></person-group>. <article-title>Deep learning to determine the activity of pulmonary tuberculosis on chest radiographs</article-title>. <source>Radiology.</source> (<year>2021</year>) <volume>301</volume>:<fpage>435</fpage>&#x02013;<lpage>42</lpage>. <pub-id pub-id-type="doi">10.1148/radiol.2021210063</pub-id><pub-id pub-id-type="pmid">34342505</pub-id></citation></ref>
<ref id="B25">
<label>25.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Codlin</surname> <given-names>AJ</given-names></name> <name><surname>Dao</surname> <given-names>TP</given-names></name> <name><surname>Vo</surname> <given-names>LNQ</given-names></name> <name><surname>Forse</surname> <given-names>RJ</given-names></name> <name><surname>Van Truong</surname> <given-names>V</given-names></name> <name><surname>Dang</surname> <given-names>HM</given-names></name> <etal/></person-group>. <article-title>Independent evaluation of 12 artificial intelligence solutions for the detection of tuberculosis</article-title>. <source>Sci Rep.</source> (<year>2021</year>) <volume>11</volume>:<fpage>23895</fpage>. <pub-id pub-id-type="doi">10.1038/s41598-021-03265-0</pub-id><pub-id pub-id-type="pmid">34903808</pub-id></citation></ref>
<ref id="B26">
<label>26.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Tavaziva</surname> <given-names>G</given-names></name> <name><surname>Harris</surname> <given-names>M</given-names></name> <name><surname>Abidi</surname> <given-names>SK</given-names></name> <name><surname>Geric</surname> <given-names>C</given-names></name> <name><surname>Breumninger</surname> <given-names>M</given-names></name> <name><surname>Dheda</surname> <given-names>K</given-names></name> <etal/></person-group>. <article-title>Chest X-ray analysis with deep learning-based software as a triage test for pulmonary tuberculosis: an individual patient data meta-analysis of diagnostic accuracy</article-title>. <source>Clin Infect Dis.</source> (<year>2022</year>) <volume>74</volume>:<fpage>1390</fpage>&#x02013;<lpage>400</lpage>. <pub-id pub-id-type="doi">10.1093/cid/ciab639</pub-id><pub-id pub-id-type="pmid">34286831</pub-id></citation></ref>
<ref id="B27">
<label>27.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Zhou</surname> <given-names>W</given-names></name> <name><surname>Cheng</surname> <given-names>G</given-names></name> <name><surname>Zhang</surname> <given-names>Z</given-names></name> <name><surname>Zhu</surname> <given-names>L</given-names></name> <name><surname>Jaeger</surname> <given-names>S</given-names></name> <name><surname>Lure</surname> <given-names>FYM</given-names></name> <etal/></person-group>. <article-title>Deep learning-based pulmonary tuberculosis automated detection on chest radiography: large-scale independent testing</article-title>. <source>Quant Imaging Med Surg.</source> (<year>2022</year>) <volume>12</volume>:<fpage>2344</fpage>&#x02013;<lpage>55</lpage>. <pub-id pub-id-type="doi">10.21037/qims-21-676</pub-id><pub-id pub-id-type="pmid">35371946</pub-id></citation></ref>
<ref id="B28">
<label>28.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Harris</surname> <given-names>M</given-names></name> <name><surname>Qi</surname> <given-names>A</given-names></name> <name><surname>Jeagal</surname> <given-names>L</given-names></name> <name><surname>Torabi</surname> <given-names>N</given-names></name> <name><surname>Menzies</surname> <given-names>D</given-names></name> <name><surname>Korobitsyn</surname> <given-names>A</given-names></name> <etal/></person-group>. <article-title>A systematic review of the diagnostic accuracy of artificial intelligence-based computer programs to analyze chest x-rays for pulmonary tuberculosis</article-title>. <source>PLoS ONE.</source> (<year>2019</year>) <volume>14</volume>:<fpage>e0221339</fpage>. <pub-id pub-id-type="doi">10.1371/journal.pone.0221339</pub-id><pub-id pub-id-type="pmid">31479448</pub-id></citation></ref>
<ref id="B29">
<label>29.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Tavaziva</surname> <given-names>G</given-names></name> <name><surname>Majidulla</surname> <given-names>A</given-names></name> <name><surname>Nazish</surname> <given-names>A</given-names></name> <name><surname>Saeed</surname> <given-names>S</given-names></name> <name><surname>Benedetti</surname> <given-names>A</given-names></name> <name><surname>Khan</surname> <given-names>AJ</given-names></name> <etal/></person-group>. <article-title>Diagnostic accuracy of a commercially available, deep learning-based chest X-ray interpretation software for detecting culture-confirmed pulmonary tuberculosis</article-title>. <source>Int J Infect Dis.</source> (<year>2022</year>) <volume>122</volume>:<fpage>15</fpage>&#x02013;<lpage>20</lpage>. <pub-id pub-id-type="doi">10.1016/j.ijid.2022.05.037</pub-id><pub-id pub-id-type="pmid">35597555</pub-id></citation></ref>
<ref id="B30">
<label>30.</label>
<citation citation-type="web"><person-group person-group-type="author"><collab>WHO</collab></person-group>. <source>High Priority Target Product Profiles For New Tuberculosis Diagnostics: Report of a Consensus Meeting</source>. Geneva: World Health Organization (<year>2014</year>). Available online at: <ext-link ext-link-type="uri" xlink:href="https://www.who.int/publications/i/item/WHO-HTM-TB-2014.18">https://www.who.int/publications/i/item/WHO-HTM-TB-2014.18</ext-link> (accessed January 3, 2023).</citation>
</ref>
<ref id="B31">
<label>31.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Khan</surname> <given-names>FA</given-names></name> <name><surname>Majidulla</surname> <given-names>A</given-names></name> <name><surname>Tavaziva</surname> <given-names>G</given-names></name> <name><surname>Nazish</surname> <given-names>A</given-names></name> <name><surname>Abidi</surname> <given-names>SK</given-names></name> <name><surname>Benedetti</surname> <given-names>A</given-names></name> <etal/></person-group>. <article-title>Chest x-ray analysis with deep learning-based software as a triage test for pulmonary tuberculosis: a prospective study of diagnostic accuracy for culture-confirmed disease</article-title>. <source>Lancet Digit Health.</source> (<year>2020</year>) <volume>2</volume>:<fpage>e573</fpage>&#x02013;<lpage>81</lpage>. <pub-id pub-id-type="doi">10.1016/S2589-7500(20)30221-1</pub-id><pub-id pub-id-type="pmid">33328086</pub-id></citation></ref>
<ref id="B32">
<label>32.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kulkarni</surname> <given-names>S</given-names></name> <name><surname>Jha</surname> <given-names>S</given-names></name></person-group>. <article-title>Artificial intelligence, radiology, and tuberculosis: a review</article-title>. <source>Acad Radiol.</source> (<year>2020</year>) <volume>27</volume>:<fpage>71</fpage>&#x02013;<lpage>5</lpage>. <pub-id pub-id-type="doi">10.1016/j.acra.2019.10.003</pub-id><pub-id pub-id-type="pmid">31759796</pub-id></citation></ref>
<ref id="B33">
<label>33.</label>
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Gerke</surname> <given-names>S</given-names></name> <name><surname>Minssen</surname> <given-names>T</given-names></name> <name><surname>Cohen</surname> <given-names>G</given-names></name></person-group>. <article-title>Chapter 12 - Ethical and legal challenges of artificial intelligence-driven healthcare</article-title>. <source>Artif Intell Health</source>. (<year>2020</year>) <fpage>295</fpage>&#x02013;<lpage>336</lpage>. <pub-id pub-id-type="doi">10.1016/B978-0-12-818438-7.00012-5</pub-id></citation>
</ref>
</ref-list> 
</back>
</article> 