<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article article-type="research-article" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Health Serv.</journal-id>
<journal-title>Frontiers in Health Services</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Health Serv.</abbrev-journal-title>
<issn pub-type="epub">2813-0146</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/frhs.2023.1142598</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Health Services</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Toward a more comprehensive understanding of organizational influences on implementation: the organization theory for implementation science framework</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" corresp="yes"><name><surname>Birken</surname><given-names>Sarah A.</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="corresp" rid="cor1">&#x002A;</xref><uri xlink:href="https://loop.frontiersin.org/people/1559057/overview"/></contrib>
<contrib contrib-type="author"><name><surname>Wagi</surname><given-names>Cheyenne R.</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1558959/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Peluso</surname><given-names>Alexandra G.</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref></contrib>
<contrib contrib-type="author"><name><surname>Kegler</surname><given-names>Michelle C.</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/2022455/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Baloh</surname><given-names>Jure</given-names></name>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/2285016/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Adsul</surname><given-names>Prajakta</given-names></name>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
<xref ref-type="aff" rid="aff5"><sup>5</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/982389/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Fernandez</surname><given-names>Maria E.</given-names></name>
<xref ref-type="aff" rid="aff6"><sup>6</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/386393/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Masud</surname><given-names>Manal</given-names></name>
<xref ref-type="aff" rid="aff7"><sup>7</sup></xref></contrib>
<contrib contrib-type="author"><name><surname>Huang</surname><given-names>Terry T-K</given-names></name>
<xref ref-type="aff" rid="aff8"><sup>8</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1232078/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Lee</surname><given-names>Matthew</given-names></name>
<xref ref-type="aff" rid="aff9"><sup>9</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1812810/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Wangen</surname><given-names>Mary</given-names></name>
<xref ref-type="aff" rid="aff10"><sup>10</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1801609/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Nilsen</surname><given-names>Per</given-names></name>
<xref ref-type="aff" rid="aff11"><sup>11</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1510436/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Bender</surname><given-names>Miriam</given-names></name>
<xref ref-type="aff" rid="aff12"><sup>12</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1710477/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Choy-Brown</surname><given-names>Mimi</given-names></name>
<xref ref-type="aff" rid="aff13"><sup>13</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1808011/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Ryan</surname><given-names>Grace</given-names></name>
<xref ref-type="aff" rid="aff14"><sup>14</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/2320946/overview" /></contrib>
<contrib contrib-type="author"><name><surname>Randazzo</surname><given-names>Aliza</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref></contrib>
<contrib contrib-type="author"><name><surname>Ko</surname><given-names>Linda K.</given-names></name>
<xref ref-type="aff" rid="aff7"><sup>7</sup></xref><uri xlink:href="https://loop.frontiersin.org/people/1308933/overview" /></contrib>
</contrib-group>
<aff id="aff1"><label><sup>1</sup></label><addr-line>Department of Implementation Science</addr-line>, <institution>Wake Forest University School of Medicine</institution>, <addr-line>Winston-Salem, NC</addr-line>, <country>United States</country></aff>
<aff id="aff2"><label><sup>2</sup></label><addr-line>Emory Prevention Research Center, Rollins School of Public Health</addr-line>, <institution>Emory University</institution>, <addr-line>Atlanta, GA</addr-line>, <country>United States</country></aff>
<aff id="aff3"><label><sup>3</sup></label><addr-line>College of Public Health, Department of Health Policy and Management</addr-line>, <institution>University of Arkansas for Medical Sciences</institution>, <addr-line>Little Rock, AR</addr-line>, <country>United States</country></aff>
<aff id="aff4"><label><sup>4</sup></label><addr-line>Department of Internal Medicine</addr-line>, <institution>University of New Mexico</institution>, <addr-line>Albuquerque, NM</addr-line>, <country>United States</country></aff>
<aff id="aff5"><label><sup>5</sup></label><addr-line>Comprehensive Cancer Center</addr-line>, <institution>University of New Mexico</institution>, <addr-line>Albuquerque, NM</addr-line>, <country>United States</country></aff>
<aff id="aff6"><label><sup>6</sup></label><addr-line>Center for Health Promotion and Prevention Research</addr-line>, <institution>University of Texas Health Science Center at Houston School of Public Health</institution>, <addr-line>Houston, TX</addr-line>, <country>United States</country></aff>
<aff id="aff7"><label><sup>7</sup></label><addr-line>Department of Health Systems and Population Health, School of Public Health</addr-line><institution>, University of Washington</institution>, <addr-line>Seattle, WA</addr-line>, <country>United States</country></aff>
<aff id="aff8"><label><sup>8</sup></label><addr-line>Center for Systems and Community Design and NYU-CUNY Prevention Research Center, Graduate School of Public Health and Health Policy</addr-line>, <institution>City University of New York</institution>, <addr-line>New York, NY</addr-line>, <country>United States</country></aff>
<aff id="aff9"><label><sup>9</sup></label><addr-line>Department of Population Health</addr-line>, <institution>New York University Grossman School of Medicine</institution>, <addr-line>New York, NY</addr-line>, <country>United States</country></aff>
<aff id="aff10"><label><sup>10</sup></label><addr-line>UNC Center for Health Promotion and Disease Prevention</addr-line>, <institution>University of North Carolina Chapel Hill</institution>, <addr-line>Chapel Hill, NC</addr-line>, <country>United States</country></aff>
<aff id="aff11"><label><sup>11</sup></label><addr-line>Department of Health, Medicine and Caring Sciences, Division of Society and Health</addr-line>, <institution>Link&#x00F6;ping University</institution>, <addr-line>Link&#x00F6;ping</addr-line>, <country>Sweden</country></aff>
<aff id="aff12"><label><sup>12</sup></label><addr-line>Sue &#x0026; Bill Gross School of Nursing</addr-line>, <institution>University of California Irvine</institution>, <addr-line>Irvine, CA</addr-line>, <country>United States</country></aff>
<aff id="aff13"><label><sup>13</sup></label><addr-line>College of Education and Human Development</addr-line>, <institution>School of Social Work, University of Minnesota</institution>, <addr-line>St. Paul, MN</addr-line>, <country>United States</country></aff>
<aff id="aff14"><label><sup>14</sup></label><addr-line>Department of Population and Quantitative Health Sciences</addr-line>, <institution>University of Massachusetts Chan Medical School</institution>, <addr-line>Worcester, MA</addr-line>, <country>United States</country></aff>
<author-notes>
<fn fn-type="edited-by"><p><bold>Edited by:</bold> Jose M Valderas, National University of Singapore, Singapore</p></fn>
<fn fn-type="edited-by"><p><bold>Reviewed by:</bold> Alexandra Ziemann, University of Bath, United Kingdom Nick Sevdalis, National University of Singapore, Singapore</p></fn>
<corresp id="cor1"><label>&#x002A;</label><bold>Correspondence:</bold> Sarah A. Birken <email>sbirken@wakehealth.edu</email></corresp>
<fn fn-type="other" id="fn001"><p>Abbreviations CFIR, consolidated framework for implementation research; CPCRN, cancer prevention and control research network; EPIS, exploration, preparation, implementation, sustainment; OTIS, organization theory for implementation science; TDF, theoretical domains framework.</p></fn>
</author-notes>
<pub-date pub-type="epub"><day>31</day><month>08</month><year>2023</year></pub-date>
<pub-date pub-type="collection"><year>2023</year></pub-date>
<volume>3</volume><elocation-id>1142598</elocation-id>
<history>
<date date-type="received"><day>11</day><month>01</month><year>2023</year></date>
<date date-type="accepted"><day>17</day><month>08</month><year>2023</year></date>
</history>
<permissions>
<copyright-statement>&#x00A9; 2023 Birken, Wagi, Peluso, Kegler, Baloh, Adsul, Fernandez, Masud, Huang, Lee, Wangen, Nilsen, Bender, Choy-Brown, Ryan, Randazzo and Ko.</copyright-statement>
<copyright-year>2023</copyright-year><copyright-holder>Birken, Wagi, Peluso, Kegler, Baloh, Adsul, Fernandez, Masud, Huang, Lee, Wangen, Nilsen, Bender, Choy-Brown, Ryan, Randazzo and Ko</copyright-holder><license license-type="open-access" xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution License (CC BY)</ext-link>. The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p></license>
</permissions>
<abstract><sec><title>Introduction</title>
<p>Implementation is influenced by factors beyond individual clinical settings. Nevertheless, implementation research often focuses on factors related to individual providers and practices, potentially due to limitations of available frameworks. Extant frameworks do not adequately capture the myriad organizational influences on implementation. Organization theories capture diverse organizational influences but remain underused in implementation science. To advance their use among implementation scientists, we distilled 70 constructs from nine organization theories identified in our previous work into theoretical domains in the Organization Theory for Implementation Science (OTIS) framework.</p>
</sec><sec><title>Methods</title>
<p>The process of distilling organization theory constructs into domains involved concept mapping and iterative consensus-building. First, we recruited organization and implementation scientists to participate in an online concept mapping exercise in which they sorted organization theory constructs into domains representing similar theoretical concepts. Multidimensional scaling and hierarchical cluster analyses were used to produce visual representations (clusters) of the relationships among constructs in concept maps. Second, to interpret concept maps, we engaged members of the Cancer Prevention and Control Research Network (CPCRN) OTIS workgroup in consensus-building discussions.</p>
</sec><sec><title>Results</title>
<p>Twenty-four experts participated in concept mapping. Based on resulting construct groupings&#x0027; coherence, OTIS workgroup members selected the 10-cluster solution (from options of 7&#x2013;13 clusters) and then reorganized clusters in consensus-building discussions to increase coherence. This process yielded six final OTIS domains: organizational characteristics (e.g., size; age); governance and operations (e.g., organizational and social subsystems); tasks and processes (e.g., technology cycles; excess capacity); knowledge and learning (e.g., tacit knowledge; sense making); characteristics of a population of organizations (e.g., isomorphism; selection pressure); and interorganizational relationships (e.g., dominance; interdependence).</p>
</sec><sec><title>Discussion</title>
<p>Organizational influences on implementation are poorly understood, in part due to the limitations of extant frameworks. To improve understanding of organizational influences on implementation, we distilled 70 constructs from nine organization theories into six domains. Applications of the OTIS framework will enhance understanding of organizational influences on implementation, promote theory-driven strategies for organizational change, improve understanding of mechanisms underlying relationships between OTIS constructs and implementation, and allow for framework refinement. Next steps include testing the OTIS framework in implementation research and adapting it for use among policymakers and practitioners.</p>
</sec>
</abstract>
<kwd-group>
<kwd>organization theory</kwd>
<kwd>implementation</kwd>
<kwd>determinant framework</kwd>
<kwd>concept mapping</kwd>
<kwd>consensus-building</kwd>
</kwd-group><counts>
<fig-count count="3"/>
<table-count count="2"/><equation-count count="0"/><ref-count count="32"/><page-count count="0"/><word-count count="0"/></counts><custom-meta-wrap><custom-meta><meta-name>section-at-acceptance</meta-name><meta-value>Implementation Science</meta-value></custom-meta></custom-meta-wrap>
</article-meta>
</front>
<body><sec id="s1" sec-type="intro"><title>Introduction</title>
<p>Individual healthcare providers&#x0027; behaviors are often constrained by factors that are beyond their own control (<xref ref-type="bibr" rid="B1">1</xref>). The assumption that all behaviors are largely under conscious control has taken a &#x201C;theoretical battering&#x201D; due to research showing the importance of non-conscious processes that operate in organizations (<xref ref-type="bibr" rid="B2">2</xref>). Research suggests that many healthcare provider behaviors that are repeatedly performed become non-reflective and more or less automatic (<xref ref-type="bibr" rid="B3">3</xref>). Individual behavior is also constrained by factors at collective levels (<xref ref-type="bibr" rid="B1">1</xref>). Collective levels include interpersonal (e.g., relations between healthcare providers), group (e.g., healthcare professionals providing care in a breast medical oncology practice), intraorganizational (e.g., hospital culture), and interorganizational (e.g., accreditation standards). Collective-level influences may also be largely non-conscious, having become internalized and taken for granted (e.g., norms and values of a professional culture) (<xref ref-type="bibr" rid="B1">1</xref>).</p>
<p>Various implementation determinant frameworks include factors at the organizational level. For example, the Consolidated Framework for Implementation Research (CFIR) and the Exploration, Preparation, Implementation, Sustainment (EPIS) Framework include inner setting (i.e., intraorganizational) and outer setting (i.e., interorganizational) domain (<xref ref-type="bibr" rid="B4">4</xref>, <xref ref-type="bibr" rid="B5">5</xref>). Domains are comprised of constructs (i.e., explanatory concepts that cannot be directly observed but can be inferred from observed data) (<xref ref-type="bibr" rid="B6">6</xref>). Organization-level domains include constructs such as <italic>structural characteristics</italic> (&#x201C;the social architecture, age, maturity, and size of an organization&#x201D;), <italic>cosmopolitanism</italic> (&#x201C;the degree to which an organization is networked with other external organizations&#x201D;), and funding (&#x201C;fiscal support provided by the system in which implementation occurs&#x201D;) (<xref ref-type="bibr" rid="B7">7</xref>&#x2013;<xref ref-type="bibr" rid="B9">9</xref>). The Theoretical Domains Framework (TDF) similarly includes the environmental context and resources domain, which includes constructs such as <italic>material resources</italic> and <italic>barriers and facilitators</italic> (<xref ref-type="bibr" rid="B10">10</xref>).</p>
<p>Commonly used implementation determinant frameworks encourage implementation scientists to consider organizational influences on implementation; however, the scope of organization-level constructs described in extant frameworks is limited. Furthermore, determinant frameworks often lack explanations of the mechanisms underlying the relationships between organization-level constructs and implementation. Extant frameworks&#x0027; limited scope impedes progress in implementation science by obscuring the influence of organization-level constructs that may drive implementation outcomes. A substantial body of work in industries other than healthcare provides evidence of the significant influence of organizational influences on implementation, pointing to high-leverage strategies to promote organizational change. Organization theory has been applied to educational and budgetary reform, elucidating the critical importance of addressing power dynamics among leadership and fostering positive change culture to facilitate implementation (<xref ref-type="bibr" rid="B11">11</xref>, <xref ref-type="bibr" rid="B12">12</xref>). In the non-profit industry, organization theory can be used to build capacity, assist with decision-making, narrow target populations, and clarify organizational needs (<xref ref-type="bibr" rid="B13">13</xref>).</p>
<p>Failing to account for the critical influence of organization-level constructs on implementation introduces omitted variable bias&#x2013;i.e., the faulty attribution of the influence of the omitted variable(s) to variables that were included (<xref ref-type="bibr" rid="B14">14</xref>). In the case of implementation research, this may amount to, for example, attributing the influence of organizational inertia (i.e., resistance to change) to a construct that is related but distinct (e.g., readiness for implementation) or an unrelated construct (e.g., individual provider motivation). The misattribution of omitted organization-level constructs to the constructs that extant implementation frameworks include has important implications for subsequent stages of implementation research, such as selecting and identifying strategies to target the constructs influencing implementation.</p>
<p>Many extant implementation determinant frameworks are conceptual frameworks, in that they offer a menu of constructs thought to influence implementation, but they do not address how change takes place or any causal mechanisms, which is critical for falsifying hypothesized relationships through empirical study (<xref ref-type="bibr" rid="B15">15</xref>). The ability to falsify hypothesized relationships between constructs depends on explanations of the mechanisms underlying relationships between constructs that are derived from theory (<xref ref-type="bibr" rid="B16">16</xref>). The constructs in conceptual frameworks such as the CFIR derive from a combination of theory and empirical studies. For example, the CFIR <italic>peer pressure</italic> construct derives from Institutional Theory, but <italic>patient needs and resources</italic> derives from a combination of empirical evidence and other conceptual frameworks rather than theory.</p>
<p>In contrast to conceptual frameworks, theoretical frameworks are based on theories, which propose mechanisms underlying the relationship between constructs and implementation. One commonly used theoretical framework in implementation science is the TDF. As a theoretical framework, the TDF can be used to identify mechanisms proposed in included theories; however, the TDF does not offer nuanced insight into organization-level influences on implementation. The TDF&#x0027;s environmental context and resources domains contains constructs that derive from several theories that are identified as organization theories; however, many of the included theories are not in fact organization theories (e.g., decision-making theory). As such, the TDF is limited in its contributions to understanding organization-level influences on implementation.</p>
<p>Organization theories provide explanations for the complex interactions within and between organizations and their context (environment, surrounding policies, cultural norms). These theories not only describe and explain these interactions, but can also be used to predict implementation outcomes based on contextual factors. Organization theories have the potential to explain how policies, institutions, funding, and workforce dynamics affect implementation outcomes (<xref ref-type="bibr" rid="B17">17</xref>). Organization theories have been historically used to an explanatory tool in fields of education, nonprofit organizations, management, and health services research, dating back to the 1950&#x2005;s (<xref ref-type="bibr" rid="B11">11</xref>&#x2013;<xref ref-type="bibr" rid="B13">13</xref>, <xref ref-type="bibr" rid="B18">18</xref>, <xref ref-type="bibr" rid="B19">19</xref>). These theories, while widely used and published, remain largely inaccessible outside of organization science. Organization theories provide their own inventory of constructs, which often require significant training to apply with fidelity.</p>
<p>To equip implementation scientists with understanding of a broader scope of organization-level constructs and their hypothesized influence on implementation, a comprehensive yet accessible framework of organizational influences on implementation is needed. In this paper, we describe the development of the Organization Theory for Implementation Science (OTIS) framework, which summarizes constructs from nine organization theories identified as relevant to implementation in preliminary studies (<xref ref-type="bibr" rid="B20">20</xref>). Our overarching goal is to increase implementation scientists&#x0027; familiarity with and conceptualization of the myriad organizational factors that influence implementation through mechanisms clearly articulated by organization theories.</p>
</sec>
<sec id="s2" sec-type="methods"><title>Materials and methods</title>
<p>We developed the OTIS framework using a combination of concept mapping and iterative consensus-building, with support for interpretation from members of the Cancer Prevention and Control Research Network (CPCRN) OTIS workgroup (<xref ref-type="bibr" rid="B21">21</xref>). CPCRN is a national network of academic, public health, and community partners whose work focuses on reducing the burden of cancer within specific workgroup and interest group projects. CPCRN OTIS workgroup members include investigators conducting research at the intersection of implementation science and cancer prevention and control. This study was approved by the Wake Forest University School of Medicine IRB (IRB00072134) on 6/2/21.</p>
<sec id="s2a"><title>Concept mapping</title>
<sec id="s2a1"><title>Recruitment and sampling</title>
<p>We used a purposive sampling approach to recruit approximately 25 scholars with expertise at the intersection of implementation and organization science to participate in an online concept mapping exercise via the Concept Systems Global MAX&#x2122; web platform (<xref ref-type="bibr" rid="B22">22</xref>). The premise of our sampling approach for the survey on organization theories of relevance to implementation science that provided the foundation for this study was that scholars with primary training in implementation and organization science had the knowledge required to identify organization theories with relevance to implementation science. For this study, we purposively included a more diverse group of scholars with implementation and organization expertise with the objective of generating a framework that would reflect the perspective of targeted users of the OTIS framework. Between 20 and 30 sorters have been found to maximize concept mapping fit consistency, yielding results similar to concept mapping by several hundred participants (<xref ref-type="bibr" rid="B23">23</xref>). Members of the study team identified potential participants from their respective professional networks in Canada, the UK, and the USA, as well as professional organizations such as the VA QUERI Implementation Research Group. We sent up to three emails offering potential participants a &#x0024;50 incentive to engage in the concept mapping exercise.</p>
</sec>
<sec id="s2a2"><title>Procedure</title>
<p>To identify conceptually distinct categories (domains) of constructs, we asked participants to sort virtual cards for each of the 70 constructs from nine organization theories relevant to implementation identified in previous work (<xref ref-type="bibr" rid="B20">20</xref>), accompanied by their definitions, into piles as they deemed appropriate. We then asked participants to name each pile. Participants could engage in the activities in the order of their choosing and could do so over multiple online sessions, at their convenience, until their responses were complete.</p>
</sec>
<sec id="s2a3"><title>Analysis</title>
<p>Data analysis involved the use of multidimensional scaling and hierarchical cluster analyses to produce visual representations of the relationships among the constructs (<xref ref-type="bibr" rid="B23">23</xref>). Specifically, multidimensional scaling was used to generate a point map depicting each of the constructs and the relationships between them based upon a summed square similarity matrix. Constructs frequently sorted together were placed closer together on the point map (<xref ref-type="bibr" rid="B23">23</xref>). Hierarchical cluster analysis was used to partition the point map into non-overlapping clusters (i.e., domains) (<xref ref-type="bibr" rid="B23">23</xref>). The Concept Systems Global Max&#x2122; suggested potential cluster labels based upon participant responses. Model fit was assessed using the stress value, an indicator of goodness of fit between the point map and the total similarity matrix. Cross-study syntheses of concept mapping studies have consistently found mean stress values of 0.28 (<xref ref-type="bibr" rid="B24">24</xref>). The stress value of the concept map represents goodness of fit of the configuration, demonstrating how close the solution is to the original groupings made by the participants. Lower stress values indicate a better fit than higher stress values (<xref ref-type="bibr" rid="B24">24</xref>).</p>
</sec>
</sec>
<sec id="s2b"><title>Consensus-building</title>
<sec id="s2b1"><title>Recruitment and sampling</title>
<p>We invited members of the CPCRN OTIS workgroup to review concept mapping results and provide feedback. All CPCRN OTIS workgroup members were eligible to participate.</p>
</sec>
<sec id="s2b2"><title>Procedure</title>
<p>To build upon the results of the concept mapping activity, CPCRN OTIS workgroup members provided their expertise in reviewing results of the concept mapping activity. Participation occurred over the course of three months, beginning with the CPCRN Annual Meeting and continuing through regular workgroup meetings.</p>
</sec>
<sec id="s2b3"><title>Analysis</title>
<p>During a hybrid meeting held in May 2022, CPCRN OTIS workgroup members (6 in-person; 4 virtual) considered a range of potential cluster solutions, ranging from seven to 10 clusters, to determine which solution best suited the purposes of the current study. Each member identified the cluster map that they deemed most conceptually clear based on their knowledge of the field. The group then discussed their choices and worked to reach consensus on what the group believed to provide the most conceptually clear map and moved constructs to clusters that provided the best fit. The group also discussed and altered the automatically generated labels created by Global Max&#x2122;. Following the initial analysis, two workgroup members reviewed notes, and a third member reconciled discrepancies, suggesting additional shifts of constructs among clusters. Finally, the lead investigator revised clusters based on extensive knowledge of organization theory. The resulting clusters were again reviewed, revised, and approved by CPCRN OTIS workgroup members during workgroup meetings until a consensus was reached.</p>
</sec>
</sec>
</sec>
<sec id="s3" sec-type="results"><title>Results</title>
<sec id="s3a"><title>Concept mapping</title>
<p>Twenty-four scholars participated in the concept mapping exercise. Participant demographics are described in <xref ref-type="table" rid="T1">Table&#x00A0;1</xref>. Most participants (84&#x0025;) held a PhD degree and worked in an academic institution (80&#x0025;). The plurality of participants had multidisciplinary training (29&#x0025;), and the majority had multidisciplinary expertise (57&#x0025;).</p>
<table-wrap id="T1" position="float"><label>Table 1</label>
<caption><p>Concept mapping participant demographics.</p></caption>
<table frame="hsides" rules="groups">
<colgroup>
<col align="left"/>
<col align="center"/>
</colgroup>
<thead>
<tr>
<th valign="top" align="left">Characteristic</th>
<th valign="top" align="center">Percent Total</th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left"/>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;25</td>
</tr>
<tr>
<td valign="top" align="left">Education</td>
<td valign="top" align="center"><italic>N</italic> (&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;PhD</td>
<td valign="top" align="center">21 (84.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;MD</td>
<td valign="top" align="center">1 (4.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">3 (12.0)</td>
</tr>
<tr>
<td valign="top" align="left">Academic Title</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;24</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Assistant professor</td>
<td valign="top" align="center">6 (25.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Post-doctoral fellow</td>
<td valign="top" align="center">5 (20.8)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Professor</td>
<td valign="top" align="center">5 (20.8)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Associate professor</td>
<td valign="top" align="center">4 (16.7)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">2 (8.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Did not respond</td>
<td valign="top" align="center">2 (8.3)</td>
</tr>
<tr>
<td valign="top" align="left">Organization</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;24</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;University</td>
<td valign="top" align="center">19 (80.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Government Agency</td>
<td valign="top" align="center">2 (8.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">2 (8.0)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Research Institute</td>
<td valign="top" align="center">1 (4.0)</td>
</tr>
<tr>
<td valign="top" align="left">Field, Specialty, or Discipline</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;24</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Multidisciplinary</td>
<td valign="top" align="center">7 (29.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health policy and management</td>
<td valign="top" align="center">3 (12.5)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Implementation science</td>
<td valign="top" align="center">3 (12.5)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Social work</td>
<td valign="top" align="center">2 (8.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Behavioral science/public health</td>
<td valign="top" align="center">2 (8.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health care management</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health services research</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Sociology</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Clinical psychologist</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Behavioral science</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Healthcare</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Organizational behavior</td>
<td valign="top" align="center">1 (4.2)</td>
</tr>
<tr>
<td valign="top" align="left">Content Expertise</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;23</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Multidisciplinary</td>
<td valign="top" align="center">13 (56.5)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Cancer (prevention, control, survivorship)</td>
<td valign="top" align="center">4 (17.4)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Mental health</td>
<td valign="top" align="center">2 (8.7)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health services research</td>
<td valign="top" align="center">1 (4.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health and social care</td>
<td valign="top" align="center">1 (4.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Digital technology in healthcare</td>
<td valign="top" align="center">1 (4.3)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Maternal and child health</td>
<td valign="top" align="center">1 (4.3)</td>
</tr>
</tbody>
</table>
</table-wrap>
<p>All 24 participants completed the sorting exercise. We confirmed that sorts were valid by checking 5 participants&#x0027; responses to ensure that criteria were sorted into generally logical categories. The stress value was 0.32, demonstrating poor fit. Our consensus-building process was designed to address poor fit by developing a more coherent solution.</p>
</sec>
<sec id="s3b"><title>Consensus-building</title>
<p>Workgroup members (<italic>n</italic>&#x2009;&#x003D;&#x2009;18) participated throughout in-person and virtual discussion sessions. Participant demographics are described in <xref ref-type="table" rid="T2">Table&#x00A0;2</xref>. The concept mapping software produced multiple cluster options, ranging from 7 to 13 clusters. CPCRN OTIS workgroup members narrowed the clusters to 8&#x2013;10 (<xref ref-type="fig" rid="F1">Figures&#x00A0;1</xref>&#x2013;<xref ref-type="fig" rid="F3">3</xref>), ultimately selecting the 10-cluster solution to use as a starting point for the consensus-building process. Workgroup members then reorganized the clusters to increase coherence, yielding six final OTIS framework domains: organizational characteristics; governance and operations; characteristics of a population of organizations; tasks and processes; knowledge and learning; and interorganizational relationships. The final solution was informed by the 10-cluster solution. A total of 70 constructs are organized across the six domains. <xref ref-type="sec" rid="s11">Supplementary File S1</xref> organizes constructs by domain and includes brief descriptions/definitions for each, as well as the source theory.</p>
<table-wrap id="T2" position="float"><label>Table 2</label>
<caption><p>Consensus gathering participant demographics.</p></caption>
<table frame="hsides" rules="groups">
<colgroup>
<col align="left"/>
<col align="center"/>
</colgroup>
<thead>
<tr>
<th valign="top" align="left">Characteristic</th>
<th valign="top" align="center">Total</th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left"/>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;18</td>
</tr>
<tr>
<td valign="top" align="left">Education</td>
<td valign="top" align="center"><italic>N</italic> (&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;PhD</td>
<td valign="top" align="center">13 (72.2&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;MD</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">5 (27.8&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">Academic Title</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;18</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Assistant professor</td>
<td valign="top" align="center">4 (22.2&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Post-doctoral fellow</td>
<td valign="top" align="center">1 (5.6&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Professor</td>
<td valign="top" align="center">5 (27.8&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Associate professor</td>
<td valign="top" align="center">3 (16.7&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">5 (27.8&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">Organization</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;18</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;University</td>
<td valign="top" align="center">18 (100&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Government Agency</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Other</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Research Institute</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">Field, Specialty, or Discipline</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;18</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Multidisciplinary</td>
<td valign="top" align="center">5 (27.8&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health policy and management</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Implementation science</td>
<td valign="top" align="center">7 (38.9&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Social work</td>
<td valign="top" align="center">1 (5.6&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Behavioral science/public health</td>
<td valign="top" align="center">5 (27.8&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health care management</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health services research</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Sociology</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Clinical psychologist</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Behavioral science</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Healthcare</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Organizational behavior</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">Content Expertise</td>
<td valign="top" align="center"><italic>N</italic>&#x2009;&#x003D;&#x2009;18</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Multidisciplinary</td>
<td valign="top" align="center">6 (33.3&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Cancer (prevention, control, survivorship)</td>
<td valign="top" align="center">8 (44.4&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Mental health</td>
<td valign="top" align="center">1 (5.6&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health services research</td>
<td valign="top" align="center">3 (16.7&#x0025;)</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Health and social care</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Digital technology in healthcare</td>
<td valign="top" align="center">0</td>
</tr>
<tr>
<td valign="top" align="left">&#x2003;Maternal and child health</td>
<td valign="top" align="center">0</td>
</tr>
</tbody>
</table>
</table-wrap>
<fig id="F1" position="float"><label>Figure 1</label>
<caption><p>Eight-cluster concept map solution.</p></caption>
<graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="frhs-03-1142598-g001.tif"/>
</fig>
<fig id="F2" position="float"><label>Figure 2</label>
<caption><p>Nine-cluster concept map solution.</p></caption>
<graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="frhs-03-1142598-g002.tif"/>
</fig>
<fig id="F3" position="float"><label>Figure 3</label>
<caption><p>Ten-cluster concept map solution and constructs.</p></caption>
<graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="frhs-03-1142598-g003.tif"/>
</fig>
<p><italic>Organizational Characteristics</italic> (number of constructs&#x2009;&#x003D;&#x2009;6) refers to the features of an organization that may predispose it to governance, operations, interorganizational relationships, etc. Included constructs relate to change dynamics (e.g., inertia; adaptability), orientation to operations (e.g., professionalization; specialization); and dominance within its population [e.g., age; size (i.e., indicators of viability; on average, older, larger organizations are more likely to survive than younger, smaller organizations)].</p>
<p><italic>Governance and Operations</italic> (<italic>n</italic>&#x2009;&#x003D;&#x2009;7) refer to the rules and operating procedures that govern an organization. An organization&#x0027;s rules and operating procedures may be established explicitly (e.g., intentionally, by a governing body) or implicitly (e.g., passively, through repeated operations). Constructs include approaches to operating (e.g., governance structure; internal arrangements) and structures that characterize an organization&#x0027;s operations (e.g., internal arrangements; feedback loops).</p>
<p><italic>Characteristics of a Population of Organizations</italic> (<italic>n</italic>&#x2009;&#x003D;&#x2009;16) refer to the features of a group of organizations of which the referent organization is a member (<xref ref-type="bibr" rid="B25">25</xref>). The institutions that comprise an organization&#x0027;s population may vary depending on the objective or problem in question. That is, a referent organization may be part of several populations. For example, a hospital&#x0027;s population may be defined as local healthcare organizations with respect to competition for physicians and patients, but with respect to adherence to government regulations, a hospital&#x0027;s population may be defined as all of the country&#x0027;s hospitals. Constructs included in the <italic>Characteristics of a Population of Organizations</italic> domain are features of the population as a whole rather than features of the organizations that comprise the population. Constructs relate to change within the population (e.g., dynamism; stability); competition (e.g., competition; selection pressure); variation within the population (e.g., isomorphism; spatial variation); and availability of resources (e.g., munificence; constraint).</p>
<p><italic>Tasks and Processes</italic> (<italic>n</italic>&#x2009;&#x003D;&#x2009;16) characterize the work that an organization pursues and the conditions that influence its approach to accomplishing the work. Included constructs refer to features of the processes used to accomplish tasks (e.g., un/programmed coordination task structure; transaction costs); features of the environment in which tasks are accomplished (e.g., dependence; excess capacity); and features of the task (e.g., frequency of transactions; technology cycles).</p>
<p><italic>Knowledge and Learning</italic> (<italic>n</italic>&#x2009;&#x003D;&#x2009;5) refers to the information available to an organization in pursuing its goals and the processes used to acquire the information. Included constructs relate to characteristics of knowledge (e.g., tacit and implicit knowledge) and approaches to acquiring knowledge (e.g., learning (sub)processes; sense making).</p>
<p><italic>Interorganizational Relationships</italic> (<italic>n</italic>&#x2009;&#x003D;&#x2009;20) refer to characteristics of the interactions that an organization has with other institutions. In contrast to the <italic>Characteristics of a Population of Organizations</italic> domain, which refers to features of a population of organizations as a whole, the <italic>Interorganizational Relationships</italic> domain characterizes communal (e.g., communication) or exchange (e.g., monetary or other resource exchange) interactions (<xref ref-type="bibr" rid="B26">26</xref>). Included constructs characterize an organization&#x0027;s dependence on other institutions (e.g., interdependence; community interdependence); the pressure that organizations exert on each other (e.g., normative, mimetic, and coercive pressure; dominance; power).</p>
</sec>
</sec>
<sec id="s4" sec-type="discussion"><title>Discussion</title>
<p>This study describes how we created the OTIS framework to increase implementation scientists&#x0027; familiarity with and conceptualization of the diverse set of organizational influences on implementation. Increasing implementation scientists&#x0027; conceptualization of organizational influences may contribute to more comprehensive understanding of the key drivers of implementation and, in turn, our ability to identify and select strategies to accelerate the translation of evidence into practice, as found in other industries, such as business and education (<xref ref-type="bibr" rid="B11">11</xref>, <xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B17">17</xref>, <xref ref-type="bibr" rid="B18">18</xref>). Our efforts yielded six conceptually distinct domains, encompassing 70 constructs from nine organization theories with relevance to implementation. Distilling many constructs from several theories into a limited number of domains limits the burden on implementation scientists to account for the vast array of potentially important organizational influences on implementation. The six domains that we identified in this study reflect concepts that are central to organization theory, including power, structure, autonomy, control (<xref ref-type="bibr" rid="B20">20</xref>), but which are less commonly addressed in implementation science. The concepts reflected in the OTIS framework offer perspective on key questions in implementation science, such as how and why organizations adopt, implement, and sustain evidence-based practices&#x2014;or resist doing so.</p>
<p>The OTIS framework considerably expands upon existing implementation determinant frameworks&#x0027; conceptualization of organizational influences on implementation. OTIS includes constructs such as <italic>specialization</italic>, which is not explicitly captured in the CFIR or EPIS frameworks, but which may influence the decision to adopt an evidence-based practice. For example, a study of determinants of low-value use of computed tomography to evaluate microscopic hematuria found that, while urologists&#x0027; evaluation practices changed following the American Urological Association&#x0027;s revised guidelines, primary care providers&#x0027; evaluation practices often went unchanged, highlighting the need to tailor strategies for the various specialties involved in implementation (<xref ref-type="bibr" rid="B27">27</xref>, <xref ref-type="bibr" rid="B28">28</xref>). OTIS greatly expands upon the TDF&#x0027;s organization-level <italic>environmental context and resources</italic> domain with more nuanced domains, such as <italic>interorganizational relationships</italic>. Future efforts should systematically map OTIS onto extant determinant frameworks to clearly articulate OTIS&#x0027;s unique contribution. For example, OTIS&#x0027;s Governance and Operations and Tasks and Processes domains include several constructs that may add critical nuance to EPIS&#x0027;s Funding/Contracting construct. Before systematic mapping of OTIS onto extant frameworks, OTIS may be used in its current form in conjunction with other frameworks, such as the CFIR and TDF, which are increasingly used in combination and already capture intra-organizational constructs, such as climate and leadership (<xref ref-type="bibr" rid="B28">28</xref>). For example, some OTIS domains or constructs that appear not to be captured in CFIR (e.g., tasks and processes; stability of the population of organizations; normative pressures) could be included in implementation determinant studies.</p>
<p>OTIS also expands upon commonly used implementation frameworks by allowing users to access organization theories articulate the mechanisms underlying relationships between included constructs and implementation. For example, the EPIS framework identifies sociopolitical influences on implementation (e.g., legislation; monitoring and review); however, EPIS does not articulate <italic>how</italic> or <italic>why</italic> these constructs influence implementation. In contrast, OTIS&#x0027;s basis in theory allows users to identify hypothesized relationships between included constructs and implementation, as clearly articulated in publicly available OTIS abstraction forms (<xref ref-type="bibr" rid="B28">28</xref>). Specifically, users may consult the <italic>propositions</italic> section of OTIS abstraction forms to identify mechanisms underlying included constructs. For example, OTIS describes how coercive influences of governments and accrediting bodies exert normative pressure (Interorganizational Relationships domain) on healthcare organizations to comply with legislation and monitoring by virtue of organizations&#x0027; dependence on these governing bodies for permission to operate. Therefore, OTIS could be used in conjunction with extant frameworks to explain the mechanisms underlying constructs&#x0027; influence on implementation (<xref ref-type="bibr" rid="B29">29</xref>). Clearly articulated mechanisms are critical for identifying strategies that are best-suited to influence the construct identified as influencing implementation.</p>
<p>Members of the CPCRN OTIS workgroup are currently applying the OTIS framework in the following projects: Project 1 a) tests the conceptual validity and applicability of the OTIS framework in community oncology practices and b) develops, tests, and disseminates tools using OTIS in implementation research, including a qualitative interview guide and codebook. Project 2 is an American Society of Clinical Oncology collaborative study on Sexual Orientation and Gender Identity data collection. OTIS will be used in this project to a) reanalyze data that have been previously analyzed using the CFIR, and b) compare results between CFIR and OTIS findings. Project 3 applies OTIS to a CDC-funded U01 cooperative agreement to reduce health inequities for cancer survivors in the District of Columbia. OTIS will be used to a) to build community coalitions of approximately 10 organizations to improve infrastructure and communication and b) to think consider power dynamics and the elimination of disparities and health inequalities. Some limitations of our study should be noted. Concept mapping requires participants to have pre-existing knowledge and experience with the topic they are mapping, limiting the pool of potential participants. There is a limited population of researchers with the required familiarity of organization theories and implementation science to participate in concept mapping. As a result, our purposive sampling approach was necessary to increase the likelihood that participants would understand included constructs enough to sort and rate them. However, it is possible that participants lacking refined expertise in implementation or organization science would have valuable perspective on included constructs. For example, hospital administrators may lack fluency in the terminology included in organization theories, but they may have unique insight into how, for example, normative pressure from professional organizations influences implementation. Future work should refine the language used in OTIS to increase its accessibility to an audience without expertise in organization science. Additionally, the clusters that Global Max generated, in many cases, lacked coherence as indicated by the stress value of 0.32, suggesting variation in concept mapping participants&#x0027; interpretation of the constructs and their relationships. To address this concern, OTIS workgroup members used their expertise to reorganize many clusters in our consensus-building process, potentially suggesting the limited utility of concept mapping for developing the framework. We view the OTIS framework as a living document to be revised through application. For example, implementation scientists may find through qualitative interview data collection that study participants describe OTIS constructs in combinations not reflected in the domains identified in this study. Future iterations of the OTIS framework will be revised to reflect empirical evidence.</p>
<p>Despite these limitations and the need for continued development, OTIS may be used in its current form in implementation research. OTIS could be used to inform data collection or analysis. For example, OTIS could be used to develop guides for interviews with cancer program leadership to understand the potential influence of participation in quality improvement networks, professional norms, and the ability to recruit providers influence compliance with cancer program accreditation standards (<xref ref-type="bibr" rid="B30">30</xref>). We plan to use OTIS to analyze data that were previously collected regarding factors influencing cancer programs&#x0027; implementation of exercise interventions. In each of these cases, OTIS offers researchers the tools necessary to understand the mechanisms underlying factors that influence implementation, pointing toward strategies to facilitate implementation (e.g., strengthening or reorganizing quality improvement networks to support compliance with accreditation standards).</p>
</sec>
<sec id="s5" sec-type="conclusions"><title>Conclusions</title>
<p>We distilled 70 constructs from nine organization theories into six domains in the OTIS framework. The OTIS framework has several potential benefits. First, OTIS may enhance implementation scientists&#x0027; consideration of organization-level constructs, which to date has been insufficient (<xref ref-type="bibr" rid="B17">17</xref>). Second, OTIS adds nuance to relatively limited conceptualizations of organizational influences in extant implementation determinant frameworks, such as the CFIR, EPIS, and TDF. Third, OTIS may increase the use of theories in implementation science. Evidence suggests that the use of theories, models, and frameworks in implementation science is inconsistent and often inappropriate (<xref ref-type="bibr" rid="B31">31</xref>). Unlike conceptual frameworks, which offer a menu of constructs thought to influence implementation, theoretical frameworks including OTIS are based on theories, which propose mechanisms underlying the relationship between constructs and implementation. OTIS links implementation scientists to theories that may explain the phenomena underlying complex implementation problems, such as slow uptake or poor sustainment. Future efforts should include expanding extant frameworks with OTIS&#x0027;s unique domains and constructs; refining OTIS&#x0027;s language to increase its accessibility to an audience without expertise in organization science; and revising OTIS to reflect empirical evidence.</p>
</sec>
</body>
<back>
<sec id="s6" sec-type="data-availability"><title>Data availability statement</title>
<p>The raw data supporting the conclusions of this article will be made available by the authors, without undue reservation by reasonable request to the corresponding author.</p>
</sec>
<sec id="s7" sec-type="ethics-statement"><title>Ethics statement</title>
<p>This study was approved by the Wake Forest University School of Medicine IRB (IRB00072134) on 6/2/21. Written informed consent for participation was not required for this study in accordance with the national legislation and the institutional requirements.</p>
</sec>
<ack><title>Acknowledgments</title>
<p>We would like to thank all of those who took part in the concept mapping activities. Additionally, we would like to thank Jennifer Leeman and the members of the CPCRN OTIS workgroup for their contributions.</p>
<p>Some of the data analyzed during this study have been previously published and are available from (<xref ref-type="bibr" rid="B20">20</xref>).</p>
</ack>
<sec id="s8" sec-type="author-contributions"><title>Author contributions</title>
<p>SB: contributed to the study conception and design, drafted the manuscript, and participated in the data collection, analysis, and interpretation of results; CW and AP: drafted the manuscript and participated in the data collection, analysis, and interpretation of results; MK, JB, PA, MF, MM, TH, ML, MW, PN, MB, MC-B, GR, and AR: participated in the analysis and interpretation of concept mapping results; LK contributed to the study conception and design and participated in the data collection, analysis, and interpretation of results. All authors contributed to the article and approved the submitted version.</p>
</sec>
<sec id="s9" sec-type="funding-information"><title>Funding</title>
<p>This publication was supported by the Centers for Disease Control and Prevention of the U.S. Department of Health and Human Services (HHS) as part of a financial assistance award with 100 percent funded by CDC/HHS (Cooperative Agreement Number U48 DP006400). The contents are those of the author(s) and do not necessarily represent the official views of, nor an endorsement, by CDC/HHS, or the U.S. Government.</p>
</sec>
<sec id="s10" sec-type="COI-statement"><title>Conflict of interest</title>
<p>The authors PN, TH, MEF, and PA declared that they were an editorial board member of Frontiers, at the time of submission. This had no impact on the peer review process and the final decision.</p>
<p>The remaining authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec id="s12" sec-type="disclaimer"><title>Publisher&#x0027;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<sec id="s11" sec-type="supplementary-material"><title>Supplementary material</title>
<p>The Supplementary Material for this article can be found online at: <ext-link ext-link-type="uri" xlink:href="https://www.frontiersin.org/articles/10.3389/frhs.2023.1142598/full#supplementary-material">https://www.frontiersin.org/articles/10.3389/frhs.2023.1142598/full&#x0023;supplementary-material</ext-link></p>
<supplementary-material id="SD1" content-type="local-data">
<media mimetype="application" mime-subtype="vnd.openxmlformats-officedocument.wordprocessingml.document" xlink:href="Datasheet1.docx"/></supplementary-material>
</sec>
<ref-list><title>References</title>
<ref id="B1"><label>1.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nilsen</surname><given-names>P</given-names></name><name><surname>Potthoff</surname><given-names>S</given-names></name><name><surname>Birken</surname><given-names>SA</given-names></name></person-group>. <article-title>Conceptualising four categories of behaviours: implications for implementation strategies to achieve behaviour change</article-title>. <source>Frontiers in Health Services</source>. (<year>2022</year>) <volume>1</volume>:<fpage>795144</fpage>. <pub-id pub-id-type="doi">10.3389/frhs.2021.795144</pub-id><pub-id pub-id-type="pmid">36926485</pub-id></citation></ref>
<ref id="B2"><label>2.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Greenwald</surname><given-names>AG</given-names></name><name><surname>Krieger</surname><given-names>LH</given-names></name></person-group>. <article-title>Implicit bias: scientific foundations</article-title>. <source>Calif Law Rev</source>. (<year>2006</year>) <volume>94</volume>(<issue>4</issue>):<fpage>945</fpage>&#x2013;<lpage>67</lpage>. <pub-id pub-id-type="doi">10.2307/20439056</pub-id></citation></ref>
<ref id="B3"><label>3.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Potthoff</surname><given-names>S</given-names></name><name><surname>Kwasnicka</surname><given-names>D</given-names></name><name><surname>Avery</surname><given-names>L</given-names></name><name><surname>Finch</surname><given-names>T</given-names></name><name><surname>Gardner</surname><given-names>B</given-names></name><name><surname>Hankonen</surname><given-names>N</given-names></name><etal/></person-group> <article-title>Changing healthcare professionals&#x2019; non-reflective processes to improve the quality of care</article-title>. <source>Soc Sci Med</source>. (<year>2022</year>) <volume>298</volume>:<fpage>114840</fpage>. <pub-id pub-id-type="doi">10.1016/j.socscimed.2022.114840</pub-id><pub-id pub-id-type="pmid">35287065</pub-id></citation></ref>
<ref id="B4"><label>4.</label><citation citation-type="other"><collab>EPIS FRAMEWORK</collab><comment>. Available at:</comment> <ext-link ext-link-type="uri" xlink:href="https://episframework.com/">https://episframework.com/</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B5"><label>5.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Damschroder</surname><given-names>LJ</given-names></name><name><surname>Aron</surname><given-names>DC</given-names></name><name><surname>Keith</surname><given-names>RE</given-names></name><name><surname>Kirsh</surname><given-names>SR</given-names></name><name><surname>Alexander</surname><given-names>JA</given-names></name><name><surname>Lowery</surname><given-names>JC</given-names></name></person-group>. <article-title>Fostering implementation of health services research findings into practice: a consolidated framework for advancing implementation science</article-title>. <source>Implement Sci</source>. (<year>2009</year>) <volume>4</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>15</lpage>. <pub-id pub-id-type="doi">10.1186/1748-5908-4-50</pub-id><pub-id pub-id-type="pmid">19123945</pub-id></citation></ref>
<ref id="B6"><label>6.</label><citation citation-type="other"><collab>Association AP</collab>. <comment>APA Dictionary of Psychology - Theoretical Construct (2022). Available at:</comment> <ext-link ext-link-type="uri" xlink:href="https://dictionary.apa.org/theoretical-construct">https://dictionary.apa.org/theoretical-construct</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B7"><label>7.</label><citation citation-type="other"><person-group person-group-type="author"><name><surname>Smith</surname><given-names>LR</given-names></name><name><surname>Ashok</surname><given-names>M</given-names></name><name><surname>Dy</surname><given-names>SM</given-names></name><name><surname>Wines</surname><given-names>RC</given-names></name><name><surname>Teixeira-Poit</surname><given-names>S</given-names></name></person-group>. <comment>Contextual frameworks for research on the implementation of complex system interventions (2014)</comment>.</citation></ref>
<ref id="B8"><label>8.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Greenhalgh</surname><given-names>T</given-names></name><name><surname>Robert</surname><given-names>G</given-names></name><name><surname>Macfarlane</surname><given-names>F</given-names></name><name><surname>Bate</surname><given-names>P</given-names></name><name><surname>Kyriakidou</surname><given-names>O</given-names></name></person-group>. <article-title>Diffusion of innovations in service organizations: systematic review and recommendations</article-title>. <source>Milbank Q</source>. (<year>2004</year>) <volume>82</volume>(<issue>4</issue>):<fpage>581</fpage>&#x2013;<lpage>629</lpage>. <pub-id pub-id-type="doi">10.1111/j.0887-378X.2004.00325.x</pub-id><pub-id pub-id-type="pmid">15595944</pub-id></citation></ref>
<ref id="B9"><label>9.</label><citation citation-type="other"><collab>EPIS Framework. Outer context</collab>. Available at: <ext-link ext-link-type="uri" xlink:href="https://episframework.com/service-environment-1#:~:text=Funding,system%20in%20which%20implementation%20occurs">https://episframework.com/service-environment-1#:&#x02DC;:text&#x003D;Funding,system&#x0025;20in&#x0025;20which&#x0025;20implementation&#x0025;20occurs</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B10"><label>10.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Atkins</surname><given-names>L</given-names></name><name><surname>Francis</surname><given-names>J</given-names></name><name><surname>Islam</surname><given-names>R</given-names></name><name><surname>O&#x2019;Connor</surname><given-names>D</given-names></name><name><surname>Patey</surname><given-names>A</given-names></name><name><surname>Ivers</surname><given-names>N</given-names></name><etal/></person-group> <article-title>A guide to using the theoretical domains framework of behaviour change to investigate implementation problems</article-title>. <source>Implement Sci</source>. (<year>2017</year>) <volume>12</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>18</lpage>. <pub-id pub-id-type="doi">10.1186/s13012-017-0605-9</pub-id><pub-id pub-id-type="pmid">28057027</pub-id></citation></ref>
<ref id="B11"><label>11.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bonner</surname><given-names>M</given-names></name><name><surname>Koch</surname><given-names>T</given-names></name><name><surname>Langmeyer</surname><given-names>D</given-names></name></person-group>. <article-title>Organizational theory applied to school reform: a critical analysis</article-title>. <source>Sch Psychol Int</source>. (<year>2004</year>) <volume>25</volume>(<issue>4</issue>):<fpage>455</fpage>&#x2013;<lpage>71</lpage>. <pub-id pub-id-type="doi">10.1177/0143034304048779</pub-id></citation></ref>
<ref id="B12"><label>12.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Forrester</surname><given-names>JP</given-names></name><name><surname>Adams</surname><given-names>GB</given-names></name></person-group>. <article-title>Budgetary reform through organizational learning: toward an organizational theory of budgeting</article-title>. <source>Adm Soc</source>. (<year>1997</year>) <volume>28</volume>(<issue>4</issue>):<fpage>466</fpage>&#x2013;<lpage>88</lpage>. <pub-id pub-id-type="doi">10.1177/009539979702800403</pub-id></citation></ref>
<ref id="B13"><label>13.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hunter</surname><given-names>DE</given-names></name></person-group>. <article-title>Using a theory of change approach to build organizational strength, capacity and sustainability with not-for-profit organizations in the human services sector</article-title>. <source>Eval Program Plann</source>. (<year>2006</year>) <volume>29</volume>(<issue>2</issue>):<fpage>193</fpage>&#x2013;<lpage>200</lpage>. <pub-id pub-id-type="doi">10.1016/j.evalprogplan.2005.10.003</pub-id></citation></ref>
<ref id="B14"><label>14.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Trochim</surname><given-names>W</given-names></name><name><surname>Spiegelman</surname><given-names>C</given-names></name></person-group>. <article-title>The relative assignment variable approach to selection bias in pretest-posttest group designs</article-title>. <source>Proceedings of the Survey Research Section</source>. (<year>1980</year>) <volume>1</volume>:<fpage>376</fpage>&#x2013;<lpage>80</lpage>.</citation></ref>
<ref id="B15"><label>15.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Nilsen</surname><given-names>P</given-names></name></person-group>. <article-title>Making sense of implementation theories, models, and frameworks</article-title>. <source>Implement Sci</source>. (<year>2020</year>) <volume>30</volume>:<fpage>53</fpage>&#x2013;<lpage>79</lpage>. <pub-id pub-id-type="doi">10.1007/978-3-030-03874-8_3</pub-id></citation></ref>
<ref id="B16"><label>16.</label><citation citation-type="other"><person-group person-group-type="author"><name><surname>Schafersman</surname><given-names>SD</given-names></name></person-group>. <comment>An introduction to science: Scientific thinking and the scientific method. Online lecture note)</comment> (<year>1997</year>). Available at: <ext-link ext-link-type="uri" xlink:href="https://gehrcke.de/2014/06/steven-d-schafersmans-introduction-to-science/">https://gehrcke.de/2014/06/steven-d-schafersmans-introduction-to-science/</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B17"><label>17.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Birken</surname><given-names>SA</given-names></name><name><surname>Bunger</surname><given-names>AC</given-names></name><name><surname>Powell</surname><given-names>BJ</given-names></name><name><surname>Turner</surname><given-names>K</given-names></name><name><surname>Clary</surname><given-names>AS</given-names></name><name><surname>Klaman</surname><given-names>SL</given-names></name><etal/></person-group> <article-title>Organizational theory for dissemination and implementation research</article-title>. <source>Implement Sci</source>. (<year>2017</year>) <volume>12</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>15</lpage>. <pub-id pub-id-type="doi">10.1186/s13012-017-0592-x</pub-id><pub-id pub-id-type="pmid">28057027</pub-id></citation></ref>
<ref id="B18"><label>18.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Payne</surname><given-names>J</given-names></name><name><surname>Leiter</surname><given-names>J</given-names></name></person-group>. <article-title>Structuring agency: examining healthcare management in the USA and Australia using organizational theory</article-title>. <source>J Health Organ Manag</source>. (<year>2013</year>) <volume>27</volume>(<issue>1</issue>). <pub-id pub-id-type="doi">10.1108/14777261311311825</pub-id><pub-id pub-id-type="pmid">23734479</pub-id></citation></ref>
<ref id="B19"><label>19.</label><citation citation-type="book"><person-group person-group-type="author"><name><surname>Daft</surname><given-names>RL</given-names></name></person-group>. <source>Organizational theory &#x0026; design</source>. <publisher-loc>Boston, MA</publisher-loc>: <publisher-name>Cengage Learning</publisher-name> (<year>2016</year>).</citation></ref>
<ref id="B20"><label>20.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Birken</surname><given-names>SA</given-names></name><name><surname>Ko</surname><given-names>LK</given-names></name><name><surname>Wangen</surname><given-names>M</given-names></name><name><surname>Wagi</surname><given-names>CR</given-names></name><name><surname>Bender</surname><given-names>M</given-names></name><name><surname>Nilsen</surname><given-names>P</given-names></name><etal/></person-group> <article-title>Increasing access to organization theories for implementation science</article-title>. <source>Front Health Serv</source>. (<year>2022</year>) <volume>2</volume>:<fpage>1</fpage>&#x2013;<lpage>13</lpage>. <pub-id pub-id-type="doi">10.3389/frhs.2022.891507</pub-id></citation></ref>
<ref id="B21"><label>21.</label><citation citation-type="other"><collab>Network CPaCR</collab>. <comment>(2020). Available at:</comment> <ext-link ext-link-type="uri" xlink:href="https://cpcrn.org/">https://cpcrn.org/</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B22"><label>22.</label><citation citation-type="other"><collab>Concept Systems, Incorporated</collab>. <comment>2022.273.21 B. The Concept System&#x00AE; Global MAX&#x2122; Ithaca, NY. (2021). Available at:</comment> <ext-link ext-link-type="uri" xlink:href="http://www.conceptsystemsglobal.com">http://www.conceptsystemsglobal.com</ext-link> (Accessed January 24, 2023).</citation></ref>
<ref id="B23"><label>23.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Rosas</surname><given-names>SR</given-names></name><name><surname>Kane</surname><given-names>M</given-names></name></person-group>. <article-title>Quality and rigor of the concept mapping methodology: a pooled study analysis</article-title>. <source>Eval Program Plann</source>. (<year>2012</year>) <volume>35</volume>(<issue>2</issue>):<fpage>236</fpage>&#x2013;<lpage>45</lpage>. <pub-id pub-id-type="doi">10.1016/j.evalprogplan.2011.10.003</pub-id><pub-id pub-id-type="pmid">22221889</pub-id></citation></ref>
<ref id="B24"><label>24.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>P&#x00E9;ladeau</surname><given-names>N</given-names></name><name><surname>Dagenais</surname><given-names>C</given-names></name><name><surname>Ridde</surname><given-names>V</given-names></name></person-group>. <article-title>Concept mapping internal validity: a case of misconceived mapping?</article-title> <source>Eval Program Plann</source>. (<year>2017</year>) <volume>62</volume>:<fpage>56</fpage>&#x2013;<lpage>63</lpage>. <pub-id pub-id-type="doi">10.1016/j.evalprogplan.2017.02.005</pub-id></citation></ref>
<ref id="B25"><label>25.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hannan</surname><given-names>MT</given-names></name><name><surname>Freeman</surname><given-names>J</given-names></name></person-group>. <article-title>The population ecology of organizations</article-title>. <source>Am J Sociol</source>. (<year>1977</year>) <volume>82</volume>(<issue>5</issue>):<fpage>929</fpage>&#x2013;<lpage>64</lpage>. <pub-id pub-id-type="doi">10.1086/226424</pub-id></citation></ref>
<ref id="B26"><label>26.</label><citation citation-type="book"><person-group person-group-type="author"><name><surname>Giacalone</surname><given-names>RA</given-names></name><name><surname>Jurkiewicz</surname><given-names>CL</given-names></name></person-group>. <source>Handbook of workplace spirituality and organizational performance</source>. <publisher-loc>New York, NY</publisher-loc>: <publisher-name>M.E. Sharpe</publisher-name> (<year>2003</year>).</citation></ref>
<ref id="B27"><label>27.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Barocas</surname><given-names>DA</given-names></name><name><surname>Boorjian</surname><given-names>SA</given-names></name><name><surname>Alvarez</surname><given-names>RD</given-names></name><name><surname>Downs</surname><given-names>TM</given-names></name><name><surname>Gross</surname><given-names>CP</given-names></name><name><surname>Hamilton</surname><given-names>BD</given-names></name><etal/></person-group> <article-title>Microhematuria: Aua/sufu guideline</article-title>. <source>J Urol</source>. (<year>2020</year>) <volume>204</volume>(<issue>4</issue>):<fpage>778</fpage>&#x2013;<lpage>86</lpage>. <pub-id pub-id-type="doi">10.1097/JU.0000000000001297</pub-id><pub-id pub-id-type="pmid">32698717</pub-id></citation></ref>
<ref id="B28"><label>28.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Birken</surname><given-names>SA</given-names></name><name><surname>Powell</surname><given-names>BJ</given-names></name><name><surname>Presseau</surname><given-names>J</given-names></name><name><surname>Kirk</surname><given-names>MA</given-names></name><name><surname>Lorencatto</surname><given-names>F</given-names></name><name><surname>Gould</surname><given-names>NJ</given-names></name><etal/></person-group> <article-title>Combined use of the consolidated framework for implementation research (CFIR) and the theoretical domains framework (TDF): a systematic review</article-title>. <source>Implement Sci</source>. (<year>2017</year>) <volume>12</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>14</lpage>. <pub-id pub-id-type="doi">10.1186/s13012-016-0534-z</pub-id><pub-id pub-id-type="pmid">28057027</pub-id></citation></ref>
<ref id="B29"><label>29.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sales</surname><given-names>AE</given-names></name><name><surname>Barnaby</surname><given-names>DP</given-names></name><name><surname>Rentes</surname><given-names>VC</given-names></name></person-group>. <article-title>Letter to the editor on &#x201C;the implementation research logic model: a method for planning, executing, reporting, and synthesizing implementation projects&#x201D;(Smith JD, Li DH, Rafferty MR. The implementation research logic model: a method for planning, executing, reporting, and synthesizing implementation projects. Implement Sci. 2020; 15 (1): 84</article-title>. <source>Implement Sci</source>. (<year>2021</year>) <volume>16</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>3</lpage>. <pub-id pub-id-type="doi">10.1186/s13012-021-01169-1</pub-id><pub-id pub-id-type="pmid">33413491</pub-id></citation></ref>
<ref id="B30"><label>30.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Wahlen</surname><given-names>MM</given-names></name><name><surname>Schroeder</surname><given-names>MC</given-names></name><name><surname>Johnson</surname><given-names>EC</given-names></name><name><surname>Lizarraga</surname><given-names>IM</given-names></name><name><surname>Engelbart</surname><given-names>JM</given-names></name><name><surname>Tatman</surname><given-names>DJ</given-names></name><etal/></person-group> <article-title>Identifying core functions of an evidence-based intervention to improve cancer care quality in rural hospitals</article-title>. <source>Front Health Serv</source>. (<year>2022</year>) <volume>2</volume>:<fpage>891574</fpage>. <comment>-</comment>. <pub-id pub-id-type="doi">10.3389/frhs.2022.891574</pub-id><pub-id pub-id-type="pmid">36188431</pub-id></citation></ref>
<ref id="B31"><label>31.</label><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Moore</surname><given-names>GF</given-names></name><name><surname>Evans</surname><given-names>RE</given-names></name></person-group>. <article-title>What theory, for whom and in which context? Reflections on the application of theory in the development and evaluation of complex population health interventions</article-title>. <source>SSM Popul Health</source>. (<year>2017</year>) <volume>3</volume>:<fpage>132</fpage>&#x2013;<lpage>5</lpage>. <pub-id pub-id-type="doi">10.1016/j.ssmph.2016.12.005</pub-id><pub-id pub-id-type="pmid">29302610</pub-id></citation></ref></ref-list>
</back>
</article>