<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article xml:lang="EN" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. High Perform. Comput.</journal-id>
<journal-title>Frontiers in High Performance Computing</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. High Perform. Comput.</abbrev-journal-title>
<issn pub-type="epub">2813-7337</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fhpcp.2024.1303358</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>High Performance Computing</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>Nek5000/RS performance on advanced GPU architectures</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" corresp="yes">
<name><surname>Min</surname> <given-names>Misun</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="corresp" rid="c001"><sup>&#x0002A;</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/2177096/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/formal-analysis/"/>
<role content-type="https://credit.niso.org/contributor-roles/funding-acquisition/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/resources/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/visualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Lan</surname> <given-names>Yu-Hsiang</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/2526300/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/visualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Fischer</surname> <given-names>Paul</given-names></name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/2111011/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
<role content-type="https://credit.niso.org/contributor-roles/formal-analysis/"/>
<role content-type="https://credit.niso.org/contributor-roles/funding-acquisition/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/resources/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Rathnayake</surname> <given-names>Thilina</given-names></name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Holmen</surname> <given-names>John</given-names></name>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
<uri xlink:href="http://loop.frontiersin.org/people/2526292/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/resources/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>Mathematics and Computer Science Divison, Argonne National Laboratory</institution>, <addr-line>Lemont, IL</addr-line>, <country>United States</country></aff>
<aff id="aff2"><sup>2</sup><institution>Department of Computer Science, University of Illinois Urbana-Champaign</institution>, <addr-line>Urbana, IL</addr-line>, <country>United States</country></aff>
<aff id="aff3"><sup>3</sup><institution>Department of Mechanical Science and Engineering, University of Illinois Urbana-Champaign</institution>, <addr-line>Urbana, IL</addr-line>, <country>United States</country></aff>
<aff id="aff4"><sup>4</sup><institution>Oak Ridge National Laboratory, Oak Ridge Leadership Computing Facility</institution>, <addr-line>Oak Ridge, TN</addr-line>, <country>United States</country></aff>
<author-notes>
<fn fn-type="edited-by"><p>Edited by: Erik Draeger, Lawrence Livermore National Laboratory (DOE), United States</p></fn>
<fn fn-type="edited-by"><p>Reviewed by: Stefano Markidis, KTH Royal Institute of Technology, Sweden</p>
<p>Julian Kunkel, University of G&#x000F6;ttingen, Germany</p></fn>
<corresp id="c001">&#x0002A;Correspondence: Misun Min <email>mmin&#x00040;mcs.anl.gov</email></corresp>
</author-notes>
<pub-date pub-type="epub">
<day>21</day>
<month>02</month>
<year>2025</year>
</pub-date>
<pub-date pub-type="collection">
<year>2024</year>
</pub-date>
<volume>2</volume>
<elocation-id>1303358</elocation-id>
<history>
<date date-type="received">
<day>27</day>
<month>09</month>
<year>2023</year>
</date>
<date date-type="accepted">
<day>19</day>
<month>11</month>
<year>2024</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x000A9; 2025 Min, Lan, Fischer, Rathnayake and Holmen.</copyright-statement>
<copyright-year>2025</copyright-year>
<copyright-holder>Min, Lan, Fischer, Rathnayake and Holmen</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p></license>
</permissions>
<abstract>
<p>The authors explore performance scalability of the open-source thermal-fluids code, NekRS, on the U.S. Department of Energy&#x00027;s leadership computers, Crusher, Frontier, Summit, Perlmutter, and Polaris. Particular attention is given to analyzing performance and time-to-solution at the strong-scale limit for a target efficiency of 80%, which is typical for production runs on the DOE&#x00027;s high-performance computing systems. Several examples of anomalous behavior are also discussed and analyzed.</p></abstract>
<kwd-group>
<kwd>Nek5000/RS</kwd>
<kwd>exascale</kwd>
<kwd>strong scaling</kwd>
<kwd>small modular reactor</kwd>
<kwd>rod-bundle</kwd>
</kwd-group>
<contract-sponsor id="cn001">U.S. Department of Energy<named-content content-type="fundref-id">10.13039/100000015</named-content></contract-sponsor>
<counts>
<fig-count count="8"/>
<table-count count="3"/>
<equation-count count="12"/>
<ref-count count="27"/>
<page-count count="15"/>
<word-count count="8928"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Architecture and Systems</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="s1">
<title>1 Introduction</title>
<p>As part of its Exascale Computing Project, the U.S. Department of Energy has deployed a sequence of platforms at its leadership computing facilities leading up to those capable of reaching &#x0003E; 1 exaFLOPS (10<sup>18</sup> floating point operations per second). These highly parallel computers feature &#x02248;10<sup>3</sup>&#x02013;10<sup>4</sup> nodes, each equipped with powerful CPUs and anywhere from 4 to 12 accelerators (i.e., GPUs), which provide the bulk of the compute power. For reasons of efficiency, a favored programming model for these architectures is to assign a single process (i.e., MPI rank) to each GPU (or GPU processing unit, such as a GCD on the AMD MI250X or a tile on the Intel PVC) and to execute across the accelerators using a private distributed-memory programming model. With <italic>P</italic> &#x0003D; 10<sup>3</sup>&#x02013;10<sup>5</sup> MPI ranks, this approach affords a significant amount of internode parallelism and contention-free bandwidth with no increase in memory-access latency, save for the relatively sparse internode communication that is handled by MPI.</p>
<p>Here, we explore <italic>parallel scalability</italic> for the open source thermal-fluids simulation code Nek5000/RS (Fischer et al., <xref ref-type="bibr" rid="B11">2008</xref>, <xref ref-type="bibr" rid="B10">2022</xref>, <xref ref-type="bibr" rid="B9">2021</xref>) on serveral of these high-performance computing (HPC) platforms. Computational scientists engaged in HPC are typically interested in reducing simulation campaign times that can take days, weeks, or months to hours, days, or weeks. At runtime, these performance gains are realizable by increasing the number of compute units assigned to the task at hand, provided that order-unity parallel efficiency is sustained for the target values of <italic>P</italic>. The purpose of the present study is to characterize parallel efficiency of DOE&#x00027;s exascale and pre-exascale platforms as a function of problem size, <italic>n</italic>, and process count, <italic>P</italic>. An important measure is the <italic>local problem size</italic>, <italic>n</italic>/<italic>P</italic> required to sustain a parallel efficiency of &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.8. We denote this value of <italic>n</italic>/<italic>P</italic> as <italic>n</italic><sub>0.8</sub> (the precise value of &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.8 is somewhat arbitrary but is not atypical for production campaigns).</p>
<p>The baseline simulation code used for this study is NekRS, which is the GPU-oriented version of Nek5000. NekRS was developed as part of the Center for Efficient Exascale Discretizations (CEED), supported by the U.S. Department of Energy&#x00027;s Exascale Computing Project (ECP). We expect that many of the scalability findings herein will be similar to those realized by other HPC codes that are based on partial differential equations (PDEs). For example, Fischer et al. (<xref ref-type="bibr" rid="B12">2020</xref>) found that deal.ii (Arndt et al., <xref ref-type="bibr" rid="B3">2017</xref>), MFEM (Anderson et al., <xref ref-type="bibr" rid="B2">2020</xref>), and Nek5000 had similar performance characteristics over a suite of benchmark problems on Argonne&#x00027;s CPU-based IBM BG/Q, <italic>Cetus</italic>.</p>
<sec>
<title>1.1 Code overview</title>
<p>Nek5000/RS is based on the spectral element method (SEM) (Patera, <xref ref-type="bibr" rid="B24">1984</xref>). The SEM uses <italic>N</italic>th-order tensor-product polynomials in the reference domain, <inline-formula><mml:math id="M1"><mml:mover accent="true"><mml:mrow><mml:mo>&#x003A9;</mml:mo></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:math></inline-formula> := [&#x02212;1, 1]<sup>3</sup>. Use of nodal bases situated at the Gauss-Lobatto-Legendre quadrature points provides numerical stability as well as convenient and highly accurate numerical quadrature, which largely obviates the need for interpolation to alternative quadrature points, save for evaluation of the advection term, where accurate integration is required to preserve skew-symmetry, as noted in Malm et al. (<xref ref-type="bibr" rid="B17">2013</xref>). To handle complex geometries, the computational domain &#x003A9; is partitioned into <italic>E</italic> nonoverlapping elements, &#x003A9;<sup><italic>e</italic></sup>, <italic>e</italic> &#x0003D; 1, &#x02026;, <italic>E</italic>, each of which is a mapped image of <inline-formula><mml:math id="M2"><mml:mover accent="true"><mml:mrow><mml:mo>&#x003A9;</mml:mo></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:math></inline-formula>. Tensor-product-sum factorization allows for efficient matrix-free operator evaluation in <inline-formula><mml:math id="M3"><mml:mover accent="true"><mml:mrow><mml:mo>&#x003A9;</mml:mo></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:math></inline-formula> with only <italic>O</italic>(<italic>n</italic>) memory references and <italic>O</italic>(<italic>nN</italic>) BLAS3-type (tensor-contraction) operations, where the number of grid points is <italic>n</italic>&#x02248;<italic>EN</italic><sup>3</sup> (Deville et al., <xref ref-type="bibr" rid="B7">2002</xref>; Orszag, <xref ref-type="bibr" rid="B22">1980</xref>). Semi-implicit timestepping is used to advance the unsteady incompressible Navier-Stokes equations (and other physical processes such as thermal transport and combustion, which we do not address here).</p>
<p>The Navier-Stokes time-advancement involves three principal substeps to update the vector velocity field, <bold>u</bold><sup><italic>m</italic></sup>, and pressure, <italic>p</italic><sup><italic>m</italic></sup>, at time <italic>t</italic><sup><italic>m</italic></sup>. Omitting details of boundary conditions and constants, the first step amounts to gathering known data, including explicit treatment of the nonlinear advection terms,</p>
<disp-formula id="E1"><label>(1)</label><mml:math id="M4"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mstyle displaystyle="true"><mml:munderover accentunder="false" accent="false"><mml:mrow><mml:mo>&#x02211;</mml:mo></mml:mrow><mml:mrow><mml:mi>j</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>k</mml:mi></mml:mrow></mml:munderover></mml:mstyle><mml:msub><mml:mrow><mml:mi>&#x003B2;</mml:mi></mml:mrow><mml:mrow><mml:mi>j</mml:mi></mml:mrow></mml:msub><mml:msup><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mo>-</mml:mo><mml:mo>&#x00394;</mml:mo><mml:mi>t</mml:mi><mml:msub><mml:mrow><mml:mi>&#x003B1;</mml:mi></mml:mrow><mml:mrow><mml:mi>j</mml:mi></mml:mrow></mml:msub><mml:msup><mml:mrow><mml:mi>N</mml:mi><mml:mi>L</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where the &#x003B2;<sub><italic>j</italic></sub>s and &#x003B1;<sub><italic>j</italic></sub>s are order-unity coefficients associated with respective <italic>k</italic>th-order backward-difference and extrapolation formulas. Evaluation of the advection term, <italic>NL</italic><sup><italic>m</italic>&#x02212;<italic>j</italic></sup>, is compute intensive because it is evaluated using overintegration with a 3/2s-rule to ensure numerical stability (Malm et al., <xref ref-type="bibr" rid="B17">2013</xref>). The explicit update (<xref ref-type="disp-formula" rid="E1">Equation 1</xref>) leads to a Courant-Friedrichs-Lewy (CFL) limit on the stepsize of CFL := <inline-formula><mml:math id="M5"><mml:munder class="msub"><mml:mrow><mml:mo class="qopname">max</mml:mo></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>x</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>&#x02208;</mml:mo><mml:mo>&#x003A9;</mml:mo></mml:mrow></mml:munder><mml:mfrac><mml:mrow><mml:mo>&#x00394;</mml:mo><mml:mi>t</mml:mi><mml:mo>|</mml:mo><mml:msub><mml:mrow><mml:mi>u</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>|</mml:mo></mml:mrow><mml:mrow><mml:mo>&#x00394;</mml:mo><mml:msub><mml:mrow><mml:mi>x</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>&#x02248;</mml:mo><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>5</mml:mn></mml:math></inline-formula>. To circumvent this constraint, a frequently used alternative in Nek5000/RS is the characteristics based formulation of Maday et al. (<xref ref-type="bibr" rid="B16">1990</xref>) and Patel et al. (<xref ref-type="bibr" rid="B23">2019</xref>),</p>
<disp-formula id="E2"><label>(2)</label><mml:math id="M6"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mstyle displaystyle="true"><mml:munderover accentunder="false" accent="false"><mml:mrow><mml:mo>&#x02211;</mml:mo></mml:mrow><mml:mrow><mml:mi>j</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>k</mml:mi></mml:mrow></mml:munderover></mml:mstyle><mml:msub><mml:mrow><mml:mi>&#x003B2;</mml:mi></mml:mrow><mml:mrow><mml:mi>j</mml:mi></mml:mrow></mml:msub><mml:msup><mml:mrow><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where <inline-formula><mml:math id="M7"><mml:msup><mml:mrow><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mo>:</mml:mo><mml:mo>=</mml:mo><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle><mml:mo>,</mml:mo><mml:msup><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi></mml:mrow></mml:msup></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:math></inline-formula> and <inline-formula><mml:math id="M8"><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle><mml:mo>,</mml:mo><mml:mi>t</mml:mi></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:math></inline-formula> satisfies</p>
<disp-formula id="E3"><label>(3)</label><mml:math id="M9"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover></mml:mrow><mml:mrow><mml:mi>t</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mo>-</mml:mo><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle><mml:mo>&#x000B7;</mml:mo><mml:mo>&#x02207;</mml:mo><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle><mml:mo>,</mml:mo><mml:mi>t</mml:mi></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>,</mml:mo><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle><mml:mo>&#x02208;</mml:mo><mml:mo>&#x003A9;</mml:mo><mml:mo>,</mml:mo><mml:mi>t</mml:mi><mml:mo>&#x02208;</mml:mo><mml:mrow><mml:mo>[</mml:mo><mml:mrow><mml:msup><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mo>,</mml:mo><mml:msup><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi></mml:mrow></mml:msup></mml:mrow><mml:mo>]</mml:mo></mml:mrow><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>with initial condition <inline-formula><mml:math id="M10"><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>&#x0007E;</mml:mo></mml:mover><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle><mml:mo>,</mml:mo><mml:msup><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>=</mml:mo><mml:msup><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>-</mml:mo><mml:mi>j</mml:mi></mml:mrow></mml:msup><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>x</mml:mtext></mml:mstyle></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:math></inline-formula>. For CFL &#x02248;2<italic>S</italic>, solution of <xref ref-type="disp-formula" rid="E3">Equation 3</xref> requires 4<italic>Sk</italic> nonlinear evaluations per timestep, &#x00394;<italic>t</italic>, where <italic>S</italic> = 1 or 2 are typical numbers of subcycle steps, and <italic>k</italic> is the temporal order of accuracy. With <italic>k</italic> &#x0003D; 2 and <italic>S</italic> &#x0003D; 1 the advection update (<xref ref-type="disp-formula" rid="E2">Equations 2</xref>, <xref ref-type="disp-formula" rid="E3">3</xref>) can account for &#x02248;30% of the runtime and a significant fraction of the flops.</p>
<p>The second Navier-Stokes step is the pressure Poisson solve,</p>
<disp-formula id="E4"><label>(4)</label><mml:math id="M11"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mo>-</mml:mo><mml:msup><mml:mrow><mml:mo>&#x02207;</mml:mo></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msup><mml:msup><mml:mrow><mml:mi>p</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi></mml:mrow></mml:msup></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mo>&#x00394;</mml:mo><mml:mi>t</mml:mi></mml:mrow></mml:mfrac><mml:mo>&#x02207;</mml:mo><mml:mo>&#x000B7;</mml:mo><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>^</mml:mo></mml:mover></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>This step uses GMRES, preconditioned by FP32 <italic>p</italic>-multigrid with overlapping Schwarz smoothers. Each Schwarz subdomain consists of a single spectral element with additional degrees-of-freedom lifted from neighboring elements. The local subdomain problems are solved using a fast diagonalization method that is implemented as a sequence of tensor contractions, <inline-formula><mml:math id="M12"><mml:mrow><mml:msup><mml:mrow><mml:mover accent='true'><mml:mrow><mml:munder accentunder='true'><mml:mi>p</mml:mi><mml:mo stretchy='true'>_</mml:mo></mml:munder></mml:mrow><mml:mo stretchy='true'>&#x00AF;</mml:mo></mml:mover></mml:mrow><mml:mi>e</mml:mi></mml:msup><mml:mo>=</mml:mo><mml:mo stretchy='false'>(</mml:mo><mml:msub><mml:mi>S</mml:mi><mml:mi>&#x03B6;</mml:mi></mml:msub><mml:mo>&#x2297;</mml:mo><mml:msub><mml:mi>S</mml:mi><mml:mi>&#x03B7;</mml:mi></mml:msub><mml:mo>&#x2297;</mml:mo><mml:msub><mml:mi>S</mml:mi><mml:mi>&#x03BE;</mml:mi></mml:msub><mml:mo stretchy='false'>)</mml:mo><mml:msup><mml:mi>D</mml:mi><mml:mrow><mml:mo>&#x2212;</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:msup><mml:mo stretchy='false'>(</mml:mo><mml:msubsup><mml:mi>S</mml:mi><mml:mi>&#x03B6;</mml:mi><mml:mi>T</mml:mi></mml:msubsup><mml:mo>&#x2297;</mml:mo><mml:msubsup><mml:mi>S</mml:mi><mml:mi>&#x03B7;</mml:mi><mml:mi>T</mml:mi></mml:msubsup><mml:mo>&#x2297;</mml:mo><mml:msubsup><mml:mi>S</mml:mi><mml:mi>&#x03BE;</mml:mi><mml:mi>T</mml:mi></mml:msubsup><mml:mo stretchy='false'>)</mml:mo><mml:msup><mml:mrow><mml:mover accent='true'><mml:mrow><mml:munder accentunder='true'><mml:mi>f</mml:mi><mml:mo stretchy='true'>_</mml:mo></mml:munder></mml:mrow><mml:mo stretchy='true'>&#x00AF;</mml:mo></mml:mover></mml:mrow><mml:mi>e</mml:mi></mml:msup><mml:mo>,</mml:mo></mml:mrow></mml:math></inline-formula> <italic>e</italic> &#x0003D; 1, &#x02026;, <italic>E</italic>, where <italic>D</italic> is a diagonal matrix of eigenvalues, <italic>S</italic><sub>&#x0002A;</sub> is the matrix of eigenvectors for the 1D Poisson operator in each coordinate (&#x003BE;, &#x003B7;, &#x003B6;) in an extended reference element, <inline-formula><mml:math id="M13"><mml:mover accent='true'><mml:mo>&#x03A9;</mml:mo><mml:mo>&#x00AF;</mml:mo></mml:mover></mml:math></inline-formula>, and <inline-formula><mml:math id="M14"><mml:mrow><mml:msup><mml:mrow><mml:mover accent='true'><mml:mrow><mml:munder accentunder='true'><mml:mi>u</mml:mi><mml:mo stretchy='true'>_</mml:mo></mml:munder></mml:mrow><mml:mo stretchy='true'>&#x00AF;</mml:mo></mml:mover></mml:mrow><mml:mi>e</mml:mi></mml:msup></mml:mrow></mml:math></inline-formula> and <inline-formula><mml:math id="M15"><mml:mrow><mml:msup><mml:mrow><mml:mover accent='true'><mml:mrow><mml:munder accentunder='true'><mml:mi>f</mml:mi><mml:mo stretchy='true'>_</mml:mo></mml:munder></mml:mrow><mml:mo stretchy='true'>&#x00AF;</mml:mo></mml:mover></mml:mrow><mml:mi>e</mml:mi></mml:msup></mml:mrow></mml:math></inline-formula> are respective solutions and data in the extended subdomain, <inline-formula><mml:math id="M16"><mml:msup><mml:mrow><mml:mover accent="true"><mml:mrow><mml:mo>&#x003A9;</mml:mo></mml:mrow><mml:mo>&#x00304;</mml:mo></mml:mover></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msup></mml:math></inline-formula>. The lowest level of the <italic>p</italic>-multigrid entails applying one or two V-cycles of algebraic multigrid with Hypre to a coarse-grid problem based on the spectral element vertices. Overall, the pressure solve is communication intensive because of the long tails of the Green&#x00027;s functions associated with the Poisson operator.</p>
<p>The final Navier-Stokes update entails solving a diagonally-dominant Helmholtz system for the velocity components,</p>
<disp-formula id="E5"><label>(5)</label><mml:math id="M17"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mstyle mathvariant="bold"><mml:mtext>H</mml:mtext></mml:mstyle><mml:msup><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mrow><mml:mi>m</mml:mi></mml:mrow></mml:msup></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mover accent="true"><mml:mrow><mml:mstyle mathvariant="bold"><mml:mtext>u</mml:mtext></mml:mstyle></mml:mrow><mml:mo>^</mml:mo></mml:mover><mml:mo>-</mml:mo><mml:mo>&#x00394;</mml:mo><mml:mi>t</mml:mi><mml:mo>&#x02207;</mml:mo><mml:msup><mml:mrow><mml:mi>p</mml:mi></mml:mrow><mml:mrow><mml:mi>m</mml:mi></mml:mrow></mml:msup><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where <bold>H</bold> := <inline-formula><mml:math id="M18"><mml:mo>-</mml:mo><mml:mfrac><mml:mrow><mml:mo>&#x00394;</mml:mo><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>R</mml:mi><mml:mi>e</mml:mi></mml:mrow></mml:mfrac><mml:msup><mml:mrow><mml:mo>&#x02207;</mml:mo></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msup><mml:mo>&#x0002B;</mml:mo><mml:msub><mml:mrow><mml:mi>&#x003B2;</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn></mml:mrow></mml:msub><mml:mstyle mathvariant="bold"><mml:mtext>I</mml:mtext></mml:mstyle></mml:math></inline-formula>, represents the update of the momentum equation with the implicitly treated viscous terms. Iterative solution of the block system <bold>H</bold> updates all velocity components simultaneously, which reduces the number of message exchanges by a factor of three in the Jacobi-preconditioned conjugate gradient solver and therefore amortizes communication latency.</p>
<p>Parallel-work decomposition is realized by partitioning the set of elements into contiguous subsets using recursive spectral bisection (Pothen et al., <xref ref-type="bibr" rid="B25">1990</xref>) with load-balanced partitioning such that the number of elements per subdomain (i.e., per MPI rank) differs by at most 1. Given that we typically have several thousand spectral elements per GPU, load balance is not a source of inefficiency in production simulations.</p>
<p>For portability, all the GPU kernels are written in OCCA (Medina et al., <xref ref-type="bibr" rid="B18">2014</xref>; OCCA, <xref ref-type="bibr" rid="B21">2021</xref>), which was developed by Tim Warburton&#x00027;s group at Virginia Tech. and Rice University. Many of the high-performance kernels originated with the libParanumal library (Chalmers et al., <xref ref-type="bibr" rid="B5">2020</xref>), also developed by Warburton&#x00027;s group. Roofline performance and scaling results can be found in the Hipbone study of Chalmers et al. (<xref ref-type="bibr" rid="B6">2023</xref>) and the NekRS study (Min et al., <xref ref-type="bibr" rid="B20">2022</xref>). In the latter, which includes detail NVIDIA Nsight timing data, the authors note that all kernels in NekRS, save for the latency-bound gather-scatter kernels in the coarse <italic>p</italic>-multigrid levels, <italic>achieve near-roofline performance defined as</italic> &#x0003E; <italic>70 % of the leading-performance limiter</italic>. Additional details of the NekRS GPU implementation, including kernel pseudocode, can be found in Fischer et al. (<xref ref-type="bibr" rid="B10">2022</xref>).</p>
</sec>
<sec>
<title>1.2 Performance metrics</title>
<p>A primary concern for computational scientists is the speed that can be realized for a particular application for a given architecture (here, an <italic>application</italic> is a particular <italic>problem</italic> that uses Nek5000/RS, which is an <italic>application code</italic>). For example, one frequently needs to estimate the number of node-hours and number of wall-clock hours that might be required for a large simulation campaign. A common metric, which is very much case-specific, is the number of degrees of freedom (dofs) per second that can be realized on a platform, or perhaps the number of dofs per second per accelerator (i.e., per MPI rank<xref ref-type="fn" rid="fn0001"><sup>1</sup></xref>). In the sequel, we will assign GDOFS to the quantity &#x0201C;billions of dofs per second per rank.&#x0201D; The case-specificity aspect of GDOFS is that one can realize a much larger GDOFS value for linear solution of <italic>A</italic><underline><italic>x</italic></underline> &#x0003D; <underline><italic>b</italic></underline> than would be possible for, say, a single timestep of an incompressible Navier&#x02013;Stokes solver.</p>
<p>Despite the large variance in GDOFS from one problem class to the next, it is nonetheless a worthy metric when making platform-to-platform comparisons. A related metric is the time-to-solution or, in the case of a timestepping simulation code that could take an arbitrarily large number of steps, the time per step, <italic>t</italic><sub><italic>step</italic></sub>, <xref ref-type="fn" rid="fn0002"><sup>2</sup></xref> which we measure in seconds. Even for a given code and architecture this latter quantity is subject to significant variability because some problems or computational meshes are more ill-conditioned than others, which leads to higher iteration counts in the linear solvers (e.g., in the pressure solve for an incompressible flow simulation) and hence a longer time per step.</p>
<p>GDOFS and <italic>t</italic><sub><italic>step</italic></sub> are dependent, or output, parameters. For a fixed platform, code, and problem, users still have two independent parameters at their disposal: <italic>n</italic>, the problem size or number of dofs,<xref ref-type="fn" rid="fn0003"><sup>3</sup></xref> and <italic>P</italic>, the number of ranks (here, accelerator devices) to use. For a fixed problem size, <italic>n</italic> (which is determined by resolution requirements), there is only one variable, namely, <italic>P</italic>. A user who is contemplating a simulation campaign will often be interested in predicting performance over a range of <italic>n</italic>. Under the given conditions, we will see that the most important performance predictor is</p>
<disp-formula id="E6"><label>(6)</label><mml:math id="M19"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mfrac><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:mfrac></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mtext class="textrm" mathvariant="normal">the number of grid points per device</mml:mtext><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where we reiterate that we are assigning a single MPI rank to each device.</p>
<p>Two other dependent quantities of interest are parallel efficiency, &#x003B7;<sub><italic>P</italic></sub>, and realized FLOPS, the measurable number of 64-bit floating-point operations per second. We typically will report FLOPS per rank, which is more universal than aggregate FLOPS. Also, in cases where mixed precision is used (e.g., when 32-bit arithmetic is used in a preconditioner), we count the FP32 flops as a half-flop each. A typical definition of (strong-scaling) parallel efficiency is</p>
<disp-formula id="E7"><label>(7)</label><mml:math id="M20"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>&#x003B7;</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub></mml:mtd><mml:mtd><mml:mo>:</mml:mo><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn></mml:mrow></mml:msub><mml:msub><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>s</mml:mi><mml:mi>t</mml:mi><mml:mi>e</mml:mi><mml:mi>p</mml:mi></mml:mrow></mml:msub><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn></mml:mrow></mml:msub></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow><mml:mrow><mml:mi>P</mml:mi><mml:msub><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>s</mml:mi><mml:mi>t</mml:mi><mml:mi>e</mml:mi><mml:mi>p</mml:mi></mml:mrow></mml:msub><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mtext class="textrm" mathvariant="normal">GDOFS</mml:mtext><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow><mml:mrow><mml:mtext class="textrm" mathvariant="normal">GDOFS</mml:mtext><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn></mml:mrow></mml:msub></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mfrac><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where <italic>t</italic><sub><italic>step</italic></sub>(<italic>P</italic>) is the time per step when running a fixed problem of fixed size, <italic>n</italic>, on <italic>P</italic> ranks and GDOFS(<italic>P</italic>) is the corresponding number of gigadofs per second per rank. Here, <italic>P</italic><sub>0</sub> is the smallest number of ranks that is able to hold the problem. On some architectures, the amount of memory per GPU is relatively small, which prevents extensive strong-scaling studies (from a user&#x00027;s perspective, however, this is potentially a happy circumstance since there is &#x0201C;just enough&#x0201D; memory and not a lot of idle memory that incurs unnecessary capital and power overhead).</p>
<p>An alternative definition of parallel efficiency is given by the relationship</p>
<disp-formula id="E8"><label>(8)</label><mml:math id="M21"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub></mml:mtd><mml:mtd><mml:mo>=</mml:mo></mml:mtd><mml:mtd><mml:msub><mml:mrow><mml:mi>&#x003B7;</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub><mml:mi>P</mml:mi><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub><mml:mo>.</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>Here, <italic>S</italic><sub><italic>P</italic></sub> is the <italic>speed</italic> on <italic>P</italic> processors, which could be measured in total (not per rank) FLOPS or GDOFS. This definition is equivalent to <xref ref-type="disp-formula" rid="E7">Equation 7</xref> when <italic>P</italic><sub>0</sub> &#x0003D; 1. The utility of this definition is that one can consider it for either weak- (fixed <italic>n</italic>/<italic>P</italic>) or strong- (fixed <italic>n</italic>) scaling studies. If FLOPS are used, it is relatively easy to get FLOPS on one rank for a smaller version of the application problem (although that might not be a useful starting point in the exascale era given that no exascale problem comes anywhere close to fitting on one rank).</p>
<p>What <xref ref-type="disp-formula" rid="E8">Equation 8</xref> tells us is that the speed on <italic>P</italic> ranks should be &#x02248;<italic>P</italic> times the speed on 1 rank, provided we can sustain close to unity efficiency, &#x003B7;<sub><italic>P</italic></sub>&#x02248;1. We remark that HPC users generally want to run as fast as possible, particularly for large campaigns, so they want <italic>P</italic>&#x0226B;1. However, they also need to efficiently use their allocation, which implies &#x003B7;<sub><italic>P</italic></sub>&#x02248;1. This latter condition places a constraint on time-to-solution that is generally stronger than the unconstrained &#x0201C;min-time-to-solution&#x0201D; result. In our studies we will assume that the user is willing to run at 80% parallel efficiency, &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.8. Of course, other target efficiencies are possible, and a user can change <italic>P</italic> and, hence, &#x003B7;<sub><italic>P</italic></sub> on a submission-by-submission basis for each case run in a given campaign. However, &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.8 is a reasonable starting point for analysis.</p>
<p>The first analysis question we address is, <italic>For a fixed problem size</italic> <italic>n</italic><italic>, how many ranks can we use before</italic> &#x003B7;<sub><italic>P</italic></sub> &#x0003C; 0.8<italic>?</italic> An accompanying question is, <italic>What is the time per step at that value?</italic> Users and developers are also interested in <italic>the cause</italic> of the departure from unity efficiency. For example, is it load imbalance? Message-passing overhead? Or lack of parallelism on the accelerator? In our examples, as is the case for Nek5000/RS in practice, load imbalance will not be a leading contributor to inefficiency as the work is sufficiently fine-grained such that it can be distributed in a balanced way across all ranks.</p>
<p>The key to the first question about number of ranks is to recognize that parallel efficiency typically drops as the local amount of work, <italic>n</italic>/<italic>P</italic> tends toward zero. So, fixing &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.8 implies <italic>n</italic>/<italic>P</italic> is a fixed value (for a given fixed-sized problem with <italic>P</italic> varying). We denote this value as <italic>n</italic><sub>0.8</sub>. It is the number of points per rank where the application realizes 80% efficiency, which is where we anticipate that users will typically run. For a given problem size, <italic>n</italic>,</p>
<disp-formula id="E9"><label>(9)</label><mml:math id="M22"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub><mml:mo>:</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:mfrac><mml:mtext class="textrm" mathvariant="normal">such that&#x000A0;</mml:mtext><mml:mstyle class="math"><mml:msub><mml:mrow><mml:mi>&#x003B7;</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn><mml:mtext class="textrm" mathvariant="normal"></mml:mtext></mml:mstyle><mml:mo>.</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>With this definition, we can address the question of the expected <italic>t</italic><sub><italic>step</italic></sub> under these conditions. Assume that a given problem requires a certain amount of work, <italic>W</italic>, that is measured in total number of floating-point operations (FLOPS). Usually, <italic>W</italic>&#x0007E;<italic>Cn</italic>, where <italic>C</italic> is an <italic>n</italic>-independent constant, which implies that, to leading order, the amount of work scales with <italic>n</italic>. On <italic>P</italic> processors, we therefore expect</p>
<disp-formula id="E10"><label>(10)</label><mml:math id="M23"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mi>s</mml:mi><mml:mi>t</mml:mi><mml:mi>e</mml:mi><mml:mi>p</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi>W</mml:mi></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi>C</mml:mi><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>&#x003B7;</mml:mi></mml:mrow><mml:mrow><mml:mi>P</mml:mi></mml:mrow></mml:msub><mml:mi>P</mml:mi><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>.</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>We define <italic>t</italic><sub>0.8</sub> to be the value of <italic>t</italic><sub><italic>step</italic></sub> at 80% efficiency,</p>
<disp-formula id="E11"><label>(11)</label><mml:math id="M24"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi>C</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:mfrac><mml:mfrac><mml:mrow><mml:mi>n</mml:mi><mml:mo>/</mml:mo><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>.</mml:mo><mml:mo>=</mml:mo><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mfrac><mml:mrow><mml:mi>C</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:mfrac></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>.</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>We note that <xref ref-type="disp-formula" rid="E10">Equations 10</xref>, <xref ref-type="disp-formula" rid="E11">11</xref> are predicated on &#x003B7;<sub><italic>P</italic></sub> being strongly dependent on (<italic>n</italic>/<italic>P</italic>) with no direct <italic>P</italic> dependence. There are times when there is a weak <italic>P</italic> dependence, particularly for <italic>P</italic>&#x0003E;10<sup>4</sup>. In this case, one can simply modify the analysis to have a <italic>P</italic>-dependent <italic>n</italic><sub>0.8</sub>.</p>
<p>We see from <xref ref-type="disp-formula" rid="E11">Equation 11</xref> that the time per step is governed by the speed on a single rank, <italic>S</italic><sub>1</sub> (larger is better), and the amount of work on a single rank, <italic>n</italic><sub>0.8</sub> (smaller is better), where 80% efficiency is realized. If a new platform comes out with a 2 &#x000D7; increase in <italic>S</italic><sub>1</sub> but a 4 &#x000D7; increase in <italic>n</italic><sub>0.8</sub>, then the net time-to-solution <italic>increases by</italic> 2 &#x000D7; . In HPC, it is the <italic>ratio</italic>, <italic>n</italic><sub>0.8</sub>/<italic>S</italic><sub>1</sub>, that is critical to fast time-to-solution. Much of this analysis can be found in Fischer et al. (<xref ref-type="bibr" rid="B8">2015</xref>, <xref ref-type="bibr" rid="B12">2020</xref>). Communication overhead on GPU-based architectures is discussed in Bienz et al. (<xref ref-type="bibr" rid="B4">2021</xref>).</p>
<p>A typical use case for <xref ref-type="disp-formula" rid="E11">Equation 11</xref> is that a user knows <italic>n</italic>, which is the number of gridpoints required to resolve a given simulation, and wants to know how many processors will be required to efficiently solve this problem and how long it will take to execute. The user also knows <italic>n</italic><sub>0.8</sub> from scaling studies of the type provided here. From that, one can determine</p>
<disp-formula id="E12"><label>(12)</label><mml:math id="M25"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>which is the maximum number of ranks that can be employed while sustaining 80% efficiency. The time per step will be <italic>t</italic><sub>0.8</sub>, and the total required node-hours will be</p>
<disp-formula id="E13"><label>(13)</label><mml:math id="M26"><mml:mtable class="eqnarray" columnalign="left"><mml:mtr><mml:mtd><mml:mtext class="textrm" mathvariant="normal">node hours</mml:mtext><mml:mo>&#x02248;</mml:mo><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub></mml:mrow><mml:mrow><mml:mtext class="textrm" mathvariant="normal">ranks-per-node</mml:mtext></mml:mrow></mml:mfrac><mml:mo>&#x000D7;</mml:mo><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi>N</mml:mi></mml:mrow><mml:mrow><mml:mi>s</mml:mi><mml:mi>t</mml:mi><mml:mi>e</mml:mi><mml:mi>p</mml:mi><mml:mi>s</mml:mi></mml:mrow></mml:msub><mml:msub><mml:mrow><mml:mi>t</mml:mi></mml:mrow><mml:mrow><mml:mn>0</mml:mn><mml:mo>.</mml:mo><mml:mn>8</mml:mn></mml:mrow></mml:msub></mml:mrow><mml:mrow><mml:mn>3600</mml:mn><mml:mtext class="textrm" mathvariant="normal">s/hour</mml:mtext></mml:mrow></mml:mfrac><mml:mo>,</mml:mo></mml:mtd></mml:mtr></mml:mtable></mml:math></disp-formula>
<p>where <italic>N</italic><sub><italic>steps</italic></sub> is the estimated number of timesteps.</p>
</sec>
<sec>
<title>1.3 Test cases</title>
<p>In the following sections we characterize these relevant parameters for NekRS across several of DOE&#x00027;s pre-exascale and exascale platforms, including Frontier, Crusher, Polaris, Perlmutter, ThetaGPU, and Summit. The principals from this list are described in <xref ref-type="table" rid="T1">Table 1</xref>. Simulations are performed using ExaSMR&#x00027;s 17 &#x000D7; 17 rod-bundle geometry, illustrated in <xref ref-type="fig" rid="F1">Figure 1</xref>. This geometry is periodic in the axial flow direction, which allows us to weak-scale the problem by adding more layers of elements in the <italic>z</italic> direction (the model problem is essentially homogeneous in <italic>z</italic>). Each case starts with a pseudo-turbulent initial condition so that the iterative solvers, which compute only the change in the solution on each step, are not working on void solutions. Most of the cases are run under precisely the same conditions of timestep size, iteration tolerances, and averaging procedures, which are provided case by case in the sequel.</p>
<table-wrap position="float" id="T1">
<label>Table 1</label>
<caption><p>Systems overview.</p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919498;color:#ffffff">
<th/>
<th valign="top" align="left"><bold>Summit</bold></th>
<th valign="top" align="left"><bold>Frontier</bold></th>
<th valign="top" align="left"><bold>Perlmutter</bold></th>
<th valign="top" align="left"><bold>Polaris</bold></th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">Processor</td>
<td valign="top" align="left">IBM Power9 22C 3.0GHz</td>
<td valign="top" align="left">AMD EPYC 64C 2GHz</td>
<td valign="top" align="left">AMD EPYC 64C 2.4GHz</td>
<td valign="top" align="left">AMD EPYC 32C 2.8GHz</td>
</tr>
<tr>
<td valign="top" align="left">GPU Nodes &#x00023;</td>
<td valign="top" align="left">4,608</td>
<td valign="top" align="left">9,408</td>
<td valign="top" align="left">1,792</td>
<td valign="top" align="left">560</td>
</tr>
<tr>
<td valign="top" align="left">GPUs &#x00023;</td>
<td valign="top" align="left">27,648</td>
<td valign="top" align="left">37,632</td>
<td valign="top" align="left">7,168</td>
<td valign="top" align="left">2,240</td>
</tr>
<tr>
<td valign="top" align="left">Each node</td>
<td valign="top" align="left">6 &#x000D7; NVIDIA V100</td>
<td valign="top" align="left">4 &#x000D7; AMD MI250X</td>
<td valign="top" align="left">4 &#x000D7; NVIDIA A100 SXM4</td>
<td valign="top" align="left">4 &#x000D7; NVIDIA A100 SXM4</td>
</tr>
<tr>
<td valign="top" align="left">Theoretical Peak</td>
<td valign="top" align="left">200 PF/s</td>
<td valign="top" align="left">1.7 EF/s</td>
<td valign="top" align="left">59 PF/s</td>
<td valign="top" align="left">113 PF/s</td>
</tr>
<tr>
<td valign="top" align="left">Network</td>
<td valign="top" align="left">Mellanox EDR InfiniBand</td>
<td valign="top" align="left">Slingshot-11</td>
<td valign="top" align="left">Slingshot-11</td>
<td valign="top" align="left">Slingshot-11</td>
</tr>
<tr>
<td valign="top" align="left">Peak Power</td>
<td valign="top" align="left">14.5 MW</td>
<td valign="top" align="left">29.6 MW</td>
<td valign="top" align="left">4.86 MW</td>
<td valign="top" align="left">1.8 MW</td>
</tr></tbody>
</table>
</table-wrap>
<fig id="F1" position="float">
<label>Figure 1</label>
<caption><p>ExaSMR full-core configuration on the <bold>left</bold> and a single 17 &#x000D7; 17 rod bundle on the <bold>right</bold>.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0001.tif"/>
</fig>
<p>We remark that the following performance summaries are for full Navier&#x02013;Stokes solution times. We present a few plots that reflect work in salient kernels, such as the advection operator, which is largely communication-free, and the pressure-Poisson coarse-grid solve, which is highly communication-intensive. Detailed kernel-by-kernel breakdowns are presented in Fischer et al. (<xref ref-type="bibr" rid="B10">2022</xref>) and Min et al. (<xref ref-type="bibr" rid="B20">2022</xref>) and are available in every logfile generated by NekRS.</p>
<p>Further, we note that NekRS supports multiple versions of each of its high-intensity kernels and communication utilities. At runtime, NekRS selects the fastest kernel by running a small suite of tests for each invocation of the given utility. that particular kernel for the particular platform for the particular application. An example of these outputs, along with the kernel-by-kernel breakdown, is presented in Section 5.</p>
</sec>
<sec>
<title>1.4 Additional scaling studies</title>
<p>There have been multiple performance studies on GPUs that are relevant to the high-order methods used in the current work. Bienz et al. (<xref ref-type="bibr" rid="B4">2021</xref>) explored communication characteristics for the NVIDIA V100-based platforms, Summit (using Spectrum MPI) and Lassen (using MVAPICH). Kronbichler and Ljungkvist (<xref ref-type="bibr" rid="B14">2019</xref>) study the performance of matrix-free <italic>p</italic>-type finite element methods (FEMs) in deal.ii on several CPU and GPU architectures, including a single NVIDIA P100, where they observe 80% efficiency for <italic>n</italic><sub>0.8</sub>&#x02248; 3&#x02013;4 &#x000D7; 10<sup>5</sup> when evaluating matrix-vector products. A more recent study, Kronbichler et al. (<xref ref-type="bibr" rid="B15">2023</xref>), demonstrates that deal.ii realizes 80% efficiency for Jacobi PCG on a single NVIDIA V100 when <italic>n</italic><sub>0.8</sub>&#x02248; 3&#x02013;4 &#x000D7; 10<sup>5</sup>. The same paper explores alternative PCG formulations to provide better data caching with fewer vector reductions on both CPUs and GPUs. Multi-GPU throughput studies presented in Vargas et al. (<xref ref-type="bibr" rid="B27">2022</xref>) reveal that MFEM needs <italic>n</italic><sub>0.8</sub>&#x02248;0.75&#x02013;1.0 M points per V100 for <italic>single-node</italic> (<italic>P</italic> &#x0003D; 4) hydrodynamics simulations on LLNL&#x00027;s Sierra platform. For <italic>P</italic> &#x0003D; 64 V100s on LLNL&#x00027;s Lassen, the MFEM mini-app Laghos has <italic>n</italic><sub>0.8</sub>&#x02248;2M. The observed increase is expected at higher node counts because of MPI overhead. Sathyanarayana et al. (<xref ref-type="bibr" rid="B26">2025</xref>) find <italic>n</italic><sub>0.8</sub>&#x02248; 2&#x02013;4 M for finite-difference based WENO schemes when running on single NVIDIA A100s and AMD MI250X GCDs. Chalmers et al. (<xref ref-type="bibr" rid="B6">2023</xref>) provide in-depth performance analysis of high-order matrix-free kernel performance, including roofline models and throughput scaling on NVIDIA V100, AMD MI100, and AMD MI250X accelerators, all of which is of direct relevance to NekRS, which uses many of the same kernels.</p></sec>
</sec>
<sec id="s2">
<title>2 NekRS performance on a single GPU</title>
<p><xref ref-type="table" rid="T2">Table 2</xref> presents the time-per-step metric for NekRS performance for ExaSMR&#x00027;s single-rod simulation on a single GPU. Simulations are performed for 500 steps; and the average time per step, <italic>t</italic><sub>step</sub>, is measured in seconds for the last 400 steps. For a given system, the speedup is the inverse ratio of <italic>t</italic><sub>step</sub> to that of Summit. <italic>v</italic><sub><italic>i</italic></sub> and <italic>p</italic><sub><italic>i</italic></sub> represent the average iteration counts per step of the velocity components and pressure. Timestepping is based on the second-order characteristics method (<xref ref-type="disp-formula" rid="E2">Equations 2</xref>, <xref ref-type="disp-formula" rid="E3">3</xref>) with one substep and the timestep size is &#x00394;<italic>t</italic> &#x0003D; 1.2e-3 (CFL = 1.82). Pressure preconditioning is based on <italic>p</italic>-multigrid with Chebyshev and additive Schwarz method (CHEBYSHEV&#x0002B;ASM) smoothing and hypre AMG (algebraic multigrid) for the coarse-grid solve (Fischer et al., <xref ref-type="bibr" rid="B10">2022</xref>). Tolerances for pressure and velocity are 1e-4 and 1e-6, respectively. We note that this test case has been explored in the context of NekRS kernel and algorithm development on other architectures in earlier work, including Kolev et al. (<xref ref-type="bibr" rid="B13">2021</xref>), Abdelfattah et al. (<xref ref-type="bibr" rid="B1">2021</xref>), and Fischer et al. (<xref ref-type="bibr" rid="B9">2021</xref>).</p>
<table-wrap position="float" id="T2">
<label>Table 2</label>
<caption><p>NekRS performance on various architectures using a single GPU.</p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919498;color:#ffffff">
<th valign="top" align="left" colspan="8"><bold>GPU performance on a single GPU: singlerod</bold>, <italic><bold>E</bold></italic> <bold>= 7,168</bold>, <italic><bold>n</bold></italic> <bold>= 2,458,624</bold>, <italic><bold>N</bold></italic> <bold>= 7</bold></th>
</tr>
</thead>
<tbody>
<tr style="background-color:#919498;color:#ffffff">
<td valign="top" align="left"><bold>System</bold></td>
<td valign="top" align="center"><bold>GPU</bold></td>
<td valign="top" align="center"><bold>Device</bold></td>
<td valign="top" align="center"><bold>API</bold></td>
<td valign="top" align="center"><italic>v</italic><sub><italic>i</italic></sub></td>
<td valign="top" align="center"><italic>p</italic><sub><italic>i</italic></sub></td>
<td valign="top" align="center"><italic>t</italic><sub>step</sub> <bold>(sec)</bold></td>
<td valign="top" align="center"><bold>Speedup</bold></td>
</tr>
<tr>
<td valign="top" align="left">Summit</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">16GB V100 GPU</td>
<td valign="top" align="center">CUDA</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">6.78e-02</td>
<td valign="top" align="center">1.00</td>
</tr>
<tr>
<td valign="top" align="left">Spock</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">32GB MI100 GPU</td>
<td valign="top" align="center">HIP</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">7.98e-02</td>
<td valign="top" align="center">0.84</td>
</tr>
<tr>
<td valign="top" align="left">Crusher</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">64GB MI250X (1 GCD)</td>
<td valign="top" align="center">HIP</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">6.01e-02</td>
<td valign="top" align="center">1.12</td>
</tr>
<tr>
<td valign="top" align="left">ThetaGPU</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">40GB A100 GPU</td>
<td valign="top" align="center">CUDA</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">4.31e-02</td>
<td valign="top" align="center">1.57</td>
</tr>
<tr>
<td valign="top" align="left">Perlmutter</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">40GB A100 GPU</td>
<td valign="top" align="center">CUDA</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">4.17e-02</td>
<td valign="top" align="center">1.62</td>
</tr>
<tr>
<td valign="top" align="left">Polaris</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">40GB A100 GPU</td>
<td valign="top" align="center">CUDA</td>
<td valign="top" align="center">4</td>
<td valign="top" align="center">1</td>
<td valign="top" align="center">4.16e-02</td>
<td valign="top" align="center">1.62</td>
</tr></tbody>
</table>
</table-wrap>
<p>The single-device results of <xref ref-type="table" rid="T2">Table 2</xref> show that, for the current version of NekRS,<xref ref-type="fn" rid="fn0004"><sup>4</sup></xref> a single GCD of the MI250X on Crusher realizes a 1.12 &#x000D7; gain in Navier&#x02013;Stokes solution performance over a single V100 on Summit. Similarly, the A100s are realizing &#x02248; 1.6-fold gain over the V100.</p>
<p>We explore the single-GPU performance as a function of local problem size, <italic>n</italic>/<italic>P</italic> (<italic>P</italic> &#x0003D; 1) in <xref ref-type="fig" rid="F2">Figure 2</xref> both with (right) and without (left) the characteristics-based timestepping (<xref ref-type="disp-formula" rid="E2">Equations 2</xref>, <xref ref-type="disp-formula" rid="E3">3</xref>). Various problem sizes are realized for the single-rod geometry by extruding a baseline two-dimensional mesh to comprise one or more levels in the axial flow direction. The polynomial order is <italic>N</italic> &#x0003D; 7 and the number of quadrature points for the dealiased advection operator is <italic>N</italic><sub><italic>q</italic></sub> &#x0003D; 11 in each direction, in each element. In these cases, there is no MPI overhead and, as can be seen from the match between Polaris and Perlmutter, no significant system noise. The saturated sustained performance on Polaris/Perlmutter is 1.75 TFLOPS when using characteristics and only 1.00 TFLOPS for the less computer-intensive formulation (<xref ref-type="disp-formula" rid="E1">Equation 1</xref>), which requires only three nonlinear evaluations per timestep (one for each velocity component). The relative drop-off on Frontier, 1.15 &#x02192; 0.75 TFLOPS, is not as steep, perhaps because a single GCD of the MI250X is sustaining only 3 TFLOPS for the advection kernel whereas the A100 is sustaining 5 TFLOPS, in isolation. On the other hand, Summit manifests a drop of 0.99 &#x02192; 0.57 TFLOPS, which is roughly the same ratio seen for the A100-based platforms.</p>
<fig id="F2" position="float">
<label>Figure 2</label>
<caption><p><italic>n</italic>/<italic>P</italic> scaling on single-GPU/GCD architectures for single-rod test case: <bold>left</bold>: using <xref ref-type="disp-formula" rid="E1">Equation 1</xref> without characteristics timestepping; <bold>right</bold>: using <xref ref-type="disp-formula" rid="E2">Equations 2</xref>, <xref ref-type="disp-formula" rid="E3">3</xref> with characteristics timestepping.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0002.tif"/>
</fig>
<p>As the problem size diminishes, performance drops from a lack of intra-device parallelism and from the overhead of kernel-launch latency, which becomes important when there is insufficient work to keep the accelerator busy. Consequently, we find <italic>n</italic><sub>0.8</sub>&#x02248;700K for Polaris/Perlmutter for the characteristics-based timestepping (80% efficiency = 1.4 TFLOPS) and <italic>n</italic><sub>0.8</sub>&#x02248;600K without characteristics. For Frontier, <italic>n</italic><sub>0.8</sub>&#x02248;1M with characteristics and &#x02248;900K without. For Summit the corresponding numbers are <italic>n</italic><sub>0.8</sub>&#x02248;1.5M with characteristics and &#x02248;700K without. These <italic>n</italic><sub>0.8</sub> values are significantly lower than the <italic>n</italic><sub>0.8</sub> &#x0003D; 2M&#x02013;5M that we observe for production runs described in the next sections, where <italic>P</italic>&#x0226B;1, which points to the significance of MPI overhead in the case of multi-GPU simulations.</p></sec>
<sec id="s3">
<title>3 NekRS performance on Frontier vs. Crusher</title>
<p>We next consider multi-GPU performance for ExaSMR&#x00027;s 17 &#x000D7; 17 rod-bundle geometry (<xref ref-type="fig" rid="F1">Figure 1</xref>, right), which we extend in the streamwise direction with 10, 17, and 170 layers, keeping the mesh density the same. This sequence corresponds to 277 thousand spectral elements of order <italic>N</italic> &#x0003D; 7, for a total of <italic>n</italic> &#x0003D; 0.27M &#x000D7; 7<sup>3</sup> &#x0003D; 95M grid points, 471 thousand spectral elements of order <italic>N</italic> &#x0003D; 7, for a total of <italic>n</italic> &#x0003D; 0.47M &#x000D7; 7<sup>3</sup> &#x0003D; 161M grid points, and 4.7 million spectral elements of order <italic>N</italic> &#x0003D; 7, for a total of <italic>n</italic> &#x0003D; 4.7M &#x000D7; 7<sup>3</sup> &#x0003D; 1.6B grid points, respectively. <xref ref-type="table" rid="T3">Table 3</xref> summarizes the configurations for these tests.</p>
<table-wrap position="float" id="T3">
<label>Table 3</label>
<caption><p>Problem setup for strong-/weak-scaling studies.</p></caption>
<table frame="box" rules="all">
<thead>
<tr style="background-color:#919498;color:#ffffff">
<th valign="top" align="center" colspan="4"><bold>Strong scaling test sets</bold></th>
</tr>
</thead>
<tbody>
<tr style="background-color:#919498;color:#ffffff">
<td/>
<td valign="top" align="center"><italic>E</italic></td>
<td valign="top" align="center"><italic>n</italic></td>
<td valign="top" align="center"><bold>Rank</bold>, <italic>P</italic></td>
</tr>
<tr>
<td valign="top" align="left">Case 1</td>
<td valign="top" align="center">277,000</td>
<td valign="top" align="center">95M</td>
<td valign="top" align="center">8&#x02013;64</td>
</tr>
<tr>
<td valign="top" align="left">Case 2</td>
<td valign="top" align="center">470,900</td>
<td valign="top" align="center">161M</td>
<td valign="top" align="center">14-128</td>
</tr>
<tr>
<td valign="top" align="left">Case 3</td>
<td valign="top" align="center">4,709,000</td>
<td valign="top" align="center">1.6B</td>
<td valign="top" align="center">128&#x02013;16,320</td>
</tr></tbody>
</table>
</table-wrap>
<p>Simulations are performed for 2,000 steps; and the average time-per-step, <italic>t</italic><sub>step</sub>, is measured in seconds for the last 1,000 steps. Timestepping is based on <xref ref-type="disp-formula" rid="E1">Equation 1</xref> with third-order backward-differencing (BDF3) for the time derivative and third-order extrapolation (EXT3) for the nonlinear advection term. The timestep size is &#x00394;<italic>t</italic> = 3.0e-04, which corresponds to a Courant number of CFL = 0.82. We run a single MPI rank per GCD, and there are 8 GCDs per node. On Frontier, <monospace>rocm/5.1.0</monospace> and <monospace>cray-mpich/8.1.17</monospace> were used. On Crusher, simulations were performed with several versions such as <monospace>rocm/5.1.0</monospace>, <monospace>rocm/5.2.0</monospace>, <monospace>cray-mpich/8.1.16</monospace>, and <monospace>cray-mpich/8.1.19</monospace>. On Crusher, <monospace>rocm/5.1.0</monospace> is 2%&#x02013;5% faster than <monospace>rocm/5.2.0</monospace>.</p>
<p><xref ref-type="fig" rid="F3">Figure 3</xref> shows strong scaling performance of Frontier and Crusher for the three test cases. The left panel shows the average time per step vs. the number of MPI ranks, <italic>P</italic>. The plot illustrates classic strong scaling as <italic>P</italic> is increased for fixed problem sizes of <italic>n</italic> = 95M, 161M, and 1.6B on Crusher (black) and on Frontier (red). Dashed lines in sky-blue represent ideal strong-scale profiles for each case. As is typical, larger problem sizes, <italic>n</italic>, correspond to cases that are able to effectively use a greater number of MPI ranks, <italic>P</italic>.</p>
<fig id="F3" position="float">
<label>Figure 3</label>
<caption><p>Strong scaling on Frontier and Crusher for 17 &#x000D7; 17 rod bundles with 10, 17, and 170 layers with total number of grid points of <italic>n</italic>= 95M, 161M, and 1.6B. Average time per step vs. rank, <italic>P</italic> <bold>(left)</bold>, and average time per step vs. <italic>n</italic>/<italic>P</italic> <bold>(right)</bold>. Frontier is set with (<monospace>cray-mpich/8.1.17</monospace>, <monospace>rocm/5.1.0</monospace>) and Crusher with (<monospace>cray-mpich/8.1.19</monospace>, <monospace>rocm/5.2.0</monospace>).</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0003.tif"/>
</fig>
<p>The critical observation is that these plots collapse to (nearly) a single curve when the independent variable is number of points per rank, <italic>n</italic>/<italic>P</italic>, which is evident in <xref ref-type="fig" rid="F3">Figure 3</xref>, right. This figure illustrates that the strong-scale performance is primarily a function of (<italic>n</italic>/<italic>P</italic>) and only weakly dependent on <italic>n</italic> or <italic>P</italic> individually, which is in accord with the extensive studies presented in Fischer et al. (<xref ref-type="bibr" rid="B12">2020</xref>) and analysis in Fischer et al. (<xref ref-type="bibr" rid="B8">2015</xref>). Based on this metric, we can estimate values of (<italic>n</italic>/<italic>P</italic>) for a given parallel efficiency and, from there, determine the number of MPI ranks required for a problem of size <italic>n</italic> to meet that expected efficiency. For example, an efficiency of &#x003B7;<sub><italic>P</italic></sub> &#x0003D; 0.80 is realized in this case for <italic>n</italic>/<italic>P</italic>&#x02248;2M&#x02013;3M. In the sequel, we explore performance behavior for a variety of problem sizes and platforms, including non-standard &#x0201C;large <italic>P</italic>&#x0201D; cases where one must consider both <italic>n</italic>/<italic>P</italic> and <italic>P</italic> to forecast performance.</p></sec>
<sec id="s4">
<title>4 NekRS performance on Summit, ThetaGPU, Perlmutter, Polaris, Crusher, and Frontier</title>
<p>In this section, we extend the scaling studies on the 17 &#x000D7; 17 rod bundle simulations to the NVIDIA-based GPU platforms, Summit (V100), ThetaGPU (A100), Perlmutter (A100), and Polaris (A100), and compare these with the AMD MI-250X platforms, Frontier and Crusher. We discuss the performance in detail in <xref ref-type="fig" rid="F4">Figures 4</xref>&#x02013;<xref ref-type="fig" rid="F6">6</xref>. In <xref ref-type="fig" rid="F5">Figure 5</xref> we include performance on ThetaGPU. We run one MPI rank per V100 or A100 on the NVIDIA-based nodes and one MPI rank per GCD on the AMD MI250X nodes.</p>
<fig id="F4" position="float">
<label>Figure 4</label>
<caption><p><bold>(A&#x02013;H)</bold> Strong-scaling on various GPU architectures for 17 &#x000D7; 17 rod bundle with 10 layers.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0004.tif"/>
</fig>
<fig id="F5" position="float">
<label>Figure 5</label>
<caption><p><bold>(A&#x02013;H)</bold> Strong-scaling on various GPU architectures for 17 &#x000D7; 17 rod bundle with 17 layers.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0005.tif"/>
</fig>
<fig id="F6" position="float">
<label>Figure 6</label>
<caption><p><bold>(A&#x02013;H)</bold> Strong-scaling on various GPU architectures for 17 &#x000D7; 17 rod bundle with 170 layers.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0006.tif"/>
</fig>
<p><xref ref-type="fig" rid="F4">Figures 4</xref>&#x02013;<xref ref-type="fig" rid="F6">6</xref> show <italic>t</italic><sub><italic>step</italic></sub> as a function of <italic>P</italic> in (a) and as a function of <italic>n</italic>/<italic>P</italic> in (b). Parallel efficiency as a function of <italic>P</italic> and of <italic>n</italic>/<italic>P</italic> is plotted in (c) and (d), respectively. Throughput, in terms of (GDOFS per step)/(time-per-step &#x000D7; <italic>P</italic>) is plotted versus time-per-step in (e) and vs. <italic>n</italic>/<italic>P</italic> in (f). The average time per step for the compute-intensive <monospace>makef</monospace> routine, which sets up the dealiased right-hand side of <xref ref-type="disp-formula" rid="E1">Equation 1</xref>, is plotted in (g), and the average time per step for the communication-intensive coarse-grid solve is plotted in (h).</p>
<p>We point out that these strong-scaling plots start from a high level of performance. NekRS currently leverages extensive tuning of several key FP64 and FP32 kernels in libParanumal, including the standard spectral element Laplacian matrix-vector product, local tensor-product solves using fast diagonalization, and dealiased evaluation of the advection operator on a finer set of quadrature points. These kernels are sustaining 3&#x02013;5 TFLOPS FP64 and 5&#x02013;8 TFLOPS FP32, per GPU or GCD. At the strong-scale limit, with MPI overhead, NekRS is sustaining &#x02248; 1 TFLOPS per rank (i.e., per A100 or GCD) for the full Navier&#x02013;Stokes solution (Merzari et al., <xref ref-type="bibr" rid="B19">2023</xref>). We also note that NekRS selects either host-based MPI or GPU-aware MPI for each kernel communication handle by making runtime tests to determine which option is fastest. We will refer to the GPU-aware MPI cases as GA-MPI in the legends and text.</p>
<p>From <xref ref-type="fig" rid="F4">Figure 4D</xref>, one can readily identify the key scalability metric, <italic>n</italic><sub>0.8</sub>, as the value of the <italic>n</italic>/<italic>P</italic> where the efficiency is 0.8 for the case <italic>n</italic> &#x0003D; 95M. Here we find <italic>n</italic><sub>0.8</sub> &#x0003C; 2M for Summit and GA-Polaris, <italic>n</italic><sub>0.8</sub> &#x0003D; 2M for Frontier, and <italic>n</italic><sub>0.8</sub> &#x0003D; 4M for Polaris without GA-MPI. We remark that the value of <italic>n</italic><sub>0.8</sub> is smaller for the relatively small values of <italic>n</italic> and <italic>P</italic> in <xref ref-type="fig" rid="F4">Figure 4</xref> than it is for the larger (<italic>n, P</italic>) pairs of <xref ref-type="fig" rid="F5">Figures 5</xref>, <xref ref-type="fig" rid="F6">6</xref>. One reason is that the coarse-grid costs rise relatively quickly when the processor count is low. Another is that the nearest-neighbor communication required for operator evaluation does not begin to saturate until <italic>P</italic>&#x0003E;26. Below this value, the domain partitioner will invariably assign each processor to a subdomain that is connected to the boundary. Once the problem is larger, more MPI ranks will have subdomains that are completely within the domain interior and will therefore have more communication. If the domain and decomposition were perfect tensor products, the smallest number of subdomains where communication saturates would be for <italic>P</italic> &#x0003D; 27. Even then, only one domain would be in the interior and it would generally not need to wait on neighboring subdomains to make their surface data available, as they would be underworked.</p>
<p><xref ref-type="fig" rid="F4">Figure 4E</xref> indicates that a remarkably small <italic>t</italic><sub><italic>step</italic></sub> value of 0.025 second is realizable on Polaris, albeit at 60% efficiency. <xref ref-type="fig" rid="F4">Figure 4G</xref> shows that the time in the advection update strong-scales quite well, as would be expected. The curves for the single GCD and A100 collapse to nearly the same performance while the older V100 technology of Summit is about 1.5 &#x000D7; slower. In the absence of communication, this kernel is sustaining 3&#x02013;4 TFLOPS FP64 on these newer architectures, although the graphs here do include the communication overhead. By contrast, <xref ref-type="fig" rid="F4">Figure 4H</xref> shows the performance for the communication-intensive coarse-grid solve, which is performed using <monospace>Hypre</monospace> on the host CPUs. Here, Crusher, Frontier, and Summit show relatively poor performance compared to Polaris and Perlmutter, although the performance also falls off for the latter once <italic>P</italic>&#x0003E;20, which corresponds to a local coarse-grid problem size of &#x02248;<italic>E</italic>/<italic>P</italic> = 277,000/20=13,850 unknowns per rank. Note that the coarse solve is completely executed on the hosts, so there is no performance difference with or without GPU-aware MPI.</p>
<p>As we increase the problem size to <italic>n</italic> &#x0003D; 161M (<xref ref-type="fig" rid="F5">Figure 5</xref>), the Polaris-GA efficiency curve coalesces with Perlmutter. We find <italic>n</italic><sub>0.8</sub> &#x0003D; 2M for Summit and <italic>n</italic><sub>0.8</sub>&#x02248;2.8M for Frontier, Crusher, Perlmutter, and Polaris-GA. ThetaGPU and Polaris without GA have <italic>n</italic><sub>0.8</sub> &#x0003D; 4.5M, which means that one can only use half the number of processors in these cases if one is targeting 80% efficiency in production runs, which in turn implies a two-fold increase in time-to-solution. In <xref ref-type="fig" rid="F5">Figure 5G</xref> we see that Summit is &#x02248;1.5 &#x000D7; slower in evaluating the nonlinear term (<monospace>makef</monospace>) than the other platforms and that, along with Crusher, it is slower for the host-based coarse-grid solves (h). At larger processor counts, Perlmutter also exhibits noise in the coarse-grid solve, as evidenced by the large oscillations in the strong-scale plot, (h).</p>
<p>Moving to the largest case of <italic>n</italic> &#x0003D; 1.6B, shown in <xref ref-type="fig" rid="F6">Figure 6</xref>, we see that Polaris exhibits system noise both with and without GA-MPI, as is clear from the parallel efficiency plots (c) and (d). From (h), we can see that the noise is due to the communication-intensive coarse-grid solve, which is susceptible to network congestion. Looking at efficiency vs. <italic>n</italic>/<italic>P</italic> (d), Frontier and Summit exhibit <italic>n</italic><sub>0.8</sub>&#x02248;2.8M&#x02013;3M, while the other platforms are clustered around <italic>n</italic><sub>0.8</sub> &#x0003D; 5M. If one is willing to accept &#x02248; 38% efficiency, per-step times as low as 0.025 s can be realized on Polaris.</p>
<p>The most crucial observation from <xref ref-type="fig" rid="F6">Figure 6</xref>, is that production runs at 80% efficiency will have roughly the time-to-solution on Frontier as on Polaris, despite the relatively high throughput (GDOFS) of Polaris that is evident in row 3, left. At <italic>n</italic><sub>0.8</sub> &#x0003D; 3M, Frontier has <italic>t</italic><sub><italic>step</italic></sub> &#x0003D; 0.065 s. For Polaris, at <italic>n</italic><sub>0.8</sub>&#x02248;4.7M, <italic>t</italic><sub><italic>step</italic></sub>&#x02248;0.066 s. The performance advantage of the A100 is offset by the increased <italic>n</italic><sub>0.8</sub> and both platforms exhibit the same time-to-solution. There is of course enough variance in these results, especially with respect to problem size, that either machine could end up with a demonstrable advantage under slightly different circumstances. The key point here, however, is to understand how large the problems are, per rank, under production circumstances and to build and optimize algorithms accordingly.</p></sec>
<sec sec-type="discussion" id="s5">
<title>5 Discussion</title>
<p>In this section we discuss a variety of &#x0201C;anomalous&#x0201D; behaviors encountered in these studies. By anomalous we mean adverse behaviors that either appear or disappear with software and hardware updates. One could argue that these are passing phenomena not worthy of reporting. However, as users work on these platforms it is important to understand potential pitfalls in system performance that might directly impact their own timing studies or production runtimes.</p>
<p>The behaviors described here include the performance of the large-memory (32 GB vs. 16 GB) nodes on Summit, the use of a nonmultiple of 8 ranks on Crusher, the influence of GPU-direct communication on Polaris, the upgrade from Slingshot 10 to Slingshot 11 on Perlmutter, and network interactions with GPU-aware MPI for large node counts on Frontier.</p>
<sec>
<title>5.1 Performance on Summit V100 16 GB vs. 32 GB</title>
<p>Most of the 4608 nodes on Summit have 16 GB of device memory, which limits how small one can take <italic>P</italic><sub>0</sub> in the efficiency definition (<xref ref-type="disp-formula" rid="E7">Equation 7</xref>). A few nodes, however, have 32 GB, which allow one to fit more points onto each V100. Unfortunately, as seen in <xref ref-type="fig" rid="F5">Figure 5</xref>, the Summit 32 GB curves perform about 10% slower than their 16 GB counterparts. The last row of graphs in <xref ref-type="fig" rid="F5">Figure 5</xref> is particularly revealing&#x02014;one can see from the <monospace>makef</monospace> plot that the V100s perform at the same rate for both the 16 GB and 32 GB nodes but that the host-based coarse-grid solve costs differ by almost 1.5 &#x000D7;, which indicates either an excessive device-host communication overhead or some type of interhost communication slowdown for the 32 GB nodes.</p>
</sec>
<sec>
<title>5.2 Performance on Crusher with rank dependency</title>
<p>In our timing studies we typically do not require that each node be fully occupied. This happens, for example, if one wants to make a device-to-device comparison between Summit, with six V100 per node, and Crusher, which has 8 GCDs per node. Our initial plot of Crusher timing data (not shown) appeared to be very erratic in its dependence on <italic>P</italic>. Closer inspection, however, revealed that the performance for mod(<italic>P</italic>, 8) &#x02260; 0 was about 2 &#x000D7; slower than for the case of mod(<italic>P</italic>, 8) = 0. Unlike the Summit behavior of the preceding section, this was clearly a device issue and not a host issue. Performance of the <monospace>makef</monospace> routine that implements <xref ref-type="disp-formula" rid="E1">Equation 1</xref> was 2 &#x000D7; slower if mod(<italic>P</italic>, 8) &#x02260; 0, but the host-based coarse-grid solve was the same for all values of mod(<italic>P</italic>, 8). This anomaly was ultimately resolved and did not appear as an issue on Frontier.</p>
</sec>
<sec>
<title>5.3 Performance on Polaris with GPU-aware MPI</title>
<p>As expected, communication overhead is more significant without GA-MPI, and consequently the <italic>no GA-MPI</italic> curves on Polaris show relatively poor performance as the amount of local work, <italic>n</italic>/<italic>P</italic>, is reduced. For example, in row 2, right, of <xref ref-type="fig" rid="F5">Figure 5</xref> the <italic>n</italic><sub>0.8</sub> for Polaris with GA-MPI is 2.5M, whereas it is 4.5M without GA-MPI. In the last row of the same figure we see that neither makef nor the coarse-grid solve are influenced by the presence or absence of GA-MPI. For makef, communication does not matter. The coarse-grid solve is communication intensive, but all of that communication originates from the host. We can also see in the results of <xref ref-type="fig" rid="F6">Figure 6</xref> that the no GA-MPI results are relatively noisy.</p>
</sec>
<sec>
<title>5.4 Performance on Perlmutter with Slingshot 10 vs. Slingshot 11</title>
<p>One other discovery was a sudden change in the behavior of Perlmutter at NERSC. Polaris and Perlmutter have similar architectures, so it was expected that they would have similar performance, as is indeed evident in, for example, <xref ref-type="fig" rid="F7">Figure 7</xref>. Later in our studies, however, the Perlmutter interconnect was upgraded from Slingshot 10 (SS10) to Slingshot 11 (SS11). The performance started to vary radically with <italic>P</italic>, albeit in a highly repeatable fashion, which indicated that the issue was <italic>not</italic> network noise.</p>
<fig id="F7" position="float">
<label>Figure 7</label>
<caption><p>Strong-scaling on Perlmutter with runs using SS10 and SS11 on different days for 17 &#x000D7; 17 rod bundle simulations with 10 layers.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0007.tif"/>
</fig>
<p>The strong-scaling plot in <xref ref-type="fig" rid="F7">Figure 7</xref> illustrates the problem for the <italic>n</italic> &#x0003D; 95M 17 &#x000D7; 17 rod-bundle case of <xref ref-type="fig" rid="F4">Figure 4</xref>. For <italic>P</italic> &#x0003D; 48 ranks, the Navier&#x02013;Stokes runtime with SS11 jumps by a factor of 3 over that with SS10. These were repeatable results, as evidenced by timings over a period of several months. Our timing profiles indicated that the anomaly was neither a GPU issue, since <monospace>makef</monospace> timings were essentially identical for SS10 and SS11, nor a host-to-host communication issue, since the coarse-grid times were also nearly the same. Inspection of the NekRS profiles showed that the SS11 timing increase was focused in the non-local Schwarz-based smoother for the <italic>p</italic>-multigrid preconditioner of the pressure Poisson problem, which was running 10 &#x000D7; slower than its SS10 counterpart. Below, we show the profiles for the two simulations with <italic>P</italic> &#x0003D; 48 (which was the slowest case). Remarkably, the issue was found to be related to short messages, since it arose only at the coarsest levels for the GPU-based communication of the <italic>p</italic>-multigrid solver, which was employing 32-bit reals.</p>
<p>Fortunately, a later release of SS11 restored the expected performance of the SS11 network, with minimal variance over a wide range of processor counts, as seen by the gold dashed-dot line in <xref ref-type="fig" rid="F7">Figure 7</xref>. For <italic>P</italic> &#x0003D; 6, the 1.5 &#x000D7; gain in the SS11 bandwidth is manifest as an 11% Navier-Stokes performance gain over SS10. For larger values of <italic>P</italic>, SS11 continues to offer performance benefits over SS10, but the gains are somewhat diminshed because the messages are smaller and hence tend to be latency-bound rather than bandwidth bound.</p>
<boxed-text>
<p><bold>SS10 profile:</bold></p>
<preformat>&#x000A0;&#x000A0;&#x000A0;name&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;time&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;%&#x000A0;&#x000A0;calls
&#x000A0;&#x000A0;&#x000A0;&#x000A0;setup&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.82904e&#x0002B;01s&#x000A0;&#x000A0;0.38&#x000A0;&#x000A0;1
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;loadKernels&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.03634e&#x0002B;01s&#x000A0;&#x000A0;0.27&#x000A0;&#x000A0;1
&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfExecuteStep&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.79398e-03s&#x000A0;&#x000A0;0.00&#x000A0;&#x000A0;2001
&#x000A0;&#x000A0;&#x000A0;&#x000A0;elapsedStepSum&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;6.13724e&#x0002B;01s&#x000A0;&#x000A0;0.62
&#x000A0;&#x000A0;&#x000A0;&#x000A0;solve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;6.12031e&#x0002B;01s&#x000A0;&#x000A0;0.61
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;min&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.31879e-02s
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;max&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.51687e-02s
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;flop/s&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.36729e&#x0002B;13
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;makef&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.59237e&#x0002B;00s&#x000A0;&#x000A0;0.09&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfUEqnSource&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.98969e-02s&#x000A0;&#x000A0;0.01&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfProperties&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.82886e-03s&#x000A0;&#x000A0;0.00&#x000A0;&#x000A0;2001
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;velocitySolve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.73346e&#x0002B;01s&#x000A0;&#x000A0;0.28&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;rhs&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.29362e&#x0002B;00s&#x000A0;&#x000A0;0.13&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;pressureSolve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.42052e&#x0002B;01s&#x000A0;&#x000A0;0.56&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;rhs&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.69203e&#x0002B;00s&#x000A0;&#x000A0;0.14&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;preconditioner&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.26178e&#x0002B;01s&#x000A0;&#x000A0;0.66&#x000A0;&#x000A0;2470
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;pMG&#x000A0;smoother&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.51609e&#x0002B;01s&#x000A0;&#x000A0;0.67&#x000A0;&#x000A0;9880
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;coarse&#x000A0;grid&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.33568e&#x0002B;00s&#x000A0;&#x000A0;0.24&#x000A0;&#x000A0;2470
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;initial&#x000A0;guess&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.18958e&#x0002B;00s&#x000A0;&#x000A0;0.09&#x000A0;&#x000A0;2000
</preformat>
</boxed-text>
<boxed-text>
<p><bold>SS11 profile:</bold></p>
<preformat>&#x000A0;&#x000A0;&#x000A0;name&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;time&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;%&#x000A0;&#x000A0;calls
&#x000A0;&#x000A0;&#x000A0;&#x000A0;setup&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.98696e&#x0002B;01s&#x000A0;&#x000A0;0.16&#x000A0;&#x000A0;1
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;loadKernels&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;8.86541e&#x0002B;00s&#x000A0;&#x000A0;0.22&#x000A0;&#x000A0;1
&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfExecuteStep&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.79946e-03s&#x000A0;&#x000A0;0.00&#x000A0;&#x000A0;2001
&#x000A0;&#x000A0;&#x000A0;&#x000A0;elapsedStepSum&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.06042e&#x0002B;02s&#x000A0;&#x000A0;0.84
&#x000A0;&#x000A0;&#x000A0;&#x000A0;solve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.05867e&#x0002B;02s&#x000A0;&#x000A0;0.84
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;min&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.50540e-02s
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;max&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.32500e-01s
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;flop/s&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.00108e&#x0002B;13
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;makef&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.57575e&#x0002B;00s&#x000A0;&#x000A0;0.03&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfUEqnSource&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.99624e-02s&#x000A0;&#x000A0;0.01&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;udfProperties&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.88246e-03s&#x000A0;&#x000A0;0.00&#x000A0;&#x000A0;2001
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;velocitySolve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.72489e&#x0002B;01s&#x000A0;&#x000A0;0.08&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;rhs&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;2.29522e&#x0002B;00s&#x000A0;&#x000A0;0.13&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;pressureSolve&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.79243e&#x0002B;02s&#x000A0;&#x000A0;0.87&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;rhs&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;4.48683e&#x0002B;00s&#x000A0;&#x000A0;0.03&#x000A0;&#x000A0;2000
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;preconditioner&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.67813e&#x0002B;02s&#x000A0;&#x000A0;0.94&#x000A0;&#x000A0;2470
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;pMG&#x000A0;smoother&#x000A0;&#x000A0;&#x000A0;&#x000A0;1.49445e&#x0002B;02s&#x000A0;&#x000A0;0.89&#x000A0;&#x000A0;9880
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;coarse&#x000A0;grid&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;5.53950e&#x0002B;00s&#x000A0;&#x000A0;0.03&#x000A0;&#x000A0;2470
&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;initial&#x000A0;guess&#x000A0;&#x000A0;&#x000A0;&#x000A0;&#x000A0;3.20173e&#x0002B;00s&#x000A0;&#x000A0;0.02&#x000A0;&#x000A0;2000
</preformat>
</boxed-text>
<p>For completeness, we also include the kernel performance numbers as reported in the NekRS logfiles:</p>
<boxed-text>
<p><bold>SS10 profile:</bold></p>
<preformat>&#x000A0;&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP64&#x000A0;GDOF/s=13.2&#x000A0;GB/s=1260&#x000A0;GFLOPS=2184&#x000A0;kv0
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP64&#x000A0;GDOF/s=13.2&#x000A0;GB/s=1260&#x000A0;GFLOPS=2183&#x000A0;kv0
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=3&#x000A0;FP64&#x000A0;GDOF/s=12.6&#x000A0;GB/s=1913&#x000A0;GFLOPS=1883&#x000A0;kv5
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP32&#x000A0;GDOF/s=25.0&#x000A0;GB/s=1194&#x000A0;GFLOPS=4145&#x000A0;kv4
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=3&#x000A0;FP32&#x000A0;GDOF/s=18.0&#x000A0;GB/s=1368&#x000A0;GFLOPS=2693&#x000A0;kv2
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;fdm:&#x000A0;N=9&#x000A0;FP32&#x000A0;GDOF/s=44.9&#x000A0;GB/s=&#x000A0;812&#x000A0;GFLOPS=7452&#x000A0;kv4
&#x000A0;&#x000A0;fdm:&#x000A0;N=5&#x000A0;FP32&#x000A0;GDOF/s=34.1&#x000A0;GB/s=&#x000A0;825&#x000A0;GFLOPS=4301&#x000A0;kv1
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;flop/s&#x000A0;&#x000A0;3.36729e&#x0002B;13&#x000A0;&#x000A0;(701&#x000A0;GFLOPS/rank)
</preformat>
</boxed-text>
<boxed-text>
<p><bold>SS11 profile:</bold></p>
<preformat>&#x000A0;&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP64&#x000A0;GDOF/s=13.2&#x000A0;GB/s=1256&#x000A0;GFLOPS=2179&#x000A0;kv0
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP64&#x000A0;GDOF/s=13.2&#x000A0;GB/s=1257&#x000A0;GFLOPS=2180&#x000A0;kv0
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=3&#x000A0;FP64&#x000A0;GDOF/s=12.6&#x000A0;GB/s=1912&#x000A0;GFLOPS=1882&#x000A0;kv5
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=7&#x000A0;FP32&#x000A0;GDOF/s=25.0&#x000A0;GB/s=1194&#x000A0;GFLOPS=4144&#x000A0;kv5
&#x000A0;&#x000A0;&#x000A0;Ax:&#x000A0;N=3&#x000A0;FP32&#x000A0;GDOF/s=18.1&#x000A0;GB/s=1369&#x000A0;GFLOPS=2696&#x000A0;kv2
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;fdm:&#x000A0;N=9&#x000A0;FP32&#x000A0;GDOF/s=44.9&#x000A0;GB/s=&#x000A0;812&#x000A0;GFLOPS=7444&#x000A0;kv4
&#x000A0;&#x000A0;fdm:&#x000A0;N=5&#x000A0;FP32&#x000A0;GDOF/s=34.1&#x000A0;GB/s=&#x000A0;825&#x000A0;GFLOPS=4303&#x000A0;kv1
&#x000A0;&#x000A0;
&#x000A0;&#x000A0;flop/s&#x000A0;&#x000A0;1.00108e&#x0002B;13&#x000A0;&#x000A0;(208&#x000A0;GFLOPS/rank)
</preformat>
</boxed-text>
<p>In SS10 profile and SS11 profile, <monospace>kv</monospace> reflects the particular kernel version chosen out of the suite of available kernels in NekRS for the particular operation. We see that the 64-bit <monospace>Ax</monospace> kernels (the matrix-vector product with the Laplace operator for spectral element order <italic>N</italic>) realize &#x02248;2 TFLOPS per device, while their 32-bit counterparts realize 3&#x02013;4 TFLOPS (32-bit arithmetic is used in the preconditioner only). The <monospace>fdm</monospace> kernel implements the fast-diagonalization method described earlier for the overlapping Schwarz preconditioners. This is a fast operation and can be seen to sustain &#x0003E; 7 GFLOPS (FP32). Note that, as expected, the kernel performance, which does not include any MPI overhead, is not dependent on the Slingshot version. NekRS also reports the total observed GFLOPS, which is seen to be 701 GFLOPS/rank for SS10 and 208 GFLOPS/rank for SS11 with <italic>P</italic> &#x0003D; 48, prior to the bug fix.</p>
</sec>
<sec>
<title>5.5 Network issues for large-<italic>P</italic> runs on Frontier</title>
<p>Our largest runs to date have been on 72,000 MPI ranks of Frontier (Merzari et al., <xref ref-type="bibr" rid="B19">2023</xref>). Unfortunately, above several thousand nodes, the Frontier network exhibited erratic behavior such that the sustained performance for the default version of NekRS dropped to &#x02248;150 GFLOPS/rank. The performance is illustrated in <xref ref-type="fig" rid="F8">Figure 8</xref> (left), which shows the NekRS time-per-step on 9000 nodes (<italic>P</italic> &#x0003D; 72000) for a simulation with 500 layers of the full-core geometry of <xref ref-type="fig" rid="F1">Figure 1</xref> (left). The fluid domain had <italic>n</italic> &#x0003D; 172B gridpoints while the solid domain (for conduction in the individual rods) had <italic>n</italic><sub><italic>s</italic></sub> &#x0003D; 198B gridpoints. In <xref ref-type="fig" rid="F8">Figure 8</xref> (left), we see that a lower bound time-per-step of <italic>t</italic><sub>step</sub>&#x02248;0.3<italic>s</italic>, which is the performance level we would expect, and a cloud of points above that with <italic>t</italic><sub>step</sub> as high as 10s, which is &#x0003E;30 &#x000D7; the expected value. Testing for these full-scale runs was challenging because of system stability issues. The limited tests that were feasible, however, indicated that the significant variance arose from from flooding the system network during the relatively simple (and normally fast) Jacobi-preconditioned Helmholtz solves <xref ref-type="disp-formula" rid="E5">Equation 5</xref> for velocity. Tests indicated that the flooding could be mitigated by moving from block (<italic>uvw</italic>) solves in the velocity to scalar (<italic>u</italic>, <italic>v</italic>, <italic>w</italic>) solves for each component (i.e., smaller messages instead of fewer large messages). It was found however, that among the limited trials that time permitted, turning off GPU-direct and routing the messages through the host was the better option, with results illustrated <xref ref-type="fig" rid="F8">Figure 8</xref> (right) for an 800-layer case that sustains 390 GFLOPS per rank for <italic>P</italic> &#x0003D; 72000.</p>
<fig id="F8" position="float">
<label>Figure 8</label>
<caption><p>Navier-Stokes time-per-step for <italic>P</italic> &#x0003D; 72, 000 runs on Frontier: <bold>(left)</bold> 500-layer full-core configuration of <xref ref-type="fig" rid="F1">Figure 1</xref> with GPU-aware MPI; <bold>(right)</bold> 800-layer case without GPU-aware MPI.</p></caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fhpcp-02-1303358-g0008.tif"/>
</fig>
</sec>
</sec>
<sec sec-type="conclusions" id="s6">
<title>6 Conclusions</title>
<p>We explored the performance of a highly-tuned incompressible flow code, NekRS, on current-generation HPC architectures featuring accelerator-based nodes. The principal accelerators under consideration are the NVIDIA V100, NVIDIA A100, and MI250X with eight GCDs per node. We found that the raw performance of a single GCD is about 85% of a single A100. We also found that the AMD-based Crusher platform had slower host-based communication than its Polaris/Perlmutter counterpart, as witnessed by the relatively poor performance of the Hypre-based coarse-grid solver, which runs on the host. This situation is significantly improved on Frontier.</p>
<p>We further identified that, despite having lower throughput, Frontier had better scalability (i.e., lower <italic>n</italic><sub>0.8</sub>) than the NVIDIA A100 platforms Perlmutter and Polaris, which led to Frontier having a slight advantage in time-to-solution (although the advantage was arguably within the variance arising from system noise).</p>
<p>A key finding was that <italic>n</italic><sub>0.8</sub> &#x0003D; 2M&#x02013;5M are typical values of <italic>n</italic>/<italic>P</italic> required to sustain 80% parallel efficiency on DOE&#x00027;s current set of leadership HPC platforms. This number is of interest for algorithmic (and hardware?) optimization because this will be the most likely operating point for production users. Running with <italic>n</italic>/<italic>P</italic>&#x0226B;<italic>n</italic><sub>0.8</sub> will result in significant slow-down. Running with <italic>n</italic>/<italic>P</italic>&#x0226A;<italic>n</italic><sub>0.8</sub> will result in significant waste of an allocation. Moreover, under these production-run conditions, time-to-solution (here, <italic>t</italic><sub><italic>step</italic></sub>) depends strongly on <italic>n</italic>/<italic>P</italic> and only weakly on <italic>n</italic> or <italic>P</italic> individually. As a consequence, one can predict <italic>t</italic><sub><italic>step</italic></sub> with reasonable fidelity for any problem size <italic>n</italic> provided it does not saturate the full machine (meaning <italic>n</italic>/<italic>P</italic>&#x0003C;<italic>n</italic><sub>0.8</sub>, where <italic>P</italic> is the maximum number of processes available).</p>
<p>Finally, several examples of anomalous behavior that were encountered during the study were identified and discussed, and ultimately resolved either by system upgrades or by altering execution paths to avoid the source of difficulty.</p></sec>
</body>
<back>
<sec sec-type="data-availability" id="s7">
<title>Data availability statement</title>
<p>Publicly available datasets were analyzed in this study. This data can be found here: <ext-link ext-link-type="uri" xlink:href="https://github.com/Nek5000/nekRS">https://github.com/Nek5000/nekRS</ext-link>.</p>
</sec>
<sec sec-type="author-contributions" id="s8">
<title>Author contributions</title>
<p>MM: Conceptualization, Data curation, Formal analysis, Funding acquisition, Investigation, Methodology, Project administration, Resources, Software, Supervision, Validation, Visualization, Writing &#x02013; original draft, Writing &#x02013; review &#x00026; editing. Y-HL: Data curation, Investigation, Methodology, Software, Validation, Visualization, Writing &#x02013; review &#x00026; editing. PF: Conceptualization, Formal analysis, Funding acquisition, Investigation, Methodology, Project administration, Resources, Software, Supervision, Validation, Writing &#x02013; original draft, Writing &#x02013; review &#x00026; editing. TR: Investigation, Software, Validation, Writing &#x02013; review &#x00026; editing. JH: Data curation, Investigation, Resources, Software, Writing &#x02013; review &#x00026; editing.</p>
</sec>
<sec sec-type="funding-information" id="s9">
<title>Funding</title>
<p>The author(s) declare financial support was received for the research, authorship, and/or publication of this article. This material was based upon work supported by the U.S. Department of Energy, Office of Science, under contract DE-AC02-06CH11357 and by the Exascale Computing Project (17-SC-20-SC). The Oak Ridge Leadership Computing Facility at the Oak Ridge National Laboratory was supported by the Office of Science of the U.S. DOE under Contract No. DE-AC05-00OR22725. The Argonne Leadership Computing Facility at Argonne National Laboratory was supported by the Office of Science of the U.S. DOE under Contract No. DE-AC02-06CH11357.</p>
</sec>
<ack><p>This research used resources at the Argonne and the Oak Ridge Leadership Computing Facility. This research also used resources of the National Energy Research Scientific Computing Center, a DOE Office of Science User Facility using NERSC award ALCC-ERCAP0030677.</p>
</ack>
<sec sec-type="COI-statement" id="conf1">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec sec-type="disclaimer" id="s10">
<title>Publisher&#x00027;s note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<fn-group>
<fn id="fn0001"><p><sup>1</sup>We prefer dofs per second per rank because AMD&#x00027;s MI250X has two compute units (GCDs) per GPU and Aurora&#x00027;s Intel has two tiles per PVC&#x02014;users view these as two processors</p></fn>
<fn id="fn0002"><p><sup>2</sup>We typically take the average time over several hundreds or thousands timesteps.</p></fn>
<fn id="fn0003"><p><sup>3</sup>For fluid flow simulations, some authors set <italic>n</italic> to be 4 times the number of grid points because there are typically three velocity components and one pressure unknown at each grid point. We prefer to take <italic>n</italic> to be the number of grid points. The problems are generally large enough that we do not need to distinguish between interior points where the solution is unknown and surface points where boundary data is prescribed. We simply set <italic>n</italic> to represent the union of these sets given that some quantities might have Neumann conditions on the boundary while others have Dirichlet.</p></fn>
<fn id="fn0004"><p><sup>4</sup>NekRS version 22.0 is used.</p></fn>
</fn-group>
<ref-list>
<title>References</title>
<ref id="B1">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Abdelfattah</surname> <given-names>A.</given-names></name> <name><surname>Barra</surname> <given-names>V.</given-names></name> <name><surname>Beams</surname> <given-names>N.</given-names></name> <name><surname>Bleile</surname> <given-names>R.</given-names></name> <name><surname>Brown</surname> <given-names>J.</given-names></name> <name><surname>Camier</surname> <given-names>J.-S.</given-names></name> <etal/></person-group>. (<year>2021</year>). <article-title>GPU algorithms for efficient exascale discretizations</article-title>. <source>Par. Comput</source>. <volume>108</volume>:<fpage>102841</fpage>. <pub-id pub-id-type="doi">10.1016/j.parco.2021.102841</pub-id></citation>
</ref>
<ref id="B2">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Anderson</surname> <given-names>R.</given-names></name> <name><surname>Andrej</surname> <given-names>J.</given-names></name> <name><surname>Barker</surname> <given-names>A.</given-names></name> <name><surname>Bramwell</surname> <given-names>J.</given-names></name> <name><surname>Camier</surname> <given-names>J.-S.</given-names></name> <name><surname>Dobrev</surname> <given-names>J. C. V.</given-names></name> <etal/></person-group>. (<year>2020</year>). <article-title>MFEM: A modular finite element library</article-title>. <source>Comp. Mathemat. Appl</source>. <volume>81</volume>, <fpage>42</fpage>&#x02013;<lpage>74</lpage>. <pub-id pub-id-type="doi">10.1016/j.camwa.2020.06.009</pub-id></citation>
</ref>
<ref id="B3">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Arndt</surname> <given-names>D.</given-names></name> <name><surname>Bangerth</surname> <given-names>W.</given-names></name> <name><surname>Davydov</surname> <given-names>D.</given-names></name> <name><surname>Heister</surname> <given-names>T.</given-names></name> <name><surname>Heltai</surname> <given-names>L.</given-names></name> <name><surname>Kronbichler</surname> <given-names>M.</given-names></name> <etal/></person-group>. (<year>2017</year>). <article-title>The <monospace>deal.II</monospace> library, version 8.5</article-title>. <source>J. Num. Math</source>. <volume>25</volume>, <fpage>137</fpage>&#x02013;<lpage>145</lpage>. <pub-id pub-id-type="doi">10.1515/jnma-2017-0058</pub-id></citation>
</ref>
<ref id="B4">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Bienz</surname> <given-names>A.</given-names></name> <name><surname>OLson</surname> <given-names>L.</given-names></name> <name><surname>Gropp</surname> <given-names>W. D.</given-names></name> <name><surname>Lockhart</surname> <given-names>S.</given-names></name></person-group> (<year>2021</year>). <article-title>&#x0201C;Modeling data movement performance on heterogeneous architectures,&#x0201D;</article-title> in <source>IEEE High Performance Extreme Computing Conference (HPEC)</source> (<publisher-loc>Waltham, MA</publisher-loc>: <publisher-name>IEEE</publisher-name>), <fpage>1</fpage>&#x02013;<lpage>7</lpage>.</citation>
</ref>
<ref id="B5">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chalmers</surname> <given-names>N.</given-names></name> <name><surname>Karakus</surname> <given-names>A.</given-names></name> <name><surname>Austin</surname> <given-names>A. P.</given-names></name> <name><surname>Swirydowicz</surname> <given-names>K.</given-names></name> <name><surname>Warburton</surname> <given-names>T.</given-names></name></person-group> (<year>2020</year>). <source>libParanumal: A Performance Portable High-Order Finite Element Library</source>. <pub-id pub-id-type="doi">10.5281/zenodo.4004744</pub-id></citation>
</ref>
<ref id="B6">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Chalmers</surname> <given-names>N.</given-names></name> <name><surname>Mishra</surname> <given-names>A.</given-names></name> <name><surname>McDougall</surname> <given-names>D.</given-names></name> <name><surname>Warburton</surname> <given-names>T.</given-names></name></person-group> (<year>2023</year>). <article-title>HipBone: a performance-portable graphics processing unit-accelerated C&#x0002B;&#x0002B; version of the NekBone benchmark</article-title>. <source>Int. J. High Perf. Comput. Appl</source>. <volume>37</volume>, <fpage>560</fpage>&#x02013;<lpage>577</lpage>. <pub-id pub-id-type="doi">10.1177/10943420231178552</pub-id></citation>
</ref>
<ref id="B7">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Deville</surname> <given-names>M.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Mund</surname> <given-names>E.</given-names></name></person-group> (<year>2002</year>). <source>High-Order Methods for Incompressible Fluid Flow</source>. <publisher-loc>Cambridge</publisher-loc>: <publisher-name>Cambridge University Press</publisher-name>.</citation>
</ref>
<ref id="B8">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Heisey</surname> <given-names>K.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name></person-group> (<year>2015</year>). <article-title>&#x0201C;Scaling limits for PDE-based simulation (invited),&#x0201D;</article-title> in <source>22nd AIAA Computational Fluid Dynamics Conference, AIAA Aviation</source> (<publisher-loc>Dallas, TX</publisher-loc>).</citation>
</ref>
<ref id="B9">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Kerkemeier</surname> <given-names>S.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Lan</surname> <given-names>Y.</given-names></name> <name><surname>Phillips</surname> <given-names>M.</given-names></name> <name><surname>Rathnayake</surname> <given-names>T.</given-names></name> <etal/></person-group>. (<year>2021</year>). <article-title>NekRS, a GPU-accelerated spectral element Navier-Stokes solver</article-title>. <source>arXiv</source> [preprint] arXiv:2104.05829. <pub-id pub-id-type="doi">10.48550/arXiv.2104.05829</pub-id></citation>
</ref>
<ref id="B10">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Kerkemeier</surname> <given-names>S.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Lan</surname> <given-names>Y.-H.</given-names></name> <name><surname>Phillips</surname> <given-names>M.</given-names></name> <name><surname>Rathnayake</surname> <given-names>T.</given-names></name> <etal/></person-group>. (<year>2022</year>). <article-title>NekRS, a GPU-accelerated spectral element Navier&#x02013;Stokes solver</article-title>. <source>Par. Comput</source>. <volume>114</volume>:<fpage>102982</fpage>. <pub-id pub-id-type="doi">10.1016/j.parco.2022.102982</pub-id></citation>
</ref>
<ref id="B11">
<citation citation-type="web"><person-group person-group-type="author"><name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Lottes</surname> <given-names>J.</given-names></name> <name><surname>Kerkemeier</surname> <given-names>S.</given-names></name></person-group> (<year>2008</year>). <source>Nek5000: Open source spectral element CFD solver</source>. Available at: <ext-link ext-link-type="uri" xlink:href="http://nek5000.mcs.anl.gov">http://nek5000.mcs.anl.gov</ext-link>; <ext-link ext-link-type="uri" xlink:href="https://github.com/nek5000/nek5000">https://github.com/nek5000/nek5000</ext-link></citation>
</ref>
<ref id="B12">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Rathnayake</surname> <given-names>T.</given-names></name> <name><surname>Dutta</surname> <given-names>S.</given-names></name> <name><surname>Kolev</surname> <given-names>T.</given-names></name> <name><surname>Dobrev</surname> <given-names>V.</given-names></name> <etal/></person-group>. (<year>2020</year>). <article-title>Scalability of high-performance PDE solvers</article-title>. <source>Int. J. of High Perf. Comp. Appl</source>. <volume>34</volume>, <fpage>562</fpage>&#x02013;<lpage>586</lpage>. <pub-id pub-id-type="doi">10.1177/1094342020915762</pub-id><pub-id pub-id-type="pmid">37679152</pub-id></citation></ref>
<ref id="B13">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kolev</surname> <given-names>T.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Dongarra</surname> <given-names>J.</given-names></name> <name><surname>Brown</surname> <given-names>J.</given-names></name> <name><surname>Dobrev</surname> <given-names>V.</given-names></name> <etal/></person-group>. (<year>2021</year>). <article-title>Efficient exascale discretizations</article-title>. <source>Int. J. of High Perf. Comp. Appl</source>. <volume>35</volume>, <fpage>527</fpage>&#x02013;<lpage>552</lpage>. <pub-id pub-id-type="doi">10.1177/1094342021102080</pub-id></citation>
</ref>
<ref id="B14">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kronbichler</surname> <given-names>M.</given-names></name> <name><surname>Ljungkvist</surname> <given-names>K.</given-names></name></person-group> (<year>2019</year>). <article-title>Multigrid for matrix-free high-order finite element computations on graphics processors</article-title>. <source>ACM Trans. on Par. Comp</source>. <volume>6</volume>, <fpage>1</fpage>&#x02013;<lpage>32</lpage>. <pub-id pub-id-type="doi">10.1145/3322813</pub-id></citation>
</ref>
<ref id="B15">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Kronbichler</surname> <given-names>M.</given-names></name> <name><surname>Sashko</surname> <given-names>D.</given-names></name> <name><surname>Munch</surname> <given-names>P.</given-names></name></person-group> (<year>2023</year>). <article-title>Enhancing data locality of the conjugate gradient method for high-order matrix-free finite-element implementations</article-title>. <source>Int. J. High Perf. Comput. Appl</source>. <volume>37</volume>, <fpage>61</fpage>&#x02013;<lpage>81</lpage>. <pub-id pub-id-type="doi">10.1177/10943420221107880</pub-id></citation>
</ref>
<ref id="B16">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Maday</surname> <given-names>Y.</given-names></name> <name><surname>Patera</surname> <given-names>A.</given-names></name> <name><surname>R&#x000F8;nquist</surname> <given-names>E.</given-names></name></person-group> (<year>1990</year>). <article-title>An operator-integration-factor splitting method for time-dependent problems: application to incompressible fluid flow</article-title>. <source>J. Sci. Comput</source>. <volume>5</volume>, <fpage>263</fpage>&#x02013;<lpage>292</lpage>. <pub-id pub-id-type="doi">10.1007/BF01063118</pub-id></citation>
</ref>
<ref id="B17">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Malm</surname> <given-names>J.</given-names></name> <name><surname>Schlatter</surname> <given-names>P.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Henningson</surname> <given-names>D.</given-names></name></person-group> (<year>2013</year>). <article-title>Stabilization of the spectral-element method in convection dominated flows by recovery of skew symmetry</article-title>. <source>J. Sci. Comp</source>. <volume>57</volume>, <fpage>254</fpage>&#x02013;<lpage>277</lpage>. <pub-id pub-id-type="doi">10.1007/s10915-013-9704-1</pub-id></citation>
</ref>
<ref id="B18">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Medina</surname> <given-names>D. S.</given-names></name> <name><surname>St-Cyr</surname> <given-names>A.</given-names></name> <name><surname>Warburton</surname> <given-names>T.</given-names></name></person-group> (<year>2014</year>). <article-title>OCCA: A unified approach to multi-threading languages</article-title>. <source>arXiv</source> [preprint] arXiv:1403.0968. <pub-id pub-id-type="doi">10.48550/arXiv.1403.0968</pub-id></citation>
</ref>
<ref id="B19">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Merzari</surname> <given-names>E.</given-names></name> <name><surname>Hamilton</surname> <given-names>S.</given-names></name> <name><surname>Evans</surname> <given-names>T.</given-names></name> <name><surname>Romano</surname> <given-names>P.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <etal/></person-group>. (<year>2023</year>). <article-title>&#x0201C;Exascale multiphysics nuclear reactor simulations for advanced designs (Gordon Bell Prize Finalist paper),&#x0201D;</article-title> in <source>Proc. of SC23: Int. Conf. for High Performance Computing, Networking, Storage and Analysis</source> (<publisher-loc>downtown Denver</publisher-loc>: <publisher-name>IEEE</publisher-name>).</citation>
</ref>
<ref id="B20">
<citation citation-type="book"><person-group person-group-type="author"><name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Lan</surname> <given-names>Y.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Merzari</surname> <given-names>E.</given-names></name> <name><surname>Kerkemeier</surname> <given-names>S.</given-names></name> <name><surname>Phillips</surname> <given-names>M.</given-names></name> <etal/></person-group>. (<year>2022</year>). <article-title>&#x0201C;Optimization of full-core reactor simulations on Summit,&#x0201D;</article-title> in <source>Proc. of SC22: Int. Conf. for High Performance Computing, Networking, Storage and Analysis</source> (<publisher-loc>Dallas, TX</publisher-loc>: <publisher-name>IEEE</publisher-name>).</citation>
</ref>
<ref id="B21">
<citation citation-type="web"><person-group person-group-type="author"><collab>OCCA</collab></person-group> (<year>2021</year>). <source>OCCA: Git Repository</source>. Available at: <ext-link ext-link-type="uri" xlink:href="https://github.com/libocca/occa">https://github.com/libocca/occa</ext-link></citation>
</ref>
<ref id="B22">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Orszag</surname> <given-names>S.</given-names></name></person-group> (<year>1980</year>). <article-title>Spectral methods for problems in complex geometry</article-title>. <source>J. Comput. Phys</source>. <volume>37</volume>:<fpage>70</fpage>&#x02013;<lpage>92</lpage>. <pub-id pub-id-type="doi">10.1016/0021-9991(80)90005-4</pub-id></citation>
</ref>
<ref id="B23">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Patel</surname> <given-names>S.</given-names></name> <name><surname>Fischer</surname> <given-names>P.</given-names></name> <name><surname>Min</surname> <given-names>M.</given-names></name> <name><surname>Tomboulides</surname> <given-names>A.</given-names></name></person-group> (<year>2019</year>). <article-title>A characteristic-based, spectral element method for moving-domain problems</article-title>. <source>J. Sci. Comp</source>. <volume>79</volume>, <fpage>564</fpage>&#x02013;<lpage>592</lpage>. <pub-id pub-id-type="doi">10.1007/s10915-018-0876-6</pub-id></citation>
</ref>
<ref id="B24">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Patera</surname> <given-names>A.</given-names></name></person-group> (<year>1984</year>). <article-title>A spectral element method for fluid dynamics : laminar flow in a channel expansion</article-title>. <source>J. Comput. Phys</source>. <volume>54</volume>, <fpage>468</fpage>&#x02013;<lpage>488</lpage>. <pub-id pub-id-type="doi">10.1016/0021-9991(84)90128-1</pub-id></citation>
</ref>
<ref id="B25">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Pothen</surname> <given-names>A.</given-names></name> <name><surname>Simon</surname> <given-names>H.</given-names></name> <name><surname>Liou</surname> <given-names>K.</given-names></name></person-group> (<year>1990</year>). <article-title>Partitioning sparse matrices with eigenvectors of graphs</article-title>. <source>SIAM J. Matrix Anal. Appl</source>. <volume>11</volume>, <fpage>430</fpage>&#x02013;<lpage>452</lpage>. <pub-id pub-id-type="doi">10.1137/0611030</pub-id><pub-id pub-id-type="pmid">33246128</pub-id></citation></ref>
<ref id="B26">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Sathyanarayana</surname> <given-names>S.</given-names></name> <name><surname>Bernardini</surname> <given-names>M.</given-names></name> <name><surname>Modesti</surname> <given-names>D.</given-names></name> <name><surname>Pirozzolia</surname> <given-names>S.</given-names></name> <name><surname>Salvadore</surname> <given-names>F.</given-names></name></person-group> (<year>2025</year>). <article-title>High-speed turbulent flows towards the exascale: STREAmS-2 porting and performance</article-title>. <source>J. Par. Dist. Comput</source>. <volume>196</volume>:<fpage>104993</fpage>. <pub-id pub-id-type="doi">10.1016/j.jpdc.2024.104993</pub-id></citation>
</ref>
<ref id="B27">
<citation citation-type="journal"><person-group person-group-type="author"><name><surname>Vargas</surname> <given-names>A.</given-names></name> <name><surname>Stitt</surname> <given-names>T.</given-names></name> <name><surname>Weiss</surname> <given-names>K.</given-names></name> <name><surname>Tomov</surname> <given-names>V.</given-names></name> <name><surname>Camier</surname> <given-names>J.</given-names></name> <name><surname>Kolev</surname> <given-names>T.</given-names></name> <etal/></person-group>. (<year>2022</year>). <article-title>Matrix-free approaches for GPU acceleration of a high-order finite element hydrodynamics application using MFEM, Umpire, and RAJA</article-title>. <source>Int. J. High Perf. Comput. Appl</source>. <volume>36</volume>, <fpage>492</fpage>&#x02013;<lpage>509</lpage>. <pub-id pub-id-type="doi">10.1177/10943420221100262</pub-id></citation>
</ref>
</ref-list>
</back>
</article>