<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.3 20070202//EN" "journalpublishing.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" article-type="research-article" dtd-version="2.3" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Comput. Sci.</journal-id>
<journal-title>Frontiers in Computer Science</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Comput. Sci.</abbrev-journal-title>
<issn pub-type="epub">2624-9898</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fcomp.2024.1295014</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Computer Science</subject>
<subj-group>
<subject>Original Research</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>AI-supported approach for human-building interaction implemented at furniture scale</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" corresp="yes">
<name><surname>Bier</surname> <given-names>Henriette</given-names></name>
<xref ref-type="corresp" rid="c001"><sup>&#x002A;</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/230553/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-review-editing/"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/supervision/"/>
<role content-type="https://credit.niso.org/contributor-roles/project-administration/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
<role content-type="https://credit.niso.org/contributor-roles/conceptualization/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Hidding</surname> <given-names>Arwin</given-names></name>
<uri xlink:href="https://loop.frontiersin.org/people/801626/overview"/>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/software/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
<role content-type="https://credit.niso.org/contributor-roles/data-curation/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Brancart</surname> <given-names>Stijn</given-names></name>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Luna-Navarro</surname> <given-names>Alessandra</given-names></name>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>Khademi</surname> <given-names>Seyran</given-names></name>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
</contrib>
<contrib contrib-type="author">
<name><surname>van Engelenburg</surname> <given-names>Casper</given-names></name>
<role content-type="https://credit.niso.org/contributor-roles/writing-original-draft/"/>
<role content-type="https://credit.niso.org/contributor-roles/validation/"/>
<role content-type="https://credit.niso.org/contributor-roles/methodology/"/>
<role content-type="https://credit.niso.org/contributor-roles/investigation/"/>
</contrib>
</contrib-group>
<aff><institution>Architecture and the Built Environment, Delft University of Technology</institution>, <addr-line>Delft</addr-line>, <country>Netherlands</country></aff>
<author-notes>
<fn fn-type="edited-by" id="fn0007"><p>Edited by: Ava Fatah gen Schieck, University College London, United Kingdom</p></fn>
<fn fn-type="edited-by" id="fn0008"><p>Reviewed by: Adamantios Koumpis, University Hospital of Cologne, Germany</p><p>Christopher Leung</p></fn>
<corresp id="c001">&#x002A;Correspondence: Henriette Bier, <email>h.h.bier@tudelft.nl</email></corresp>
</author-notes>
<pub-date pub-type="epub">
<day>21</day>
<month>06</month>
<year>2024</year>
</pub-date>
<pub-date pub-type="collection">
<year>2024</year>
</pub-date>
<volume>6</volume>
<elocation-id>1295014</elocation-id>
<history>
<date date-type="received">
<day>15</day>
<month>09</month>
<year>2023</year>
</date>
<date date-type="accepted">
<day>22</day>
<month>05</month>
<year>2024</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#x00A9; 2024 Bier, Hidding, Brancart, Luna-Navarro, Khademi and van Engelenburg.</copyright-statement>
<copyright-year>2024</copyright-year>
<copyright-holder>Bier, Hidding, Brancart, Luna-Navarro, Khademi and van Engelenburg</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p>
</license>
</permissions>
<abstract>
<p>Human-Building Interaction (HBI) relies on sensor-actuator networks that are increasingly supported by Artificial Intelligence (AI). This paper presents a novel AI-supported Design-to-Robotic-Production-Assembly and -Operation (D2RPA&#x0026;O) approach for reconfigurable furniture. It involves a multidisciplinary approach that relies on the integration of various domains such as architecture, robotics, computer, and material science. It contributes to the advancement of HBI by employing spatial reconfiguration relying on AI and lightweight material design, which is of relevance, particularly when the furniture consists of non-identical but similar components that are re&#x2212;/ configured in a variety of possible combinations.</p>
</abstract>
<kwd-group>
<kwd>human-building interaction</kwd>
<kwd>multi-disciplinary design</kwd>
<kwd>parametric and interaction design</kwd>
<kwd>robotic prototyping</kwd>
<kwd>AI</kwd>
</kwd-group>
<counts>
<fig-count count="8"/>
<table-count count="0"/>
<equation-count count="0"/>
<ref-count count="15"/>
<page-count count="9"/>
<word-count count="4600"/>
</counts>
<custom-meta-wrap>
<custom-meta>
<meta-name>section-at-acceptance</meta-name>
<meta-value>Human-Media Interaction</meta-value>
</custom-meta>
</custom-meta-wrap>
</article-meta>
</front>
<body>
<sec sec-type="intro" id="sec1">
<label>1</label>
<title>Introduction</title>
<p>Human-Building Interaction (HBI) relying on sensor-actuator networks is increasingly supported by Artificial Intelligence (AI) at various scales (<xref ref-type="bibr" rid="ref2">Alavi et al., 2019</xref>). In the case study presented in this paper, the novel AI-supported approach relies on Design-to-Robotic-Production-Assembly and -Operation (D2RPA&#x0026;O) methods that are employed for developing reconfigurable furniture. These methods integrate robotics in building processes and buildings (<xref ref-type="bibr" rid="ref3">Bier and Knight, 2014</xref>; <xref ref-type="bibr" rid="ref15">Robotic Building and Bier, 2018</xref>; <xref ref-type="bibr" rid="ref13">Pillan et al., 2020</xref>) by (a) computationally designing and robotically constructing building components (D2RP&#x0026;A), and (b) by operating those components interactively (D2RO). While some of the methods and techniques used by the involved disciplines in the furniture design process are only developed conceptually, others are virtually and/ or physically prototyped.</p>
<sec id="sec2">
<label>1.1</label>
<title>State-of-the-art</title>
<p>HBI goes back to notions of Interactive Architecture (IA) relying on intelligence integrated into the built environment that has been defined already in the 70s as &#x2018;architecture machines&#x2019; and &#x2018;intelligent environments&#x2019; adapting to their users&#x2019; needs (<xref ref-type="bibr" rid="ref10">Negroponte, 1975</xref>; <xref ref-type="bibr" rid="ref12">Pask, 1975</xref>). Later, Ambient Intelligence (AmI) was defined as a built environment imbued with computing hardware and software technology to benefit its occupant(s) (<xref ref-type="bibr" rid="ref16">Zelkha et al., 1998</xref>). More recently, architecture concerned with interaction between human and non-human agents has proliferated at various scales ranging from furniture to urban scale (<xref ref-type="bibr" rid="ref10">Negroponte, 1975</xref>; <xref ref-type="bibr" rid="ref12">Pask, 1975</xref>; <xref ref-type="bibr" rid="ref16">Zelkha et al., 1998</xref>; <xref ref-type="bibr" rid="ref5">Fox and Kemp, 2009</xref>; <xref ref-type="bibr" rid="ref4">Fox, 2010</xref>; <xref ref-type="bibr" rid="ref11">Oosterhuis and Bier, 2013</xref>; <xref ref-type="bibr" rid="ref3">Bier and Knight, 2014</xref>; <xref ref-type="bibr" rid="ref15">Robotic Building and Bier, 2018</xref>; <xref ref-type="bibr" rid="ref2">Alavi et al., 2019</xref>; <xref ref-type="bibr" rid="ref13">Pillan et al., 2020</xref>).</p>
<p>At furniture scale, examples such as The Big Data (2016) furniture and Media Block Chair (2012) are relevant as they demonstrate furniture reacting to people&#x2019;s movement by changing color.<xref ref-type="fn" rid="fn0001"><sup>1</sup></xref> Such furniture takes advantage of location-based context-aware services and Internet connectivity that are ubiquitously available. Big Data furniture, for instance, communicates with users by changing colors in response to users&#x2019; movement. The movement and behavior of users as well as environmental data patterns are shared on an online database that is used to modulate user experience.</p>
<p>While these examples rely on sensor-actuator networks and involve some level of intelligence they do not involve AI relying on Computer Vision (CV)<xref ref-type="fn" rid="fn0002"><sup>2</sup></xref> for spatial reconfiguration as in the presented case study. With AI progressing and evolving in time through various stages of innovation and practical application, spatial reconfiguration is challenged to integrate multimodal technologies and improve HBI.</p>
</sec>
<sec id="sec3">
<label>1.2</label>
<title>Contribution</title>
<p>The presented case study implemented with MSc students at Technical University (TU) Delft integrates AI-supported multimodal technologies into an alternative design approach relying on D2RPA&#x0026;O methods. It addresses both the technical as well as architectural aspects from the early stages of the design process. It introduces as main features spatial reconfiguration relying on object recognition and lightweight material design. These are of relevance, particularly when the furniture consists, as in this case, of non-identical but similar furniture components that are re&#x2212;/configured in a variety of possible combinations.</p>
</sec>
</sec>
<sec id="sec4">
<label>2</label>
<title>Implementation</title>
<p>Implemented with a group of architecture students tutored by experts from architecture, robotics, and computer science, as well as interaction and structural design, the project focused on the development of furniture components that offer reconfigurable indoor seating areas for accommodating various activities such as workshops (involving collaborative work sessions), events (requiring a stage and seating for the audience), group and individual studying activities next to temporary exhibitions taking place simultaneously or in short sequence. Hence spatial reconfiguration was paramount. Furthermore, the furniture was supposed to be taking structural, functional, environmental, and assembly considerations into account and have degrees of porosity, where the degree and distribution of porosity, i.e., density would facilitate both passive (structural strength, physical comfort, etc.) and active behaviors (interaction, etc.).</p>
<p>The approach involved D2RPA&#x0026;O methods with the aim to improve spatial experience, process- and material-efficiency as well as embed intelligence in building processes and buildings by (a) computationally and robotically optimizing design and material production, (b) AI-supporting dis&#x2212;/re&#x2212;/assembly process, and (c) embedding sensor-actuators that enable interaction with users and the physically built environment. While these procedural steps were all implemented, they achieved various degrees of completion. For instance, the material production was implemented on a fragment, the AI model was tested on a synthetic data set, and the interaction was developed only conceptually at this stage. Future work will involve completing all steps in order to test the proposed varied and multiple use of the space in the library by allowing users to modify the configuration of furniture to meet their changing needs.</p>
<sec id="sec5">
<label>2.1</label>
<title>D2RPA&#x0026;O</title>
<p>The D2RPA&#x0026;O process relied on a Voronoi-based design that facilitates spatial reconfiguration by detaching and reattaching furniture components, i.e., booths to create various functional configurations. The challenge for users to identify availability of components and possible reconfiguration modes ranging from dispersed to clustered configurations that are serving varying activities has been addressed by means of interactive lighting, color coding, and AI-supported object recognition (<xref ref-type="fig" rid="fig1">Figure 1</xref>). Furthermore, the challenge for users to move components from one location to another has been addressed by implementing a highly porous, lightweight material design. All aspects were developed at conceptual level and achieved various degrees of implementation ranging from design to virtual and physical prototyping.</p>
<fig position="float" id="fig1">
<label>Figure 1</label>
<caption>
<p>Red movable Voronoi-based component (left) and available component lighted green (left and right).</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g001.tif"/>
</fig>
<sec id="sec6">
<label>2.1.1</label>
<title>Conceptual design</title>
<p>The need for spatial reconfiguration has been identified by investigating user demands that were captured through a series of behavioral observations in the library space. These included the analysis of current occupancy patterns in terms of number of occupants and type of activities performed over one week. Additional unstructured observations were implemented ahead of the study by the students as they were themselves users of the library. Patterns of stagnation and movement were observed to understand how long users spend in various sections of the library and how users interact with others and various concerns were identified as for instance the reduced reconfigurability options of the furniture and lack of insight into available furniture meeting requirements for various activities and their location.</p>
<p>The occupancy pattern of the library showed large variability in terms of number of occupants during the year (with strong fluctuations in exam vs. vacation periods) and activities. However, even if the largest number of occupants were single users, the space was also used by groups of people for joint activities and one of the problems was the lack of furniture that could accommodate and adapt to user demands during respective activities. Reconfigurability and adaptability emerged therefore as the key objective of the presented design solution.</p>
<p>Reconfiguration has various challenges in terms of function, form, and requirements for lightweight. While precedents such as Media Block Chair presented interactive features, Cloud City, Office Excavate, FRGM: 1, and Voronoi Helmet<xref ref-type="fn" rid="fn0003"><sup>3</sup></xref> showcased the potential of Voronoi-based design at building, component, and material scale. Hence, these approaches were adopted and enhanced using AI.</p>
</sec>
<sec id="sec7">
<label>2.1.2</label>
<title>Computational design</title>
<p>Aspects of computational design were addressed by generating a Voronoi-based design at macro (i.e., building), meso (i.e., component), and micro (i.e., material) scales. This implies that the Voronoi-based material design of components and the assembly of components into a larger structure to facilitate spatial reconfiguration is scalable.</p>
<p>In order to optimize production and assembly of the furniture components, functional, structural, and material design studies were implemented. To optimize functional requirements in relation to activity patterns, reconfiguration by dis&#x2212;/ reassembly of furniture components was chosen. To structurally optimize the Voronoi-based furniture components, a Karamba3D simulation that provided insight into the forces and deformations of the components, both for the isolated wireframe as for the surface elements (<xref ref-type="fig" rid="fig2">Figure 2</xref>) was implemented. The Voronoi-based material design with various cell densities is mapped onto the frame and surface elements according to the optimization results. The variable densities pattern is efficient in terms of the material used versus the resulting degree of stiffness of the object and weight reduction.</p>
<fig position="float" id="fig2">
<label>Figure 2</label>
<caption>
<p>Structural design at component scale involved analysis of tension and compression (left and left-middle) and deformation and stiffness (right-middle and right).</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g002.tif"/>
</fig>
<p>The pattern is calculated based on the stress distribution (<xref ref-type="fig" rid="fig3">Figure 3</xref>) and the load cases that are used are the combination of the loads from the overall model and the forces on the component due to its own weight. The main aim of this study was to improve the mechanical performance of the components while keeping them as lightweight as possible. After all, excessive weight would considerably impact their movability.</p>
<fig position="float" id="fig3">
<label>Figure 3</label>
<caption>
<p>Structural design at material scale without and with optimization (left and right, respectively) showing variation in density of Voronoi cells that are represented in varying colors on <italic>x</italic> and <italic>y</italic> coordinates in meters.</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g003.tif"/>
</fig>
<p>The considered material is thermoplastic elastomer (TPE) with a density of 1.1&#x2009;g/cm3. The paneling geometry is thickened to 10&#x2009;mm and based on this volume distributed loads are applied to the surfaces of the panels. Loads applied are line loads based on the weight of the panels. The resulting principal stress concentrations are translated into a point cloud with a higher density around the edges. The framework is thickened and calculated separately. Since the booths are freestanding simple supporting points are used for the structural analysis that creates a point cloud based on the relative stress densities. The point cloud is then transformed into a Voronoi pattern.</p>
<p>An additional structural analysis is run to determine the final values for the cell sizes and wireframe thicknesses. The Voronoi edges are thickened, and the volume is calculated. The applied loads are based on the weight of TPE and the volume of the wireframe structure. The structural analysis shows the deformation of the wireframe. The final thickness of the edges and cell size parameters are determined by a resulting deflection of less than 10&#x2009;mm for each cell.</p>
</sec>
<sec id="sec8">
<label>2.1.3</label>
<title>Interaction design</title>
<p>The primary goal to create a user-centered spatial experience and provide effective and adaptive spatial configurations to meet current user needs in the library includes consideration for accommodating changing use (single or group use, with various levels of privacy) and user-furniture interaction through responsive lighting in order to communicate users the potential availability of furniture components (<xref ref-type="fig" rid="fig1">Figure 1</xref>).</p>
<p>The user-light interaction system was conceptually designed to provide visual comfort and information to users. The use of indoor lighting for communicating with users while providing appropriate indoor visual conditions was developed and tested in previous work (<xref ref-type="bibr" rid="ref1">Ackerly and Brager, 2013</xref>; <xref ref-type="bibr" rid="ref9">Meerbeek et al., 2016</xref>; <xref ref-type="bibr" rid="ref8">Luna-Navarro et al., 2020</xref>) and has been applied to this case study. While Light-emitting Diodes (LEDs) are integrated into the furniture to provide comfortable direct lighting, various, changing colors are used to communicate if it is available. Using an active system of pressure sensors integrated into the seats, the furniture responds to various types of use. Outward lighting signals availability, whereas inward lighting illuminates the occupied components while in use.</p>
<p>The challenges related to the integration of micro-controllers, sensors and actuators, i.e., LEDs into the furniture components involved ensuring that the physical and virtual subcomponents communicate data to the cloud and data is processed fast enough to allow for smooth uninterrupted interaction. This has been, however, only developed conceptually in this study, with the physical prototyping focusing on the materialization of variable stiffness with cushioning based on structural and ergonomic considerations.</p>
</sec>
<sec id="sec9">
<label>2.1.4</label>
<title>Material design and robotic prototyping</title>
<p>Component and material design was Voronoi-based and involved consideration with respect to stability, lightweight, and comfort requiring variable stiffness and 3D printing without support.<xref ref-type="fn" rid="fn0004"><sup>4</sup></xref> The variable stiffness concept developed in previous research (<xref ref-type="bibr" rid="ref7">Hidding et al., 2019</xref>) has been implemented in this case study in order to create cushioning for the seating areas while keeping the structural integrity of the furniture components, i.e., booths intact. The cushioning principle was prototyped by robotically milling into Expanded Polystyrene (EPS) that in this case was emulating the support surface, which has high stiffness. On top of the support surface is the cushioning surface with low stiffness and is implemented by 3D printing with Thermoplastic Polyurethane (TPU). The imprint of the Voronoi-based pattern designed to serve as cushioning surface was milled out of the EPS so the 3D printed part could be fitted onto the surface (<xref ref-type="fig" rid="fig4">Figure 4</xref>).</p>
<fig position="float" id="fig4">
<label>Figure 4</label>
<caption>
<p>Material design (left) and prototyped fragment (middle and right) using robotic subtractive and additive methods.</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g004.tif"/>
</fig>
<p>The variable stiffness design of the cushion is based on structural analysis, where the stiffness is controlled by the cell sizes implying that the smaller the cells, the denser the material and, therefore, the stiffer the structure. The design of the cells with their varying size and distribution depends on the structural analysis as well as the requirements for printing without support. In this context, the material design and prototyping approach relied on optimized material distribution based on structural optimization ensuring minimized material use and reduced production time. When compared with other production methodologies, the robotic prototyping of the Voronoi-based components has several major advantages. Firstly, it allows the production of complex geometries by linking the parametric design with the production data of the robotic equipment (D2RP). In the case of robotic 3D printing, the angles of the geometry have to be optimized to be within the angle constraints of &#x2212;45 to +45 degrees for printing without support material (<xref ref-type="bibr" rid="ref7">Hidding et al., 2019</xref>). Also, the size of the components and subcomponents are optimized to fit in the bounding box determined by the reachability of the robotic arm. The printability of the material, TPE, and its recyclability have to be considered as well.</p>
</sec>
</sec>
<sec id="sec10">
<label>2.2</label>
<title>Computer vision and artificial intelligence</title>
<p>Computer Vision (CV) as an Artificial Intelligence (AI) module is used for the identification of (a) detached furniture components and (b) the places where the components are supposed to be reattached (<xref ref-type="fig" rid="fig5">Figure 5</xref>). Specifically, a Deep Learning (DL) model<xref ref-type="fn" rid="fn0005"><sup>5</sup></xref> is trained from scratch on a synthetic dataset. The input and output of the model are images, pointing for the input at a specific detached, i.e., stand-alone component, and for the output indicating its position and orientation in the constellation.</p>
<fig position="float" id="fig5">
<label>Figure 5</label>
<caption>
<p>Object and placement recognition for the missing component (left), input image with the missing part (right).</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g005.tif"/>
</fig>
<sec id="sec11">
<label>2.2.1</label>
<title>Task description</title>
<p>The CV task was initially cast as a 3D puzzle reconstruction, where all the components are considered unique pieces of the constellation having their own individual base configuration. The problem could be formulated as a visual placement recognition for machine vision, given adequate training data with various configurations of missing pieces in the constellation (<xref ref-type="fig" rid="fig5">Figure 5</xref>). Even though an exciting CV challenge, the 3D puzzle reconstruction appeared out of the scope for this student project. The problem has then been reformulated to object recognition which is a well-known computer vision task with available model architectures. The components are labelled with ID numbers that are the keys to the correct placement in the constellation. Thus, the challenge is to predict the object&#x2019;s ID number given an input image of the component taken by the user.</p>
</sec>
<sec id="sec12">
<label>2.2.2</label>
<title>Dataset</title>
<p>A synthetic training and validation dataset is created to develop a deep learning model for the object detection task. Due to the specific configuration for this problem, no pre-trained model was available for effective deployment, hence, the students trained an object detection model from scratch to meet the requirements for this project. For the training set, Voronoi-based components are randomly distributed in a 3D model of the library (<xref ref-type="fig" rid="fig6">Figure 6</xref>), and around 3,000 images were generated from the scattered objects from different viewpoints. 80% of the data was used for training and 20% for evaluation. The synthetic dataset is curated by simulating a realistic scenario where the images are taken from real objects at the library. A generalization gap is, however, expected due to the distribution mismatch caused by lighting, shadowing, and occlusion, which are present in a real setting. This gap is increased by the differences between the 3D model and the 3D-printed elements.</p>
<fig position="float" id="fig6">
<label>Figure 6</label>
<caption>
<p>Random camera and components distribution within the 3D model of library (top view left), an example input image from the setting (middle), and object recognition (right) implemented via an app on the phone.</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g006.tif"/>
</fig>
<p>The differences between the digital files and physical objects after 3D printing could be significant enough for the AI to perform poorly on photographs of these components. Creating 3D models that have the 3D-printed layers in the models could help to create training images that more closely resemble the produced components. Labelled photographs of the produced components could also be used to re-train the model in the case of poor performance. This has been, however, not the focus of this study and will be implemented in the next step.</p>
</sec>
<sec id="sec13">
<label>2.2.3</label>
<title>Model and results</title>
<p>The Yolo object detection model (<xref ref-type="bibr" rid="ref14">Redmon et al., 2016</xref>) is trained on the created dataset to predict the object&#x2019;s ID number for the Voronoi-based components with acceptable accuracy and confidence. An online demo<xref ref-type="fn" rid="fn0006"><sup>6</sup></xref> and screenshots from the demo (<xref ref-type="fig" rid="fig6">Figure 6</xref>) show randomly placed components that are detected in real time using the trained object detection model. The probability of specific components being present within the bounding box is shown in the right text box next to the box (<xref ref-type="fig" rid="fig6">Figure 6</xref> right). These probabilities are used to classify an object as a true positive (correctly detected) or false positive (wrongly detected). Setting a confidence level as a hyperparameter for measuring the precision score for the object detection model. In this case, a confidence level of 0.95 from 1.00 has been achieved. There is a trade-off between precision and recall, therefore for such a high confidence level false negatives are increasing.</p>
</sec>
</sec>
</sec>
<sec id="sec14">
<label>3</label>
<title>Discussion and future work</title>
<p>Presented case study outlines D2RPA&#x0026;O strategies for AI-supported furniture reconfiguration as part of a larger HBI approach. Developed computational, interaction, material design, robotic production, and AI-supported assembly approaches (<xref ref-type="fig" rid="fig7">Figure 7</xref>) facilitate furniture adaptation to various functions and spatial requirements by allowing users to modify furniture configurations to meet changing needs in the multi-functional space of the TU Delft library.</p>
<fig position="float" id="fig7">
<label>Figure 7</label>
<caption>
<p>Diagram showing the workflow of AI-supported assembly using an app on a phone to identify the components and their base placement location in real-time.</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g007.tif"/>
</fig>
<p>In the presented case study, individual furniture components were re&#x2212;/ arranged to create various spatial configurations by moving, de&#x2212;/ re&#x2212;/ attaching, and/ or nesting together components. In this context, two main configurations, clustered (<xref ref-type="fig" rid="fig5">Figure 5</xref>) and dispersed (<xref ref-type="fig" rid="fig6">Figure 6</xref>), were considered. Additional solutions for individual cluster configurations will be investigated in the next iteration to identify the limitations of the system in terms of number of furniture components and their possible recombination in relation to the available space. Furthermore, by physically prototyping whole components and using photographs of the produced components to re-train the AI model the feasibility of this approach for real-life applications will be demonstrated.</p>
<p>In furniture components integrated interactive lighting was designed to facilitate user engagement via color- and intensity-changing LED lights that adjust to users&#x2019; preferences and communicate various states (un&#x2212;/ available) and moods (open/ closed). In this context, HBI relied on the integration of technology, design, and user-centered approaches to create spaces that are responsive to the needs and behaviors of their occupants. It involved developing an understanding of the needs, preferences, and behaviors of the people who occupy the space. However, in order to make reconfiguration accessible to people of all abilities the robotization of the system is considered in the future.</p>
</sec>
<sec sec-type="conclusions" id="sec15">
<label>4</label>
<title>Conclusion</title>
<p>While employed AI-supported approach presented in this paper is not new, its integration with D2RPA&#x0026;O methods advances however current state-of-the-art in HBI mainly through the integrative multimodal approach. It contributes to the advancement of HBI by employing spatial reconfiguration relying on AI and lightweight material design, which is of relevance, particularly when the furniture consists of non-identical but similar components that are re&#x2212;/ configured in a variety of possible combinations.</p>
<p>The D2RPA&#x0026;O methods ensuring reduction of material use and production time via various optimization techniques are complemented by AI-supported spatial reconfiguration approaches with the ultimate goal of improving efficiency of space use and users&#x2019; comfort while decreasing environmental impact.</p>
<p>The evaluation and testing were implemented by computationally designing the structure, simulating the whole process and prototyping a fragment at 1:1 scale. Since the focus was on the integration of various research areas involving architecture, computer science, interaction, and structural design, the study met its objectives as all aspects were considered although D2RO remained at conceptual and computational design levels, while D2RP&#x0026;A reached virtual and physical prototyping levels (<xref ref-type="fig" rid="fig7">Figures 7</xref>, <xref ref-type="fig" rid="fig8">8</xref>).</p>
<fig position="float" id="fig8">
<label>Figure 8</label>
<caption>
<p>Diagram showing D2RP process reaching prototyping level.</p>
</caption>
<graphic xlink:href="fcomp-06-1295014-g008.tif"/>
</fig>
<p>The use of lightweight material design for physical dis&#x2212;/ reassembly and the use of lights to provide user-building interaction strategies by communicating to users which furniture components and when they are available were conceptually and to some degree physically explored, though, tests with users still need to be conducted in order to evaluate user response and satisfaction with proposed interaction strategy. Also, the use of AI on physically prototyped components needs verification and operability by users with variable physical abilities needs consideration, which will be addressed in the future.</p>
<p>The collaborative approach required the integration of all aspects from the very beginning of the process. While architecture focused on function and form meeting structural and material design requirements, the interaction aspects relied on the integration of CV. The project, however, did not get to the point of testing the CV approach on a physically prototyped furniture component missing to establish the feasibility of the CV to adequately operate <italic>in situ</italic>, i.e., under realistic conditions. In future, the feedback from the performance of the CV on the physically prototyped furniture component will be used to update the training of the algorithm. For legitimate research purposes the data set used in this study can be obtained by contacting the corresponding author.</p>
</sec>
<sec sec-type="data-availability" id="sec16">
<title>Data availability statement</title>
<p>The datasets presented in this study can be found in online repositories. The names of the repository/repositories and accession number(s) can be found at: <ext-link xlink:href="https://repository.tudelft.nl/" ext-link-type="uri">https://repository.tudelft.nl/</ext-link>.</p>
</sec>
<sec sec-type="author-contributions" id="sec17">
<title>Author contributions</title>
<p>HB: Writing &#x2013; review &#x0026; editing, Writing &#x2013; original draft, Validation, Supervision, Project administration, Methodology, Investigation, Data curation, Conceptualization. AH: Writing &#x2013; original draft, Validation, Software, Methodology, Investigation, Data curation. SB: Writing &#x2013; original draft, Validation, Methodology, Investigation. AL-N: Writing &#x2013; original draft, Methodology, Investigation. SK: Writing &#x2013; original draft, Validation, Methodology, Investigation. CE: Writing &#x2013; original draft, Validation, Methodology, Investigation.</p>
</sec>
</body>
<back>
<sec sec-type="funding-information" id="sec18">
<title>Funding</title>
<p>The author(s) declare that no financial support was received for the research, authorship, and/or publication of this article.</p>
</sec>
<ack>
<p>This paper has profited from the contribution of MSc 2 students Oliver Post, who developed the code, Jiacheng Xu, and Jiahui Shi, who applied the AI-supported D2RPA&#x0026;O methods to the case study, under the supervision of tutors from the Robotic Building and AiDAPT labs as well as Structural Design and Mechanics and Design of Construction chairs. Presented project could not have been implemented without the support of the TU Delft Library (especially Open Spaces programmer, Vincent Cellucci) acting as client and co-founding partner.</p>
</ack>
<sec sec-type="COI-statement" id="sec19">
<title>Conflict of interest</title>
<p>The authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec sec-type="disclaimer" id="sec20">
<title>Publisher's note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
<fn-group>
<fn id="fn0001"><p><sup>1</sup>Links to projects: <ext-link xlink:href="https://www.bassala.com/big-data-furniture" ext-link-type="uri">https://www.bassala.com/big-data-furniture</ext-link> and <ext-link xlink:href="https://www.teamlab.art/w/mediablockchair/" ext-link-type="uri">https://www.teamlab.art/w/mediablockchair/</ext-link>.</p></fn>
<fn id="fn0002"><p><sup>2</sup>AI is usually used to solve problems that typically require human intelligence such as learning, reasoning, problem-solving, perceiving, etc., while CV as a subfield of AI particularly focuses on interpreting visual data.</p></fn>
<fn id="fn0003"><p><sup>3</sup>Links to Cloud City, Office Excavate, FRGM: 1, and Voronoi Helmet: <ext-link xlink:href="https://www.metalocus.es/en/news/tomas-saraceno-roof-cloud-city-met" ext-link-type="uri">https://www.metalocus.es/en/news/tomas-saraceno-roof-cloud-city-met</ext-link>, <ext-link xlink:href="https://www.coroflot.com/GrahamKelman/Office-Excavate" ext-link-type="uri">https://www.coroflot.com/GrahamKelman/Office-Excavate</ext-link>, <ext-link xlink:href="http://www.goetz-schrader.com/crack01.html" ext-link-type="uri">http://www.goetz-schrader.com/crack01.html</ext-link>, and <ext-link xlink:href="https://www.designboom.com/design/voronoi-bicycle-helmet-08-10-2020/" ext-link-type="uri">https://www.designboom.com/design/voronoi-bicycle-helmet-08-10-2020/</ext-link>.</p></fn>
<fn id="fn0004"><p><sup>4</sup>This implies that the angles of the faces of the Voronoi cells have to be within 3D printing tolerances with printing angles between &#x2212;45 to +45 degrees in relation to the printing bed.</p></fn>
<fn id="fn0005"><p><sup>5</sup>Deep learning is a machine learning method, which is based on artificial neural networks that are built using principles of organization in the biological neural networks of animal brains (<ext-link xlink:href="https://news.mit.edu/2017/explained-neural-networks-deep-learning-0414" ext-link-type="uri">https://news.mit.edu/2017/explained-neural-networks-deep-learning-0414</ext-link>).</p></fn>
<fn id="fn0006"><p><sup>6</sup>Link to RB-page: <ext-link xlink:href="http://www.roboticbuilding.eu/project/human-robot-interaction-for-d2ra/" ext-link-type="uri">http://www.roboticbuilding.eu/project/human-robot-interaction-for-d2ra/</ext-link>.</p></fn>
</fn-group>
<ref-list>
<title>References</title>
<ref id="ref1"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Ackerly</surname> <given-names>K.</given-names></name> <name><surname>Brager</surname> <given-names>G.</given-names></name></person-group> (<year>2013</year>). <article-title>Window signalling systems: control strategies and occupant behaviour</article-title>. <source>Build. Res. Inf.</source> <volume>41</volume>, <fpage>342</fpage>&#x2013;<lpage>360</lpage>. doi: <pub-id pub-id-type="doi">10.1080/09613218.2013.772044</pub-id></citation></ref>
<ref id="ref2"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Alavi</surname> <given-names>H.</given-names></name> <name><surname>Churchill</surname> <given-names>E.</given-names></name> <name><surname>Wiberg</surname> <given-names>M.</given-names></name> <name><surname>Lalanne</surname> <given-names>D.</given-names></name> <name><surname>Dalsgaard</surname> <given-names>P.</given-names></name> <name><surname>Schieck</surname> <given-names>A.</given-names></name> <etal/></person-group>. (<year>2019</year>). <article-title>Introduction to human-building interaction (HBI): interfacing HCI with architecture and Urban Design</article-title>. <source>ACM Trans. Comp. Human Interact.</source> <volume>26</volume>, <fpage>1</fpage>&#x2013;<lpage>10</lpage>. doi: <pub-id pub-id-type="doi">10.1145/3309714</pub-id></citation></ref>
<ref id="ref3"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Bier</surname> <given-names>H.</given-names></name> <name><surname>Knight</surname> <given-names>T.</given-names></name></person-group> (<year>2014</year>). <article-title>Data-driven design to production and operation</article-title>. <source>Footprint</source>. <volume>8</volume>, <fpage>1</fpage>&#x2013;<lpage>8</lpage>. doi: <pub-id pub-id-type="doi">10.7480/FOOTPRINT.8.2.807</pub-id></citation></ref>
<ref id="ref4"><citation citation-type="other"><person-group person-group-type="author"><name><surname>Fox</surname> <given-names>M.</given-names></name></person-group> Catching up with the past: a small contribution to a long history of interactive environments (<year>2010</year>).</citation></ref>
<ref id="ref5"><citation citation-type="book"><person-group person-group-type="author"><name><surname>Fox</surname> <given-names>M.</given-names></name> <name><surname>Kemp</surname> <given-names>M.</given-names></name></person-group> (<year>2009</year>). <source>Interactive architecture</source>. <publisher-loc>New York</publisher-loc>: <publisher-name>Princeton Architectural Press</publisher-name>.</citation></ref>
<ref id="ref7"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Hidding</surname> <given-names>A.</given-names></name> <name><surname>Bier</surname> <given-names>H.</given-names></name> <name><surname>Wang</surname> <given-names>Q.</given-names></name> <name><surname>Teuffel</surname> <given-names>P.</given-names></name> <name><surname>Senatore</surname> <given-names>G.</given-names></name></person-group> (<year>2019</year>). <article-title>Structural adaptation through stiffness tuning</article-title>. <source>Spool</source>. <volume>6</volume>, <fpage>43</fpage>&#x2013;<lpage>48</lpage>. doi: <pub-id pub-id-type="doi">10.7480/SPOOL.2019.1.3892</pub-id></citation></ref>
<ref id="ref8"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Luna-Navarro</surname> <given-names>A.</given-names></name> <name><surname>Loonen</surname> <given-names>R.</given-names></name> <name><surname>Juaristi</surname> <given-names>M.</given-names></name> <name><surname>Monge-Barrio</surname> <given-names>A.</given-names></name> <name><surname>Attia</surname> <given-names>S.</given-names></name> <name><surname>Overend</surname> <given-names>M.</given-names></name></person-group> (<year>2020</year>). <article-title>Occupant-facade interaction: a review and classification scheme</article-title>. <source>Build. Environ.</source> <volume>177</volume>:<fpage>106880</fpage>. doi: <pub-id pub-id-type="doi">10.1016/j.buildenv.2020.106880</pub-id></citation></ref>
<ref id="ref9"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Meerbeek</surname> <given-names>B. W.</given-names></name> <name><surname>de Bakker</surname> <given-names>C.</given-names></name> <name><surname>de Kort</surname> <given-names>Y. A. W.</given-names></name> <name><surname>van Loenen</surname> <given-names>E. J.</given-names></name> <name><surname>Bergman</surname> <given-names>T.</given-names></name></person-group> (<year>2016</year>). <article-title>Automated blinds with light feedback to increase occupant satisfaction and energy saving</article-title>. <source>Build. Environ.</source> <volume>103</volume>, <fpage>70</fpage>&#x2013;<lpage>85</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.buildenv.2016.04.002</pub-id></citation></ref>
<ref id="ref10"><citation citation-type="book"><person-group person-group-type="author"><name><surname>Negroponte</surname> <given-names>N.</given-names></name></person-group> (<year>1975</year>). <source>Soft architecture machines</source>. <publisher-loc>Cambridge, MA</publisher-loc>: <publisher-name>MIT Press</publisher-name>.</citation></ref>
<ref id="ref11"><citation citation-type="book"><person-group person-group-type="author"><name><surname>Oosterhuis</surname> <given-names>K.</given-names></name> <name><surname>Bier</surname> <given-names>H.</given-names></name></person-group> (<year>2013</year>). <source>Robotics in architecture</source>. <publisher-loc>Heijningen</publisher-loc>: <publisher-name>Jap Sam Books</publisher-name>.</citation></ref>
<ref id="ref12"><citation citation-type="book"><person-group person-group-type="author"><name><surname>Pask</surname> <given-names>G.</given-names></name></person-group> (<year>1975</year>). <source>Artificial intelligence: A preface and a theory, soft architecture machines</source>. <publisher-loc>Cambridge, MA</publisher-loc>: <publisher-name>MIT Press</publisher-name>.</citation></ref>
<ref id="ref13"><citation citation-type="journal"><person-group person-group-type="author"><name><surname>Pillan</surname> <given-names>M.</given-names></name> <name><surname>Bier</surname> <given-names>H.</given-names></name> <name><surname>Green</surname> <given-names>K.</given-names></name> <name><surname>Pavlovic</surname> <given-names>M.</given-names></name></person-group> (<year>2020</year>). <article-title>Actuated and performative architecture: emerging forms of human-machine interaction</article-title>. <source>Cyber Physical Architecture</source> <volume>7</volume>, <fpage>3</fpage>&#x2013;<lpage>4</lpage>. doi: <pub-id pub-id-type="doi">10.7480/SPOOL.2020.3</pub-id></citation></ref>
<ref id="ref14"><citation citation-type="confproc"><person-group person-group-type="author"><name><surname>Redmon</surname> <given-names>J.</given-names></name> <name><surname>Divvala</surname> <given-names>S.</given-names></name> <name><surname>Girshick</surname> <given-names>R.</given-names></name> <name><surname>Farhadi</surname> <given-names>A.</given-names></name></person-group> <article-title>You only look once: unified, real-time object detection</article-title>. In <conf-name>Proceedings of the 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)</conf-name>; <publisher-name>IEEE</publisher-name>: <publisher-loc>Las Vegas, NV, USA</publisher-loc>(<year>2016</year>).</citation></ref>
<ref id="ref15"><citation citation-type="book"><person-group person-group-type="author"><collab id="coll1">Robotic Building</collab></person-group> (<year>2018</year>). <source>Springer series in adaptive environments</source>. <publisher-loc>Cham</publisher-loc>: <publisher-name>Springer International Publishing</publisher-name>.</citation></ref>
<ref id="ref16"><citation citation-type="other"><person-group person-group-type="author"><name><surname>Zelkha</surname> <given-names>E.</given-names></name> <name><surname>Epstein</surname> <given-names>B.</given-names></name> <name><surname>Birrell</surname> <given-names>S.</given-names></name> <name><surname>Dodsworth</surname> <given-names>C.</given-names></name></person-group>. From devices to &#x201C;Ambient Intelligence&#x201D;: the transformation of consumer electronics. Digital living room conference (<year>1998</year>).</citation></ref>
</ref-list>
</back>
</article>