Publications

@inproceedings{safe_reconstruction,
title = {Safe, Occlusion-Aware Manipulation for Online Object Reconstruction in Confined Space},
author = {Y Miao and R Wang and K E Bekris},
url = {https://arxiv.org/abs/2205.11719},
year = {2022},
date = {2022-09-25},
booktitle = {International Symposium on Robotics Research (ISRR) },
abstract = {Recent work in robotic manipulation focuses on object retrieval in cluttered space under occlusion. Nevertheless, the majority of efforts lack an analysis of conditions for the completeness of the approaches or the methods apply only when objects can be removed from the workspace. This work formulates the general, occlusion-aware manipulation task, and focuses on safe object reconstruction in a confined space with in-place relocation. A framework that ensures safety with completeness guarantees is proposed. Furthermore, an algorithm, which is an instantiation of this framework for monotone instances, is developed and evaluated empirically by comparing against a random and a greedy baseline on randomly generated experiments in simulation. Even for cluttered scenes with realistic objects, the proposed algorithm significantly outperforms the baselines and maintains a high success rate across experimental conditions. },
keywords = {Manipulation, Planning, Robot Perception},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{yodo_rss22,
title = {You Only Demonstrate Once: Category-Level Manipulation from Single Visual Demonstration},
author = {B Wen and W Lian and K E Bekris and S Schaal},
url = {https://arxiv.org/abs/2201.12716},
year = {2022},
date = {2022-06-29},
booktitle = {Robotics: Science and Systems (RSS)},
abstract = {Promising results have been achieved recently in category-level manipulation that generalizes across object instances. Nevertheless, it often requires expensive real-world data collection and manual specification of semantic keypoints for each object category and task. Additionally, coarse keypoint predictions and ignoring intermediate action sequences hinder adoption in complex manipulation tasks beyond pick-and-place. This work proposes a novel, category-level manipulation framework that leverages an object-centric, category-level representation and model-free 6 DoF motion tracking. The canonical object representation is learned solely in simulation and then used to parse a category-level, task trajectory from a single demonstration video. The demonstration is reprojected to a target trajectory tailored to a novel object via the canonical representation. During execution, the manipulation horizon is decomposed into long range, collision-free motion and last-inch manipulation. For the latter part, a category-level behavior cloning (CatBC) method leverages motion tracking to perform closed-loop control. CatBC follows the target trajectory, projected from the demonstration and anchored to a dynamically selected category-level coordinate frame. The frame is automatically selected along the manipulation horizon by a local attention mechanism. This framework allows to teach different manipulation strategies by solely providing a single demonstration, without complicated manual programming. Extensive experiments demonstrate its efficacy in a range of challenging industrial tasks in high precision assembly, which involve learning complex, long-horizon policies. The process exhibits robustness against uncertainty due to dynamics as well as generalization across object instances and scene configurations. },
note = {Nomination for Best Paper Award},
keywords = {Learning, Manipulation, Robot Perception},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{lazy_confined_rearrangement,
title = {Lazy Rearrangement Planning in Confined Spaces},
author = {R Wang and K Gao and J Yu and K E Bekris },
url = {https://arxiv.org/abs/2203.10379},
year = {2022},
date = {2022-06-20},
booktitle = {International Conference on Automated Planning and Scheduling (ICAPS)},
abstract = {Object rearrangement is important for many applications but remains challenging, especially in confined spaces, such as shelves, where objects cannot be accessed from above and they block reachability to each other. Such constraints require many motion planning and collision checking calls, which are computationally expensive. In addition, the arrangement space grows exponentially with the number of objects. To address these issues, this work introduces a lazy evaluation framework with a local monotone solver and a global planner. Monotone instances are those that can be solved by moving each object at most once. A key insight is that reachability constraints at the grasps for objects’ starts and goals can quickly reveal dependencies between objects without having to execute expensive motion planning queries. Given that, the local solver builds lazily a search tree that respects these reachability constraints without verifying that the arm paths are collision free. It only collision checks when a promising solution is found. If a monotone solution is not found, the non-monotone planner loads the lazy search tree and explores ways to move objects to intermediate locations from where monotone solutions to the goal can be found. Results show that the proposed framework can solve difficult instances in confined spaces with up to 16 objects, which state-of-the-art methods fail to solve. It also solves problems faster than alter- natives, when the alternatives find a solution. It also achieves high-quality solutions, i.e., only 1.8 additional actions on av- erage are needed for non-monotone instances.},
keywords = {Manipulation, Rearrangement},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{reconstruct_lifelong_manipulation,
title = {Online Object Model Reconstruction and Reuse for Lifelong Improvement of Robot Manipulation},
author = {S Lu and R Wang and Y Miao and C Mitash and K E Bekris},
url = {https://arxiv.org/abs/2109.13910},
year = {2022},
date = {2022-05-28},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {This work proposes a robotic pipeline for picking and constrained placement of objects without geometric shape priors. Compared to recent efforts developed for similar tasks, where every object was assumed to be novel, the proposed system recognizes previously manipulated objects and performs online model reconstruction and reuse. Over a lifelong manipulation process, the system keeps learning features of objects it has interacted with and updates their reconstructed models. Whenever an instance of a previously manipulated object reappears, the system aims to first recognize it and then register its previously reconstructed model given the current observation. This step greatly reduces object shape uncertainty allowing the system to even reason for parts of objects, which are currently not observable. This also results in better manipulation efficiency as it reduces the need for active perception of the target object during manipulation. To get a reusable reconstructed model, the proposed pipeline adopts: i) TSDF for object representation, and ii) a variant of the standard particle filter algorithm for pose estimation and tracking of the partial object model. Furthermore, an effective way to construct and maintain a dataset of manipulated objects is presented. A sequence of real-world manipulation experiments is performed. They show how future manipulation tasks become more effective and efficient by reusing reconstructed models of previously manipulated objects, which were generated during their prior manipulation, instead of treating objects as novel every time. },
note = {Nomination for Best Paper Award in Manipulation},
keywords = {Manipulation, Robot Perception},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{catgrasp_icra22,
title = {CaTGrasp: Learning Category-Level Task-Relevant Grasping in Clutter from Simulation},
author = {B Wen and W Lian and K E Bekris and S Schaal},
url = {https://arxiv.org/abs/2109.09163},
year = {2022},
date = {2022-05-25},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {Task-relevant grasping is critical for industrial assembly, where downstream manipulation tasks constrain the set of
valid grasps. Learning how to perform this task, however, is challenging, since task-relevant grasp labels are hard
to define and annotate. There is also yet no consensus on proper representations for modeling or off-the-shelf tools
for performing task-relevant grasps. This work proposes a framework to learn task-relevant grasping for industrial
objects without the need of time-consuming real-world data collection or manual annotation. To achieve this, the
entire framework is trained solely in simulation, including supervised training with synthetic label generation and
self-supervised, hand-object interaction. In the context of this framework, this paper proposes a novel, object-centric canonical representation at the category level, which allows establishing dense correspondence across object instances and transferring task-relevant grasps to novel instances. Extensive experiments on task-relevant grasping of densely-cluttered industrial objects are conducted in both simulation and real-world setups, demonstrating the effectiveness of the proposed framework. Code and data is released at
https://sites.google.com/view/catgrasp. },
keywords = {Manipulation, Robot Perception},
pubstate = {published},
tppubtype = {inproceedings}
}

@article{inhand_gaiting_multimodal,
title = {Complex In-Hand Manipulation via Compliance-Enabled Finger Gaiting and Multi-Modal Planning},
author = {A Morgan and K Hang and B Wen and K E Bekris and A Dollar},
url = {https://arxiv.org/abs/2201.07928},
year = {2022},
date = {2022-05-24},
journal = {IEEE Robotics and Automation Letters (also at ICRA)},
abstract = {Constraining contacts to remain fixed on an object during manipulation limits the potential workspace size, as motion is subject to the hand's kinematic topology. Finger gaiting is one way to alleviate such restraints. It allows contacts to be freely broken and remade so as to operate on different manipulation manifolds. This capability, however, has traditionally been difficult or impossible to practically realize. A finger gaiting system must simultaneously plan for and control forces on the object while maintaining stability during contact switching. This work alleviates the traditional requirement by taking advantage of system compliance, allowing the hand to more easily switch contacts while maintaining a stable grasp. Our method achieves complete SO(3) finger gaiting control of grasped objects against gravity by developing a manipulation planner that operates via orthogonal safe modes of a compliant, underactuated hand absent of tactile sensors or joint encoders. During manipulation, a low-latency 6D pose object tracker provides feedback via vision, allowing the planner to update its plan online so as to adaptively recover from trajectory deviations. The efficacy of this method is showcased by manipulating both convex and non-convex objects on a real robot. Its robustness is evaluated via perturbation rejection and long trajectory goals. To the best of the authors' knowledge, this is the first work that has autonomously achieved full SO(3) control of objects within-hand via finger gaiting and without a support surface, elucidating a valuable step towards realizing true robot in-hand manipulation capabilities. },
keywords = {Manipulation},
pubstate = {published},
tppubtype = {article}
}

@inproceedings{highqual_prehensile_rearrangement,
title = {Efficient and High-Quality Prehensile Rearrangement in Cluttered and Confined Spaces},
author = {R Wang and Y Miao and K E Bekris},
url = {https://arxiv.org/abs/2110.02814},
year = {2022},
date = {2022-05-24},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {Prehensile object rearrangement in cluttered and confined spaces has broad applications but is also challenging. For instance, rearranging products in a grocery shelf means that the robot cannot directly access all objects and has limited free space. This is harder than tabletop rearrangement where objects are easily accessible with top-down grasps, which simplifies robot-object interactions. This work focuses on problems where such interactions are critical for completing tasks. It proposes a new efficient and complete solver under general constraints for monotone instances, which can be solved by moving each object at most once. The monotone solver reasons about robot-object constraints and uses them to effectively prune the search space. The new monotone solver is integrated with a global planner to solve non-monotone instances with high-quality solutions fast. Furthermore, this work contributes an effective pre-processing tool to significantly speed up online motion planning queries for rearrangement in confined spaces. Experiments further demonstrate that the proposed monotone solver, equipped with the pre-processing tool, results in 57.3% faster computation and 3 times higher success rate than state-of-the-art methods. Similarly, the resulting global planner is computationally more efficient and has a higher success rate, while producing high-quality solutions for non-monotone instances (i.e., only 1.3 additional actions are needed on average). },
keywords = {Manipulation, Rearrangement},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{homology_nonprehensile,
title = {Persistent Homology for Effective Non-Prehensile Manipulation},
author = {E Vieira and D Nakhimovich and K Gao and R Wang and J Yu and K E Bekris},
url = {https://arxiv.org/abs/2202.02937},
year = {2022},
date = {2022-05-24},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {This work explores the use of topological tools for achieving effective non-prehensile manipulation in cluttered, constrained workspaces. In particular, it proposes the use of persistent homology as a guiding principle in identifying the appropriate non-prehensile actions, such as pushing, to clean a cluttered space with a robotic arm so as to allow the retrieval of a target object. Persistent homology enables the automatic identification of connected components of blocking objects in the space without the need for manual input or tuning of parameters. The proposed algorithm uses this information to push groups of cylindrical objects together and aims to minimize the number of pushing actions needed to reach to the target. Simulated experiments in a physics engine using a model of the Baxter robot show that the proposed topology-driven solution is achieving significantly higher success rate in solving such constrained problems relatively to state-of-the-art alternatives from the literature. It manages to keep the number of pushing actions low, is computationally efficient and the resulting decisions and motion appear natural for effectively solving such tasks. },
keywords = {Manipulation},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{learning_sequential_manipulation,
title = {Learning Sensorimotor Primitives of Sequential Manipulation Tasks from Visual Demonstrations},
author = {J Liang and B Wen and K E Bekris and A Boularias},
url = {https://arxiv.org/abs/2203.03797},
year = {2022},
date = {2022-05-24},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {This work aims to learn how to perform complex robot manipulation tasks that are composed of several, consecutively executed low-level sub-tasks, given as input a few visual demonstrations of the tasks performed by a person. The sub-tasks consist of moving the robot's end-effector until it reaches a sub-goal region in the task space, performing an action, and triggering the next sub-task when a pre-condition is met. Most prior work in this domain has been concerned with learning only low-level tasks, such as hitting a ball or reaching an object and grasping it. This paper describes a new neural network-based framework for learning simultaneously low-level policies as well as high-level policies, such as deciding which object to pick next or where to place it relative to other objects in the scene. A key feature of the proposed approach is that the policies are learned directly from raw videos of task demonstrations, without any manual annotation or post-processing of the data. Empirical results on object manipulation tasks with a robotic arm show that the proposed network can efficiently learn from real visual demonstrations to perform the tasks, and outperforms popular imitation learning algorithms. },
keywords = {Learning, Manipulation},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{fast_tabletop_rearrangement,
title = {Fast High-Quality Tabletop Rearrangement in Bounded Workspace},
author = {K Gao and D Lau and B Huang and K E Bekris and J Yu },
url = {https://arxiv.org/abs/2110.12325},
year = {2022},
date = {2022-05-23},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {In this paper, we examine the problem of rearranging many objects on a tabletop in a cluttered setting using overhand grasps. Efficient solutions for the problem, which capture a common task that we solve on a daily basis, are essential in enabling truly intelligent robotic manipulation. In a given instance, objects may need to be placed at temporary positions ("buffers") to complete the rearrangement, but allocating these buffer locations can be highly challenging in a cluttered environment. To tackle the challenge, a two-step baseline planner is first developed, which generates a primitive plan based on inherent combinatorial constraints induced by start and goal poses of the objects and then selects buffer locations assisted by the primitive plan. We then employ the "lazy" planner in a tree search framework which is further sped up by adapting a novel preprocessing routine. Simulation experiments show our methods can quickly generate high-quality solutions and are more robust in solving large-scale instances than existing state-of-the-art approaches. },
keywords = {Manipulation, Rearrangement},
pubstate = {published},
tppubtype = {inproceedings}
}

@conference{MorWen2021,
title = {Vision-driven Compliant Manipulation for Reliable, High-Precision Assembly Tasks},
author = {A Morgan and B Wen and L Junchi and A Boularias and A Dollar and K E Bekris},
url = {https://arxiv.org/abs/2106.14070},
year = {2021},
date = {2021-07-12},
booktitle = {Robotics: Science and Systems},
abstract = {Highly constrained manipulation tasks continue to be challenging for autonomous robots as they require high levels of precision, typically less than 1mm, which is often incompatible with what can be achieved by traditional perception systems. This paper demonstrates that the combination of state-of-the-art object tracking with passively adaptive mechanical hardware can be leveraged to complete precision manipulation tasks with tight, industrially-relevant tolerances (0.25mm). The proposed control method closes the loop through vision by tracking the relative 6D pose of objects in the relevant workspace. It adjusts the control reference of both the compliant manipulator and the hand to complete object insertion tasks via within-hand manipulation. Contrary to previous efforts for insertion, our method does not require expensive force sensors, precision manipulators, or time-consuming, online learning, which is data hungry. Instead, this effort leverages mechanical compliance and utilizes an object-agnostic manipulation model of the hand learned offline, off-the-shelf motion planning, and an RGBD-based object tracker trained solely with synthetic data. These features allow the proposed system to easily generalize and transfer to new tasks and environments. This paper describes in detail the system components and showcases its efficacy with extensive experiments involving tight tolerance peg-in-hole insertion tasks of various geometries as well as open-world constrained placement tasks.},
keywords = {Manipulation, Robot Perception},
pubstate = {published},
tppubtype = {conference}
}

@conference{207,
title = {Towards Robust Product Packing with a Minimalistic End-Effector},
author = {R Shome and W N Tang and C Song and C Mitash and C Kourtev and J Yu and A Boularias and K E Bekris},
url = {http://robotpacking.org/},
year = {2019},
date = {2019-05-01},
booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
abstract = {Advances in sensor technologies, object detection algorithms, planning frameworks and hardware designs have motivated the deployment of robots in warehouse automation. A variety of such applications, like order fulfillment or packing tasks, require picking objects from unstructured piles and carefully arranging them in bins or containers. Desirable solutions need to be low-cost, easily deployable and controllable, making minimalistic hardware choices desirable. The challenge in designing an effective solution to this problem relates to appropriately integrating multiple components, so as to achieve a robust pipeline that minimizes failure conditions. The current work proposes a complete pipeline for solving such packing tasks, given access only to RGB-D data and a single robot arm with a minimalistic, vacuum-based end-effector. To achieve the desired level of robustness, three key manipulation primitives are identified, which take advantage of the environment and simple operations to successfully pack multiple cubic objects. The overall approach is demonstrated to be robust to execution and perception errors. The impact of each manipulation primitive is evaluated by considering different versions of the proposed pipeline that incrementally introduce reasoning about object poses and corrective manipulation actions.},
note = {Nomination for Best Paper Award in Automation},
keywords = {Manipulation, Robot Perception},
pubstate = {published},
tppubtype = {conference}
}