@article {332, title = {Physics-Informed Neural Nets for Control of Dynamical Systems}, journal = {Neurocomputing}, volume = {579}, year = {2024}, chapter = {127419}, abstract = {

Physics-informed neural networks (PINNs) impose known physical laws into the learning of deep neural networks, making sure they respect the physics of the process while decreasing the demand of labeled data. For systems represented by Ordinary Differential Equations (ODEs), the conventional PINN has a continuous time input variable and outputs the solution of the corresponding ODE. In their original form, PINNs do not allow control inputs neither can they simulate for long-range intervals without serious degradation in their predictions. In this context, this work presents a new framework called Physics-Informed Neural Nets for Control (PINC), which proposes a novel PINN-based architecture that is amenable to \emph{control} problems and able to simulate for longer-range time horizons that are not fixed beforehand. The framework has new inputs to account for the initial state of the system and the control action. In PINC, the response over the complete time horizon is split such that each smaller interval constitutes a solution of the ODE conditioned on the fixed values of initial state and control action for that interval. The whole response is formed by feeding back the predictions of the terminal state as the initial state for the next interval. This proposal enables the optimal control of dynamic systems, integrating a priori knowledge from experts and data collected from plants into control applications. We showcase our proposal in the control of two nonlinear dynamic systems: the Van der Pol oscillator and the four-tank system.

}, doi = {10.1016/j.neucom.2024.127419}, url = {https://doi.org/10.1016/j.neucom.2017.09.005}, author = {Eric A Antonelo and Eduardo Camponogara and Laio Oriel Seman and Eduardo Rehbein de Souza and Jean Panaioti Jordanou and Jomi Fred H{\"u}bner} } @conference {331, title = {Cone Detection with Convolutional Neural Networks for an Autonomous Formula Student Race Car}, booktitle = {26th ABCM International Congress of Mechanical Engineering (COBEM 2021)}, year = {2021}, abstract = {

Formula Student Driverless is a competition that provides an opportunity to design and test a wide set of technologies that are key for achieving autonomy in vehicle navigation: object detection and segmentation, obstacle avoidance, trajectory planning and following, etc.\ 
In this context, the objective of this work is to employ and analyse state-of-the art Convolutional Neural Networks (CNN) architectures known in the literature for robust\  detection of cones whose function is to delimit tracks for an autonomous race car in the Formula Student Driverless Competition.
Here, fast and reliable cone detection is important for achieving a robust and efficient perception pipeline for a Formula race car.
To that end, we employ the known YOLO (You Only Look Once) architecture that makes use of CNNs and several optimizations to achieve fast and accurate enough object detection to be suitable for the current application.
To train the CNN/YOLO, different datasets are considered: one is provided by MIT and another is the FSOCO dataset (Formula Student Objects in Context), which is a more complete dataset that considers four different classes of cones in the track: blue cone, yellow cone, and small orange and large orange cones.
The main metrics for measuring the performance of the trained network in cone detection tasks are the Mean Average Precision (mAP) and network inference time.\ 
Using these metrics and different versions of YOLO (the original and the tiny version), we analyze:\ 
(1) the influence of the image resolution for the input layer of the CNN;\ 
(2) the influence of different image conditions by varying brightness, exposure, blur, noise and other perturbations;\ 
(3) the impact of manual augmentations in the training dataset and its performance under different image conditions.
In general, YOLO has shown to be a strong candidate for real-time cone detection for race cars.
Results shows that mAP performance increases as the image resolution for the input layer in the YOLO net gets higher, at a cost of increasing the CNN\&$\#$39;s inference time though.\ 
Besides, manual augmentations of the training set were found to be effective for recovering the lost mAP when perturbations (of brightness, blur and noise) were added to the images.\ 
As future work, the current cone detection solution can be extended to be part of the perception pipeline of the race car from Ampera Racing, which is an electric Formula SAE (Society of Automotive Engineers) team from Federal University of Santa Catarina.

}, author = {La{\'\i}za Milena Scheid Parizotto and Eric A Antonelo} } @proceedings {336, title = {Face Reconstruction with Variational Autoencoder and Face Masks}, journal = {18th Encontro Nacional de Intelig{\^e}ncia Artificial e Computacional (ENIAC 2021)}, year = {2021}, abstract = {

Variational AutoEncoders (VAE) employ deep learning models to learn a continuous latent z-space that is subjacent to a high-dimensional observed dataset. With that, many tasks are made possible, including face reconstruction and face synthesis. In this work, we investigated how face masks can help the training of VAEs for face reconstruction, by restricting the learning to the pixels selected by the face mask. An evaluation of the proposal using the celebA dataset shows that the reconstructed images are enhanced with the face masks, especially when SSIM loss is used either with l1 or l2 loss functions. We noticed that the inclusion of a decoder for face mask prediction in the architecture affected the performance for l1 or l2 loss functions, while this was not the case for the SSIM loss. Besides, SSIM perceptual loss yielded the crispest samples between all hypotheses tested, although it shifts the original color of the image, making the usage of the l1 or l2 losses together with SSIM helpful to solve this issue.

}, doi = {10.5753/eniac.2021.18282}, url = {https://sol.sbc.org.br/index.php/eniac/article/view/18282}, author = {Rafael S Toledo and Eric A Antonelo} } @proceedings {335, title = {Generative Adversarial Imitation Learning for End-to-End Autonomous Driving on Urban Environments}, journal = {2021 IEEE Symposium Series on Computational Intelligence (SSCI)}, year = {2021}, pages = {1-7}, abstract = {

Autonomous driving is a complex task, which has been tackled since the first self-driving car ALVINN in 1989, with a supervised learning approach, or behavioral cloning (BC). In BC, a neural network is trained with state-action pairs that constitute the training set made by an expert, i.e., a human driver. However, this type of imitation learning does not take into account the temporal dependencies that might exist between actions taken in different moments of a navigation trajectory. These type of tasks are better handled by reinforcement learning (RL) algorithms, which need to define a reward function. On the other hand, more recent approaches to imitation learning, such as Generative Adversarial Imitation Learning (GAIL), can train policies without explicitly requiring to define a reward function, allowing an agent to learn by trial and error directly on a training set of expert trajectories. In this work, we propose two variations of GAIL for autonomous navigation of a vehicle in the realistic CARLA simulation environment for urban scenarios. Both of them use the same network architecture, which process high-dimensional image input from three frontal cameras, and other nine continuous inputs representing the velocity, the next point from the sparse trajectory and a high-level driving command. We show that both of them are capable of imitating the expert trajectory from start to end after training ends, but the GAIL loss function that is augmented with BC outperforms the former in terms of convergence time and training stability.

}, doi = {10.1109/SSCI50451.2021.9660156}, url = {https://arxiv.org/abs/2110.08586}, author = {Gustavo Claudio Karl Couto and Eric A Antonelo} } @proceedings {334, title = {Proximal Policy Optimization with Continuous Bounded Action Space via the Beta Distribution}, journal = {2021 IEEE Symposium Series on Computational Intelligence (SSCI)}, year = {2021}, pages = {1-8}, abstract = {

Reinforcement learning methods for continuous control tasks have evolved in recent years generating a family of policy gradient methods that rely primarily on a Gaussian distribution for modeling a stochastic policy. However, the Gaussian distribution has an infinite support, whereas real world applications usually have a bounded action space. This dissonance causes an estimation bias that can be eliminated if the Beta distribution is used for the policy instead, as it presents a finite support. In this work, we investigate how this Beta policy performs when it is trained by the Proximal Policy Optimization (PPO) algorithm on two continuous control tasks from OpenAI gym. For both tasks, the Beta policy is superior to the Gaussian policy in terms of agent\&$\#$39;s final expected reward, also showing more stability and faster convergence of the training process. For the CarRacing environment with high-dimensional image input, the agent\&$\#$39;s success rate was improved by 63\% over the Gaussian policy.

}, doi = {10.1109/SSCI50451.2021.9660123}, url = {https://ieeexplore.ieee.org/abstract/document/9660123}, author = {Irving G. B. Petrazzini and Eric A Antonelo} } @proceedings {312, title = {On importance weighting for electric fraud detection with dataset shifts}, journal = {IEEE International Conference on Systems, Man, and Cybernetics (SMC)}, year = {2019}, month = {2019}, pages = {3215-3222}, abstract = {

Covariate shift and imbalanced datasets are common in real-world scenarios. Usually, the probability distribution for the collected data\  is non-stationary due to\  the incremental and endless process of sequential data collection, which is influenced by actions and predictions of human experts, predictive models, set of rules or other unknown external factors (i.e., user interaction on a website, seasonal/cyclic or geographical factors). Thus, a predictive model may be suboptimal in terms of generalization performance under a shift in the test input. In this work, we evaluate the importance-weighted fisher discriminant analysis (FDA) classifier in an electric fraud detection task with dataset shift, where the goal is to detect customers with frauds or irregular eletricity meters, also called nontechnical loss detection in the literature.
\ The inputs to the model are mainly based on features computed from the monthly energy consumption time series of each customer, using a real-world dataset of 3.6M clients from an energy distribution network.
The importance weights which define the relevance of each training input sample are estimated via either of two methods: the Kullback-Leibler importance estimation procedure (KLIEP) and another based on a discriminative classifier with probabilistic output.\ 
On a series of experiments, we show that a misspecified (biased) classifer in the form of a Least Squares solution has its bias removed when the estimated importance weights are employed in the model, making it comparable to the solution given by the original unbiased FDA for the electric fraud detection task.

}, keywords = {covariate shift, electric fraud detection, importance weighting}, author = {Eric A Antonelo and Radu State} } @article {295, title = {Reservoir Computing for Detection of Steady State in Performance Tests of Compressors}, journal = {Neurocomputing}, volume = {275}, year = {2018}, month = {01/2018}, pages = {607}, chapter = {598}, abstract = {

Fabrication of devices in industrial plants often includes undergoing quality assurance tests or tests that seek to determine some attributes or capacities of the device. For instance, in testing refrigeration compressors, we want to find the true refrigeration capacity of the compressor being tested. Such test (also called an episode) may take up to four hours, being an actual hindrance to applying it to the total number of compressors produced. This work seeks to reduce the time spent on such industrial trials by employing Recurrent Neural Networks (RNNs) as dynamical models for detecting when a test is entering the so-called steady-state region. Specifically, we use Reservoir Computing (RC) networks which simplify the learning of RNNs by speeding up training time and showing convergence to a global optimum. Also, this work proposes a self-organized subspace projection method for RC networks which uses information from the beginning of the episode to define a cluster to which the episode belongs to. This assigned cluster defines a particular binary input that shifts the operating point of the reservoir to a subspace of trajectories for the duration of the episode. This method is shown to turn the RC model robust in performance with respect to varying combination of reservoir parameters, such as spectral radius and leak rate, when compared to a standard RC network.

}, doi = {10.1016/j.neucom.2017.09.005}, author = {Eric A Antonelo and Carlos A. Flesch and Filipe Schmitz} } @article {Antonelo2016, title = {Echo State Networks for Data-driven Downhole Pressure Estimation in Gas-lift Oil Wells}, journal = {Neural Networks}, volume = {85}, year = {2017}, note = {(in press)}, pages = {106-117}, abstract = {

Process measurements are of vital importance for monitoring and control of industrial plants. When we consider offshore oil production platforms, wells that require gas-lift technology to yield oil production from low pressure oil reservoirs can become unstable under some conditions. This undesir- able phenomenon is usually called slugging flow, and can be identified by an oscillatory behavior of the downhole pressure measurement. Given the importance of this measurement and the unreliability of the related sensor, this work aims at designing data-driven soft-sensors for downhole pressure estimation in two contexts: one for speeding up first-principle model sim- ulation of a vertical riser model; and another for estimating the downhole pressure using real-world data from an oil well from Petrobras based only on topside platform measurements. Both tasks are tackled by employing Echo State Networks (ESN) as an efficient technique for training Recurrent Neu- ral Networks. We show that a single ESN is capable of robustly modeling both the slugging flow behavior and a steady state based only on a square wave input signal representing the production choke opening in the vertical riser. Besides, we compare the performance of a standard network to the performance of a multiple timescale hierarchical architecture in the second task and show that the latter architecture performs better in modeling both large irregular transients and more commonly occurring small oscillations.

}, doi = {10.1016/j.neunet.2016.09.009}, url = {http://authors.elsevier.com/sd/article/S0893608016301393}, author = {Eric A Antonelo and Eduardo Camponogara and Bjarne Foss} } @proceedings {302, title = {Recurrent Neural Network based control of an Oil Well}, journal = {XIII Brazilian Symposium on Intelligent Automation (SBAI)}, year = {2017}, pages = {924-931}, abstract = {

Echo State Networks (ESN) are dynamical learning models composed of two parts: a recurrent network (reservoir) with fixed weights and a linear adaptive readout output layer. The output layer\’s weights are learned for the ESN to reproduce temporal patterns usually by solving a least-squares problem. Such recurrent networks have shown promising results in previous applications to dynamic system identification and closed-loop control. This work applies an echo state network to control the bottom hole pressure of an oil well, whereby the opening of the production choke is manipulated. The controller utilizes a network to learn the plant inverse model, whose model input is the plant output and the vice-versa, and another network to compute the control action that induces a desired plant behavior. Despite the nonlinearities of the well model, the ESN effectively learned the inverse model and achieved near global setpoint tracking and disturbance rejection, with little setpoint deviation in the latter case. These results show that echo state networks are a viable tool for the control of complex dynamic systems by means of online inverse-model learning.

}, url = {https://www.ufrgs.br/sbai17/papers/paper_269.pdf}, author = {Jean Panaioti Jordanou and Eric A Antonelo and Eduardo Camponogara and Marco Aurelio S. de Aguiar} } @proceedings {Antonelo2015a, title = {An Echo State Network-based Soft Sensor of Downhole Pressure for a Gas-lift Oil Well}, journal = {16th International Conference on Engineering Applications of Neural Networks}, volume = {Communications in Computer and Information Science, vol 517}, year = {2015}, pages = {379-389}, publisher = {Springer}, abstract = {

Soft sensor technology has been increasingly used in indus- try. Its importance is magnified when the process variable to be estimated is key to control and monitoring processes and the respective sensor ei- ther has a high probability of failure or is unreliable due to harsh environ- ment conditions. This is the case for permanent downhole gauge (PDG) sensors in the oil and gas industry, which measure pressure and tempera- ture in deepwater oil wells. In this paper, historical data obtained from an actual offshore oil well is used to build a black box model that estimates the PDG downhole pressure from platform variables, using Echo State Networks (ESNs), which are a class of recurrent networks with power- ful modeling capabilities. These networks, differently from other neural networks models used by most soft sensors in literature, can model the nonlinear dynamical properties present in the noisy real-world data by using a two-layer structure with efficient training: a recurrent nonlinear layer with fixed randomly generated weights and a linear adaptive read- out output layer. Experimental results show that ESNs are a promising technique to model soft sensors in an industrial setting.

}, isbn = {978-3-319-23981-1}, doi = {https://doi.org/10.1007/978-3-319-23983-5_35}, url = {https://link.springer.com/chapter/10.1007/978-3-319-23983-5_35}, author = {Eric A Antonelo and Eduardo Camponogara} } @article {Antonelo2014, title = {On Learning Navigation Behaviors for Small Mobile Robots With Reservoir Computing Architectures}, journal = {IEEE Transactions on Neural Networks and Learning Systems}, volume = {26}, number = {4}, year = {2015}, pages = {763-780}, abstract = {

This paper proposes a general reservoir computing (RC) learning framework that can be used to learn navigation behaviors for mobile robots in simple and complex unknown partially observable environments. RC provides an efficient way to train recurrent neural networks by letting the recurrent part of the network (called reservoir) be fixed while only a linear readout output layer is trained. The proposed RC framework builds upon the notion of navigation attractor or behavior that can be embedded in the high-dimensional space of the reservoir after learning. The learning of multiple behaviors is possible because the dynamic robot behavior, consisting of a sensory- motor sequence, can be linearly discriminated in the high- dimensional nonlinear space of the dynamic reservoir. Three learning approaches for navigation behaviors are shown in this paper. The first approach learns multiple behaviors based on the examples of navigation behaviors generated by a supervisor, while the second approach learns goal-directed navigation behaviors based only on rewards. The third approach learns complex goal- directed behaviors, in a supervised way, using a hierarchical architecture whose internal predictions of contextual switches guide the sequence of basic navigation behaviors toward the goal.

}, doi = {10.1109/TNNLS.2014.2323247}, author = {Eric A Antonelo and Benjamin Schrauwen} } @article {Antonelo2015, title = {System Identification of a Vertical Riser Model with Echo State Networks}, journal = {IFAC-PapersOnLine (2nd IFAC Workshop on Automatic Control in Offshore Oil and Gas Production)}, volume = {48}, year = {2015}, pages = {304-310}, abstract = {

System identification of highly nonlinear dynamical systems, important for reducing time complexity in long simulations, is not trivial using more traditional methods such as recurrent neural networks (RNNs) trained with back-propagation through time. The recently introduced Reservoir Computing (RC)\∗ approach to training RNNs is a viable and powerful alternative which renders fast training and high performance. In this work, a single Echo State Network (ESN), a flavor of RC, is employed for system identification of a vertical riser model which has stationary and oscillatory signal behaviors depending of the production choke opening input variable. It is shown experimentally that these different behaviors are learned by constraining the high-dimensional reservoir states to attractor subspaces in which the specific behavior is represented. Further experiments show the stability of the identified system.

}, issn = {2405-8963}, doi = {10.1016/j.ifacol.2015.08.048}, url = {http://www.sciencedirect.com/science/article/pii/S2405896315009155}, author = {Eric A Antonelo and Eduardo Camponogara and Agustinho Plucenio} } @article {Antonelo2011, title = {Learning Slow Features with Reservoir Computing for Biologically-inspired Robot Localization}, journal = {Neural Networks}, volume = {25}, number = {1}, year = {2012}, pages = {178-190}, abstract = {

This work proposes a hierarchical biologically-inspired architecture for learning sensor-based spatial representations of a robot environment in an unsupervised way. The first layer is comprised of a fixed randomly generated recurrent neural network, the reservoir, which projects the input into a high-dimensional, dynamic space. The second layer learns instantaneous slowly-varying signals from the reservoir states using Slow Feature Analysis (SFA), whereas the third layer learns a sparse coding on the SFA layer using Independent Component Analysis (ICA). While the SFA layer generates non-localized activations in space, the ICA layer presents high place selectivity, forming a localized spatial activation, characteristic of place cells found in the hippocampus area of the rodent\’s brain. We show that, using a limited number of noisy short-range distance sensors as input, the proposed system learns a spatial representation of the environment which can be used to predict the actual location of simulated and real robots, without the use of odometry. The results confirm that the reservoir layer is essential for learning spatial representations from low-dimensional input such as distance sensors. The main reason is that the reservoir state reflects the recent history of the input stream. Thus, this fading memory is essential for detecting locations, mainly when locations are ambiguous and characterized by similar sensor readings.

}, author = {Eric A Antonelo and Benjamin Schrauwen} } @conference {Antonelo2011a, title = {Learning navigation attractors for mobile robots with reinforcement learning and reservoir computing}, booktitle = {Proceedings of the X Brazilian Congress on Computational Intelligence (CBIC)}, year = {2011}, month = {Nov.}, address = {Fortaleza, Brazil}, abstract = {Autonomous robot navigation in partially observable environments is a complex task because the state of the environment can not be completely determined only by the current sensory readings of a robot. This work uses the recently introduced paradigm for training recurrent neural networks (RNNs), called reservoir computing (RC), to model multiple navigation attractors in partially observable environments. In RC, the RNN with randomly generated fixed weights, called reservoir, projects the input into a high-dimensional dynamic space. Only the readout output layer is trained using standard linear regression techniques, and in this work, is used to approximate the state-action value function. By using a policy iteration framework, where an alternating sequence of policy improvement (samples generation from environment interaction) and policy evaluation (network training) steps are performed, the system is able to shape navigation attractors so that, after convergence, the robot follows the correct trajectory towards the goal. The experiments are accomplished using an e-puck robot extended with 8 distance sensors in a rectangular environment with an obstacle between the robot and the target region. The task is to reach the goal through the correct side of the environment, which is indicated by a temporary stimulus previously observed at the beginning of the episode. We show that the reservoir-based system (with short-term memory) can model these navigation attractors, whereas a feedforward network without memory fails to do so.}, author = {Eric A Antonelo and Stefan Depeweg and Benjamin Schrauwen} } @mastersthesis {Antonelo2011b, title = {Reservoir Computing Architectures for Modeling Robot Navigation Systems}, year = {2011}, school = {Faculty of Engineering, Ghent University}, type = {phd}, abstract = {

This thesis proposes a new efficient and biologically inspired way of modeling navigation tasks for autonomous mobile robots having restrictions on cost, energy consumption, and computational complexity (such as household and assistant robots). It is based on the recently proposed Reservoir Computing approach for training Recurrent Neural Networks.

Robot Navigation Systems
Autonomous mobile robots must be able to safely and purposefully navigate in complex dynamic environments, preferentially considering a restricted amount of computational power as well as limited energy consumption. In order to turn these robots into commercially viable domestic products with intelligent, abstract computational capabilities, it is also necessary to use inexpensive sensory apparatus such as a few infra-red distance sensors of limited accuracy.
Current state-of-the-art methods for robot localization and navigation require fully equipped robotic platforms usually possessing expensive laser scanners for environment mapping, a considerable amount of computational power, and extensive explicit modeling of the environment and of the task.

This thesis
The research presented in this thesis is a step towards creating intelligent autonomous mobile robots with abstract reasoning capabilities using a limited number of very simple raw noisy sensory signals, such as distance sensors. The basic assumption is that the low-dimensional sensory signal can be projected into a high-dimensional dynamic space where learning and computation is performed by linear methods (such as linear regression), overcoming sensor aliasing problems commonly found in robot navigation tasks. This form of computation is known in the literature as Reservoir Computing (RC), and the Echo State Network is a particular RC model used in this work and characterized by having the high-dimensional space implemented by a discrete analog recurrent neural network with fading memory properties. This thesis proposes a number of Reservoir Computing architectures which can be used in a variety of autonomous navigation tasks, by\  modeling implicit abstract representations of an environment as well as navigation behaviors which can be sequentially executed in the physical environment or simulated as a plan in deliberative goal-directed tasks.

Navigation attractors
A navigation attractor is a reactive robot behavior defined by a temporal pattern of sensory-motor coupling through the environment space. Under this scheme, a robot tends to follow a trajectory with attractor-like characteristics in space. These navigation attractors are characterized by being robust to noise and unpredictable events and by having inherent collision avoidance skills.
In this work, it is shown that an RC network can model not only one behavior, but multiple navigation behaviors by shifting the operating point of the dynamical reservoir system into different \emph{sub-space attractors} using additional external inputs representing the selected behavior. The sub-space attractors emerge from the coupling existing between the RC network, which controls the autonomous robot, and the environment. All this is achieved under an imitation learning framework which trains the RC network using examples of navigation behaviors generated by a supervisor controller or a human.

Implicit spatial representations
From the stream of sensory input given by distance sensors, it is possible to construct implicit spatial representations of an environment by using Reservoir Computing networks. These networks are trained in a supervised way to predict locations at different levels of abstraction, from continuous-valued robot\&$\#$39;s pose in the global coordinate\&$\#$39;s frame, to more abstract locations such as small delimited areas and rooms of a robot environment. The high-dimensional reservoir projects the sensory input into a dynamic system space, whose characteristic fading memory disambiguates the sensory space, solving the sensor aliasing problems where multiple different locations generate similar sensory readings from the robot\&$\#$39;s perspective.

Hierarchical networks for goal-directed navigation
It is possible to model navigation attractors and implicit spatial representations with the same type of RC network.
By constructing an hierarchical RC architecture which combines the aforementioned modeling skills in two different reservoir modules operating at different timescales, it is possible to achieve complex context-dependent sensory-motor coupling in unknown environments. The general idea is that the network trained to predict the location and orientation of the robot in this architecture can be used to select appropriate navigation attractors according to the current context, by shifting the operating point of the navigation reservoir to a sub-space attractor.
As the robot navigates from one room to the next, a corresponding context switch selects a new reactive navigation behavior. This continuous sequence of context switches and reactive behaviors, when combined with an external input indicating the destination room, leads ultimately to a goal-directed navigation system, purely trained in a supervised way with examples of sensory-motor coupling.

Generative modeling of environment-robot dynamics
RC networks trained to predict the position of the robot from the sensory signals learns forward models of the robot.
By using a generative RC network which predicts not only locations but also sensory nodes, it is possible to use the network in the opposite direction for predicting local environmental sensory perceptions from the robot position as input, thus learning an inverse model.\ 
The implicit map learned by forward models can be made explicit, by running the RC network in reverse: predict the local sensory signals given the location of the robot as input (inverse model).
which are fed back to the reservoir, it is possible to\ 
internally predict future scenarios and behaviors without actually experiencing them in the current environment (a process analogous to dreaming), constituting a planning-like capability which opens new possibilities for deliberative navigation systems.

Unsupervised learning of spatial representations
In order to achieve a higher degree of autonomy in the learning process of RC-based navigation systems which use implicit learned models of the environment for goal-directed navigation, a new architecture is proposed. Instead of using linear regression, an unsupervised learning method which extracts slowly-varying output signals from the reservoir states, called Slow Feature Analysis, is used to generate self-organized spatial representations at the output layer, without the requirement of labeling training data with the desired locations.\ 
It is shown experimentally that the proposed RC-SFA architecture is empowered with an unique combination of short-term memory and non-linear transformations which overcomes the hidden state problem present in robot navigation tasks.\  In addition, experiments with simulated and real robots indicate that spatial activations generated by the trained network show similarities to the activations of CA1 hippocampal cells of rats (a specific group of neurons in the hippocampus).

}, author = {Eric A Antonelo} } @proceedings {Antonelo2010, title = {Supervised Learning of Internal Models for Autonomous Goal-oriented Robot Navigation using Reservoir Computing}, journal = {Proceedings of the IEEE International Conference on Robotics and Automation}, year = {2010}, month = {May}, pages = {2959-2964}, address = {Anchorage, AK}, abstract = {

In this work we propose a hierarchical architec- ture which constructs internal models of a robot environment for goal-oriented navigation by an imitation learning process. The proposed architecture is based on the Reservoir Computing paradigm for training Recurrent Neural Networks (RNN). It is composed of two randomly generated RNNs (called reservoirs), one for modeling the localization capability and one for learning the navigation skill. The localization module is trained to detect the current and previously visited robot rooms based only on 8 noisy infra-red distance sensors. These predictions together with distance sensors and the desired goal location are used by the navigation network to actually steer the robot through the environment in a goal-oriented manner. The training of this architecture is performed in a supervised way (with examples of trajectories created by a supervisor) using linear regression on the reservoir states. So, the reservoir acts as a temporal kernel projecting the inputs to a rich feature space, whose states are linearly combined to generate the desired outputs. Experimental results on a simulated robot show that the trained system can localize itself within both simple and large unknown environments and navigate successfully to desired goals.

}, doi = {10.1109/ROBOT.2010.5509212}, author = {Eric A Antonelo and Benjamin Schrauwen} } @proceedings {Antonelo2009b, title = {On Different Learning Approaches with Echo State Networks for Localization of Small Mobile Robots}, journal = {Proceedings of the IX Brazilian Conference on Neural Networks (CBRN)}, year = {2009}, publisher = {SBRN}, abstract = {

Animals such as rats have innate and robust localization capabilities which allow them to navigate to goals in a maze. The rodent\’s hippocampus, with the so called place cells, is responsible for such spatial processing. This work seeks to model these place cells using either supervised or unsupervised learning techniques. More specifically, we use a randomly generated recurrent neural network (the reservoir) as a non-linear temporal kernel to expand the input to a rich dynamic space. The reservoir states are linearly combined (using linear regression) or, in the unsupervised case, are used for extracting slowly-varying features from the input to form place cells (the architectures are organized in hierarchical layers). Experiments show that a small mobile robot with cheap and low-range distance sensors can learn to self-localize in its environment with the proposed systems.

}, doi = {10.21528/CBRN2009-067}, url = {http://abricom.org.br/en/events/cbrn_2009/067_CBRN2009/}, author = {Eric A Antonelo and Benjamin Schrauwen} } @proceedings {Waegeman2009, title = {Modular reservoir computing networks for imitation learning of multiple robot behaviors}, journal = {Proc. of the IEEE Int. Symp. on Computational Intelligence in Robotics and Automation (CIRA)}, year = {2009}, pages = {27-32}, abstract = {

Autonomous mobile robots must accomplish tasks in unknown and noisy environments. In this context, learning robot behaviors in an imitation based approach would be desirable in the perspective of service robotics as well as of learning robots. In this work, we use Reservoir Computing (RC) for learning robot behaviors by demonstration. In RC, a randomly generated recurrent neural network, the reservoir, projects the input to a dynamic temporal space. The reservoir states are mapped into a readout output layer which is the solely part being trained using standard linear regression. In this paper, we use a two layered modular structure, where the first layer comprises two RC networks, each one for learning primitive behaviors, namely, obstacle avoidance and target seeking. The second layer is composed of one RC network for behavior combination and coordination. The hierarchical RC network learns by examples given by simple controllers which implement the primitive behaviors. We use a simulation model of the e-puck robot which has distance sensors and a camera that serves as input for our system. The experiments show that, after training, the robot learns to coordinate the Goal Seeking (GS) and the Object Avoidance (OA) behaviors in unknown environments, being able to capture targets and navigate efficiently.

}, doi = {10.1109/CIRA.2009.5423194}, author = {Waegeman, Tim and Eric A Antonelo and wyffels, Francis and Benjamin Schrauwen} } @proceedings {Antonelo2009a, title = {Towards Autonomous Self-localization of Small Mobile Robots using Reservoir Computing and Slow Feature Analysis}, journal = {Proceedings of the IEEE International Conference on Systems, Man, and Cybernetics (SMC)}, year = {2009}, pages = {3818{\textendash}3823}, abstract = {

Biological systems such as rats have special brain structures which process spatial information from the envi- ronment. They have efficient and robust localization abilities provided by special neurons in the hippocampus, namely place cells. This work proposes a biologically plausible architecture which is based on three recently developed techniques: Reservoir Computing (RC), Slow Feature Analysis (SFA), and Independent Component Analysis (ICA). The bottom layer of our RC-SFA architecture is a reservoir of recurrent nodes which process the information from the robot\’s distance sensors. It provides a temporal kernel of rich dynamics which is used by the upper two layers (SFA and ICA) to autonomously learn place cells. Experiments with an e-puck robot with 8 infra-red sensors (which measure distances in [4-30] cm) show that the learning system based on RC-SFA provides a self-organized formation of place cells that can either distinguish between two rooms or to detect the corridor connecting them.

}, doi = {10.1109/ICSMC.2009.5346617}, author = {Eric A Antonelo and Benjamin Schrauwen} } @proceedings {Antonelo2009, title = {Unsupervised Learning in Reservoir Computing: Modeling Hippocampal Place Cells for Small Mobile Robots}, journal = {ICANN {\textquoteright}09: Proceedings of the 19th International Conference on Artificial Neural Networks}, volume = {5768}, year = {2009}, pages = {747-756}, publisher = {Springer-Verlag}, address = {Berlin, Heidelberg}, abstract = {

Biological systems (e.g., rats) have efficient and robust local- ization abilities provided by the so called, place cells, which are found in the hippocampus of rodents and primates (these cells encode locations of the animal\’s environment). This work seeks to model these place cells by employing three (biologically plausible) techniques: Reservoir Com- puting (RC), Slow Feature Analysis (SFA), and Independent Component Analysis (ICA). The proposed architecture is composed of three layers, where the bottom layer is a dynamic reservoir of recurrent nodes with fixed weights. The upper layers (SFA and ICA) provides a self-organized formation of place cells, learned in an unsupervised way. Experiments show that a simulated mobile robot with 17 noisy short-range distance sensors is able to self-localize in its environment with the proposed ar- chitecture, forming a spatial representation which is dependent on the robot direction.

}, issn = {978-3-642-04274-4}, doi = {http://dx.doi.org/10.1007/978-3-642-04274-4_77}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @article {Antonelo2008b, title = {Event detection and localization for small mobile robots using reservoir computing}, journal = {Neural Networks}, volume = {21}, number = {6}, year = {2008}, pages = {862{\textendash}871}, abstract = {Reservoir Computing (RC) techniques use a fixed (usually randomly created) re- current neural network, or more generally any dynamic system, which operates at the edge of stability, where only a linear static readout output layer is trained by standard linear regression methods. In this work, RC is used for detecting complex events in autonomous robot navigation. This can be extended to robot localiza- tion tasks which are solely based on a few low-range, high-noise sensory data. The robot thus builds an implicit map of the environment (after learning) that is used for efficient localization by simply processing the stream of distance sensors. These techniques are demonstrated in both a simple simulation environment and in the physically realistic Webots simulation of the commercially available e-puck robot, using several complex and even dynamic environments.}, doi = {doi:10.1016/j.neunet.2008.06.010}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @proceedings {Antonelo2008a, title = {Imitation Learning of an Intelligent Navigation System for Mobile Robots using Reservoir Computing}, journal = {Proceedings of the 10th Brazilian Symposium on Neural Networks (SBRN)}, year = {2008}, pages = {93-98}, abstract = {

The design of an autonomous navigation system for mobile robots can be a tough task. Noisy sensors, unstructured environments and unpredictability are among the problems which must be overcome. Reservoir Computing (RC) uses a randomly created recurrent neural network (the reservoir) which functions as a temporal kernel of rich dynamics that projects the input to a high dimensional space. This projection is mapped into the desired output (only this mapping must be learned with standard linear regression methods). In this work, RC is used for imitation learning of navigation behaviors generated by an intelligent navigation system in the literature. Obstacle avoidance, exploration and target seeking behaviors are reproduced with an increase in stability and robustness over the original controller. Experiments also show that the system generalizes the behaviors for new environments.

}, issn = {978-0-7695-3361-2}, doi = {10.1109/SBRN.2008.32}, url = {https://ieeexplore.ieee.org/document/4665898/}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @proceedings {Antonelo2008, title = {Mobile Robot Control in the Road Sign Problem using Reservoir Computing Networks}, journal = {Proceedings of the IEEE Int. Conf. on Robotics and Automation (ICRA)}, year = {2008}, pages = {911-916}, abstract = {

In this work we tackle the road sign problem with Reservoir Computing (RC) networks. The T-maze task (a particular form of the road sign problem) consists of a robot in a T-shaped environment that must reach the correct goal (left or right arm of the T-maze) depending on a previously received input sign. It is a control task in which the delay period between the sign received and the required response (e.g., turn right or left) is a crucial factor. Delayed response tasks like this one form a temporal problem that can be handled very well by RC networks. Reservoir Computing is a biologically plausible technique which overcomes the problems of previous algorithms such as Backpropagation Through Time - which exhibits slow (or non-) convergence on training. RC is a new concept that includes a fast and efficient training algorithm. We show that this simple approach can solve the T-maze task efficiently.

}, issn = {978-1-4244-1646-2}, doi = {10.1109/ROBOT.2008.4543321}, url = {https://ieeexplore.ieee.org/document/4543321/}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @conference {Antonelo2008c, title = {Modeling Multiple Autonomous Robot Behaviors and Behavior Switching with a Single Reservoir Computing Network}, booktitle = {Proceedings of the IEEE International Conference on Systems, Man and Cybernetics}, year = {2008}, month = {Oct.}, pages = {1843{\textendash}1848}, publisher = {IEEE}, organization = {IEEE}, address = {Singapore}, abstract = {

Reservoir Computing (RC) uses a randomly created Recurrent Neural Network as a reservoir of rich dynamics which projects the input to a high dimensional space. These projections are mapped to the desired output using a linear output layer, which is the only part being trained by standard linear regression. In this work, RC is used for imitation learning of multiple behaviors which are generated by different controllers using an intelligent navigation system for mobile robots previously published in literature. Target seeking and exploration behaviors are conflicting behaviors which are modeled with a single RC network. The switching between the learned behaviors is imple- mented by an extra input which is able to change the dynamics of the reservoir, and in this way, change the behavior of the system. Experiments show the capabilities of Reservoir Computing for modeling multiple behaviors and behavior switching.

}, isbn = {978-1-4244-2383-5}, doi = {10.1109/ICSMC.2008.4811557}, url = {https://ieeexplore.ieee.org/document/4811557/}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @proceedings {Antonelo2007, title = {Event detection and localization in mobile robot navigation using reservoir computing}, journal = {Artificial Neural Networks -- ICANN 2007}, volume = {Lecture Notes in Computer Science, vol 4669}, year = {2007}, pages = {660-669}, publisher = {Springer-Verlag}, edition = {de S{\'a} J.M., Alexandre L.A., Duch W., Mandic D. }, abstract = {

Reservoir Computing (RC) uses a randomly created recur- rent neural network where only a linear readout layer is trained. In this work, RC is used for detecting complex events in autonomous robot navi- gation. This can be extended to robot localization based solely on sensory information. The robot thus builds an implicit map of the environment without the use of odometry data. These techniques are demonstrated in simulation on several complex and even dynamic environments.

}, keywords = {ESN}, issn = {978-3-540-74695-9}, doi = {10.1007/978-3-540-74695-9_68}, url = {https://link.springer.com/chapter/10.1007/978-3-540-74695-9_68}, author = {Eric A Antonelo and Benjamin Schrauwen and Xavier Dutoit and Dirk Stroobandt and Marnix Nuttin} } @proceedings {Antonelo2007a, title = {Experiments with Reservoir Computing on the road sign problem}, journal = {Proceedings of the VIII Brazilian Congress on Neural Networks (CBRN)}, year = {2007}, publisher = {SBRN}, abstract = {

The road sign problem is tackled in this work with Reservoir Computing (RC) networks. These networks are made of a fixed recurrent neural network where only a readout layer is trained. In the road sign problem, an agent has to decide at some point in time which action to take given relevant information gathered in the past. We show that RC can handle simple and complex T-maze tasks (which are a subdomain of the road sign problem).

}, doi = {10.21528/CBRN2007-047}, url = {http://abricom.org.br/eventos/cbrn_2007/50100047-2/}, author = {Eric A Antonelo and Benjamin Schrauwen and Dirk Stroobandt} } @article {Antonelo2007b, title = {Generative Modeling of Autonomous Robots and their Environments using Reservoir Computing}, journal = {Neural Processing Letters}, volume = {26}, number = {3}, year = {2007}, pages = {233{\textendash}249}, abstract = {

Autonomous mobile robots form an important research topic in the field of robotics due to their near-term applicability in the real world as domestic service robots. These robots must be designed in an efficient way using training sequences. They need to be aware of their position in the environment and also need to create models of it for deliberative planning. These tasks have to be performed using a limited number of sensors with low accuracy, as well as with a restricted amount of computational power. In this contribution we show that the recently emerged paradigm of Reservoir Computing (RC) is very well suited to solve all of the above mentioned problems, namely learning by example, robot localization, map and path generation. Reservoir Computing is a technique which enables a system to learn any time-invariant filter of the input by training a simple linear regressor that acts on the states of a high- dimensional but random dynamic system excited by the inputs. In addition, RC is a simple technique featuring ease of training, and low computational and memory demands.

}, doi = {http://dx.doi.org/10.1007/s11063-007-9054-9}, author = {Eric A Antonelo and Benjamin Schrauwen and Jan Van Campenhout} } @proceedings {antonelo2006, title = {Modular Neural Network and Classical Reinforcement Learning for Autonomous Robot Navigation: Inhibiting Undesirable Behaviors}, journal = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)}, year = {2006}, month = {Jul.}, pages = {498-505}, publisher = {IEEE}, address = {Vancouver, BC}, abstract = {

Classical reinforcement learning mechanisms and a modular neural network are unified to conceive an intelligent autonomous system for mobile robot navigation. The conception aims at inhibiting two common navigation deficiencies: generation of unsuitable cyclic trajectories and ineffectiveness in risky configurations. Different design apparatuses are considered to compose a system to tackle with these navigation difficulties, for instance: 1) neuron parameter to simultaneously memorize neuron activities and function as a learning factor, 2) reinforcement learning mechanisms to adjust neuron parameters (not only synapse weights), and 3) a inner-triggered reinforcement. Simulation results show that the proposed system circumvents difficulties caused by specific environment configurations, improving the relation between collisions and captures.

}, keywords = {reinforcement learning}, isbn = {0-7803-9490-9}, doi = {10.1109/IJCNN.2006.246723}, url = {https://ieeexplore.ieee.org/document/1716134/}, author = {Eric A Antonelo and Albert-Jan Baerlvedt and Thorsteinn Rognvaldsson and Mauricio Figueiredo} } @mastersthesis {antonelo2006thesis, title = {A Neural Reinforcement Learning Approach for Behavior Acquisition in Intelligent Autonomous Systems}, volume = {Master of Science with a major in Computer Systems Engineering}, year = {2006}, school = {Halmstad University}, type = {masters}, abstract = {

In this work new artificial learning and innate control mechanisms are proposed for application in autonomous behavioral systems for mobile robots. An autonomous system (for mobile robots) existent in the literature is enhanced with respect to its capacity of exploring the environment and avoiding risky configurations (that lead to collisions with obstacles even after learning). The particular autonomous system is based on modular hierarchical neural networks. Initially,the autonomous system does not have any knowledge suitable for exploring the environment (and capture targets {\oe} foraging). After a period of learning,the system generates efficientobstacle avoid ance and target seeking behaviors. Two particular deficiencies of the forme rautonomous system (tendency to generate unsuitable cyclic trajectories and ineffectiveness in risky configurations) are discussed and the new learning and controltechniques (applied to the autonomous system) are verified through simulations. It is shown the effectiveness of the proposals: theautonomous system is able to detect unsuitable behaviors (cyclic trajectories) and decrease their probability of appearance in the future and the number of collisions in risky situations is significantly decreased. Experiments also consider maze environments (with targets distant from each other) and dynamic environments (with moving objects).

}, keywords = {reinforcement learning}, url = {http://hh.diva-portal.org/smash/record.jsf?pid=diva2\%3A237466\&dswid=7091}, author = {Eric A Antonelo} } @proceedings {292, title = {Evolutionary fuzzy system for architecture control in a constructive neural network}, journal = {6th IEEE International Symposium on Computational Intelligence in Robotics and Automation}, year = {2005}, pages = {541-546}, publisher = {IEEE}, abstract = {

This work describes an evolutionary system to control the growth of a constructive neural network for autonomous navigation. A classifier system generates Takagi- Sugeno fuzzy rules and controls the architecture of a constructive neural network. The performance of the mobile robot guides the evolutionary learning mechanism. Experiments show the efficiency of the classifier fuzzy system for analyzing if it is worth inserting a new neuron into the architecture.

}, issn = {0-7803-9355-4}, doi = {10.1109/CIRA.2005.1554333}, url = {https://ieeexplore.ieee.org/abstract/document/1554333/}, author = {Rodrigo Calvo and Eric A Antonelo and Mauricio Figueiredo} } @proceedings {antonelo2005, title = {Intelligent autonomous navigation for mobile robots: spatial concept acquisition and object discrimination}, journal = {Proceedings of the 6th IEEE International Symposium on Computational Intelligence in Robotics and Automation}, year = {2005}, pages = {553-557}, publisher = {IEEE}, address = {Helsinki, Finland}, abstract = {

An autonomous system able to construct its own navigation strategy for mobile robots is proposed. The navigation strategy is molded from navigation experiences (succeeding as the robot navigates) according to a classical reinforcement learning procedure. The autonomous system is based on modular hierarchical neural networks. Initially the navigation performance is poor (many collisions occur). Computer simulations show that after a period of learning the autonomous system generates efficient obstacle avoidance and target seeking behaviors. Experiments also offer support for concluding that the autonomous system develops a variety of object discrimination capability and of spatial concepts.

}, issn = {0-7803-9355-4}, doi = {10.1109/CIRA.2005.1554335}, url = {https://ieeexplore.ieee.org/document/1554335/}, author = {Eric A Antonelo and Mauricio Figueiredo and Albert-Jan Baerlvedt and Rodrigo Calvo} } @conference {293, title = {Autonomous intelligent systems applied to robot navigation: spatial concept acquisition and object discrimination}, booktitle = {2nd National Meeting of Intelligent Robotics (II ENRI) in the Congress of the Brazilian Computer Society (in Portuguese)}, year = {2004}, abstract = {

In this work, it is proposed an autonomous system capable of constructing its navigation strategy for mobile robots coherently with the acquired environmental knowledge: spatial proximity concept and color patterns associated to each type of object. The autonomous system, based on neural networks, doens\’t have innate behaviors of target seeking or obstacle avoidance, but it learns these specific behaviors for each type of object as the robot interacts with the environment. Simulation results are showed and confirm the system\’s potenciality, as well as its generalization capacity.

}, author = {Eric A Antonelo and Mauricio Figueiredo} } @conference {294, title = {Neural structures for modeling of biological declarative memory applied to robot autonomous navigation systems}, booktitle = {4th Forum of Informatics and Technology of Maring{\'a} (FITEM) (in Portuguese)}, year = {2002}, abstract = {

, pp. (2002)

}, author = {Eric A Antonelo and Mauricio Figueiredo} }