diff --git a/packages/flutter/lib/src/animation/listener_helpers.dart b/packages/flutter/lib/src/animation/listener_helpers.dart
index acecaa12fb..4813a0caf6 100644
--- a/packages/flutter/lib/src/animation/listener_helpers.dart
+++ b/packages/flutter/lib/src/animation/listener_helpers.dart
@@ -91,7 +91,7 @@ mixin AnimationEagerListenerMixin {
 /// and [didUnregisterListener]. Implementations of these methods can be obtained
 /// by mixing in another mixin from this library, such as [AnimationLazyListenerMixin].
 mixin AnimationLocalListenersMixin {
-  final ObserverList<VoidCallback> _listeners = ObserverList<VoidCallback>();
+  final HashedObserverList<VoidCallback> _listeners = HashedObserverList<VoidCallback>();
 
   /// Called immediately before a listener is added via [addListener].
   ///
diff --git a/packages/flutter/lib/src/foundation/observer_list.dart b/packages/flutter/lib/src/foundation/observer_list.dart
index 27878026ee..d1bae68546 100644
--- a/packages/flutter/lib/src/foundation/observer_list.dart
+++ b/packages/flutter/lib/src/foundation/observer_list.dart
@@ -43,12 +43,15 @@ class ObserverList<T> extends Iterable<T> {
   ///
   /// Returns whether the item was present in the list.
   bool remove(T item) {
-    _isDirty = true;
-    _set.clear(); // Clear the set so that we don't leak items.
-    return _list.remove(item);
+   final bool removed = _list.remove(item);
+    if (removed) {
+      _isDirty = true;
+      _set.clear(); // Clear the set so that we don't leak items.
+    }
+    return removed;
   }
 
-  /// Removes all items from the list.
+  /// Removes all items from the [ObserverList].
   void clear() {
     _isDirty = false;
     _list.clear();
@@ -78,6 +81,10 @@ class ObserverList<T> extends Iterable<T> {
   @override
   bool get isNotEmpty => _list.isNotEmpty;
 
+  /// Creates a List containing the elements of the [ObserverList].
+  ///
+  /// Overrides the default implementation of the [Iterable] to reduce number
+  /// of allocations.
   @override
   List<T> toList({bool growable = true}) {
     return _list.toList(growable: growable);
@@ -126,6 +133,9 @@ class HashedObserverList<T> extends Iterable<T> {
     return true;
   }
 
+  /// Removes all items from the [HashedObserverList].
+  void clear() => _map.clear();
+
   @override
   bool contains(Object? element) => _map.containsKey(element);
 
@@ -137,4 +147,18 @@ class HashedObserverList<T> extends Iterable<T> {
 
   @override
   bool get isNotEmpty => _map.isNotEmpty;
+
+  /// Creates a List containing the elements of the [HashedObserverList].
+  ///
+  /// Overrides the default implementation of [Iterable] to reduce number of
+  /// allocations.
+  @override
+  List<T> toList({bool growable = true}) {
+    final Iterator<T> iterator = _map.keys.iterator;
+    return List<T>.generate(
+      _map.length,
+      (_) => (iterator..moveNext()).current,
+      growable: growable,
+    );
+  }
 }
diff --git a/packages/flutter/test/animation/iteration_patterns_test.dart b/packages/flutter/test/animation/iteration_patterns_test.dart
index 62c905ca47..511c9fd8c9 100644
--- a/packages/flutter/test/animation/iteration_patterns_test.dart
+++ b/packages/flutter/test/animation/iteration_patterns_test.dart
@@ -40,7 +40,7 @@ void main() {
     log.clear();
 
     controller.value = 0.4;
-    expect(log, <String>['listener2', 'listener4', 'listener4']);
+    expect(log, <String>['listener2', 'listener4']);
     log.clear();
   });