diff --git a/agb-hashmap/src/lib.rs b/agb-hashmap/src/lib.rs
index 3a4e63a0..a4d6cb08 100644
--- a/agb-hashmap/src/lib.rs
+++ b/agb-hashmap/src/lib.rs
@@ -338,23 +338,12 @@ where
         }
     }
 
-    fn insert_and_get(&mut self, key: K, value: V) -> &'_ mut V {
-        let hash = self.hash(&key);
+    unsafe fn insert_new_and_get(&mut self, key: K, value: V, hash: HashType) -> &'_ mut V {
+        if self.nodes.capacity() <= self.len() {
+            self.resize(self.nodes.backing_vec_size() * 2);
+        }
 
-        let location = if let Some(location) = self.nodes.location(&key, hash) {
-            // SAFETY: location is valid due to the above
-            unsafe {
-                self.nodes
-                    .replace_at_location_unchecked(location, key, value);
-            }
-            location
-        } else {
-            if self.nodes.capacity() <= self.len() {
-                self.resize(self.nodes.backing_vec_size() * 2);
-            }
-
-            self.nodes.insert_new(key, value, hash)
-        };
+        let location = self.nodes.insert_new(key, value, hash);
 
         // SAFETY: location is always valid
         unsafe {
@@ -599,7 +588,7 @@ impl<K, V, ALLOCATOR: ClonableAllocator> IntoIterator for HashMap<K, V, ALLOCATO
 mod entries {
     use core::{alloc::Allocator, hash::Hash};
 
-    use super::{ClonableAllocator, HashMap};
+    use super::{ClonableAllocator, HashMap, HashType};
 
     /// A view into an occupied entry in a `HashMap`. This is part of the [`crate::Entry`] enum.
     pub struct OccupiedEntry<'a, K: 'a, V: 'a, ALLOCATOR: Allocator> {
@@ -695,11 +684,16 @@ mod entries {
     pub struct VacantEntry<'a, K: 'a, V: 'a, ALLOCATOR: Allocator> {
         key: K,
         map: &'a mut HashMap<K, V, ALLOCATOR>,
+        hash: HashType,
     }
 
     impl<'a, K: 'a, V: 'a, ALLOCATOR: ClonableAllocator> VacantEntry<'a, K, V, ALLOCATOR> {
-        pub(crate) fn new(key: K, map: &'a mut HashMap<K, V, ALLOCATOR>) -> Self {
-            Self { key, map }
+        pub(crate) unsafe fn new(
+            key: K,
+            hash: HashType,
+            map: &'a mut HashMap<K, V, ALLOCATOR>,
+        ) -> Self {
+            Self { key, map, hash }
         }
 
         /// Gets a reference to the key that would be used when inserting a value through `VacantEntry`
@@ -717,7 +711,8 @@ mod entries {
         where
             K: Hash + Eq,
         {
-            self.map.insert_and_get(self.key, value)
+            // SAFETY: by construction, this doesn't already exist in the hashmap and we were given the hash and key
+            unsafe { self.map.insert_new_and_get(self.key, value, self.hash) }
         }
     }
 }
@@ -832,7 +827,10 @@ where
                 unsafe { OccupiedEntry::new(key, self, location) },
             )
         } else {
-            Entry::Vacant(VacantEntry::new(key, self))
+            Entry::Vacant(
+                // SAFETY: item doesn't exist yet and the hash is correct here
+                unsafe { VacantEntry::new(key, hash, self) },
+            )
         }
     }
 }
diff --git a/agb/src/display/tiled/vram_manager.rs b/agb/src/display/tiled/vram_manager.rs
index f98673b9..e2ae2d1d 100644
--- a/agb/src/display/tiled/vram_manager.rs
+++ b/agb/src/display/tiled/vram_manager.rs
@@ -7,7 +7,7 @@ use crate::{
     agb_alloc::{block_allocator::BlockAllocator, bump_allocator::StartEnd},
     display::palette16,
     dma::dma_copy16,
-    hash_map::HashMap,
+    hash_map::{Entry, HashMap},
     memory_mapped::MemoryMapped1DArray,
 };
 
@@ -291,10 +291,10 @@ impl VRamManager {
     pub(crate) fn add_tile(&mut self, tile_set: &TileSet<'_>, tile: u16) -> TileIndex {
         let reference = self
             .tile_set_to_vram
-            .get(&TileInTileSetReference::new(tile_set, tile));
+            .entry(TileInTileSetReference::new(tile_set, tile));
 
-        if let Some(reference) = reference {
-            let tile_index = Self::index_from_reference(*reference, tile_set.format);
+        if let Entry::Occupied(reference) = reference {
+            let tile_index = Self::index_from_reference(*reference.get(), tile_set.format);
             let key = tile_index.refcount_key();
             self.reference_counts[key].increment_reference_count();
             return tile_index;
@@ -305,15 +305,13 @@ impl VRamManager {
                 .unwrap()
                 .cast();
         let tile_reference = TileReference(new_reference);
+        reference.or_insert(tile_reference);
 
         self.copy_tile_to_location(tile_set, tile, tile_reference);
 
         let index = Self::index_from_reference(tile_reference, tile_set.format);
         let key = index.refcount_key();
 
-        self.tile_set_to_vram
-            .insert(TileInTileSetReference::new(tile_set, tile), tile_reference);
-
         self.reference_counts
             .resize(self.reference_counts.len().max(key + 1), Default::default());