diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts
index 7a8a6262cc..01906c9129 100644
--- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts
+++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts
@@ -37,6 +37,10 @@ function getTestcontainers(): ContainerInfo[] {
     )
 }
 
+function removeContainer(container: ContainerInfo) {
+  execSync(`docker rm ${container.ID}`)
+}
+
 export function getContainerByImage(image: string) {
   const containers = getTestcontainers().filter(x => x.Image.startsWith(image))
   if (containers.length > 1) {
@@ -49,6 +53,10 @@ export function getContainerByImage(image: string) {
   return containers[0]
 }
 
+function getContainerByName(name: string) {
+  return getTestcontainers().find(x => x.Names === name)
+}
+
 export function getContainerById(id: string) {
   return getTestcontainers().find(x => x.ID === id)
 }
@@ -147,6 +155,15 @@ export async function startContainer(container: GenericContainer) {
   key = key.replace(/\//g, "-").replace(/:/g, "-")
   const name = `${key}_testcontainer`
 
+  // If a container has died it hangs around and future attempts to start a
+  // container with the same name will fail. What we do here is if we find a
+  // matching container and it has exited, we remove it before carrying on. This
+  // removes the need to do this removal manually.
+  const existingContainer = getContainerByName(name)
+  if (existingContainer?.State === "exited") {
+    removeContainer(existingContainer)
+  }
+
   container = container
     .withReuse()
     .withLabels({ "com.budibase": "true" })