diff --git a/.changes/unreleased/fixed-20240912-000012.yaml b/.changes/unreleased/fixed-20240912-000012.yaml
new file mode 100644
index 00000000..910b4a76
--- /dev/null
+++ b/.changes/unreleased/fixed-20240912-000012.yaml
@@ -0,0 +1,5 @@
+kind: fixed
+body: Clean up release notes files if any error takes place during the batch process
+time: 2024-09-12T00:00:12.120726403-07:00
+custom:
+    Issue: "714"
diff --git a/cmd/batch.go b/cmd/batch.go
index bfe2bd66..815d0d08 100644
--- a/cmd/batch.go
+++ b/cmd/batch.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -36,10 +37,9 @@ type Batch struct {
 	TemplateCache *core.TemplateCache
 
 	// Computed values
-	config          *core.Config // current configuration
-	writer          io.Writer    // writer we are batching to
-	version         string       // the version we are bumping to
-	versionFilePath string       // path to the file we're writing to
+	config  *core.Config // current configuration
+	writer  io.Writer    // writer we are batching to
+	version string       // the version we are bumping to
 }
 
 func NewBatch(
@@ -192,9 +192,7 @@ func (b *Batch) getBatchData() (*core.BatchData, error) {
 }
 
 //nolint:gocyclo
-func (b *Batch) Run(cmd *cobra.Command, args []string) error {
-	var err error
-
+func (b *Batch) Run(cmd *cobra.Command, args []string) (err error) {
 	// save our version for later use
 	b.version = args[0]
 
@@ -227,19 +225,28 @@ func (b *Batch) Run(cmd *cobra.Command, args []string) error {
 	if b.DryRun {
 		b.writer = cmd.OutOrStdout()
 	} else {
-		b.versionFilePath = filepath.Join(b.config.ChangesDir, b.Project, data.Version+"."+b.config.VersionExt)
+		versionFilePath := filepath.Join(b.config.ChangesDir, b.Project, data.Version+"."+b.config.VersionExt)
 
 		if !b.Force {
-			if exists, existErr := core.FileExists(b.versionFilePath); exists || existErr != nil {
-				return fmt.Errorf("%w: %v", errVersionExists, b.versionFilePath)
+			if exists, existErr := core.FileExists(versionFilePath); exists || existErr != nil {
+				return fmt.Errorf("%w: %v", errVersionExists, versionFilePath)
 			}
 		}
 
-		versionFile, createErr := os.Create(b.versionFilePath)
+		versionFile, createErr := os.Create(versionFilePath)
 		if createErr != nil {
 			return createErr
 		}
 
+		defer func() {
+			if err != nil {
+				removeErr := os.Remove(versionFilePath)
+				if removeErr != nil {
+					err = fmt.Errorf("batching error: %w, removing new file error: %w", err, removeErr)
+				}
+			}
+		}()
+
 		defer versionFile.Close()
 		b.writer = versionFile
 	}
@@ -361,7 +368,6 @@ func (b *Batch) WriteTemplate(
 	}
 
 	if err := b.TemplateCache.Execute(template, b.writer, templateData); err != nil {
-		os.Remove(b.versionFilePath)
 		return err
 	}
 
@@ -385,11 +391,11 @@ func (b *Batch) WriteTemplateFile(
 	var fileBytes []byte
 
 	fileBytes, readErr := os.ReadFile(fullPath)
-	if readErr != nil && !os.IsNotExist(readErr) {
+	if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) {
 		return readErr
 	}
 
-	if os.IsNotExist(readErr) {
+	if errors.Is(readErr, fs.ErrNotExist) {
 		return nil
 	}
 
diff --git a/cmd/batch_test.go b/cmd/batch_test.go
index 43a73e32..ea2fca63 100644
--- a/cmd/batch_test.go
+++ b/cmd/batch_test.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -489,6 +490,9 @@ func TestBatchErrorBadKindFormat(t *testing.T) {
 
 	err := batch.Run(batch.Command, []string{"v0.2.3"})
 	then.NotNil(t, err)
+
+	_, err = os.Stat(filepath.Join(cfg.ChangesDir, "v0.2.3.md"))
+	then.Err(t, fs.ErrNotExist, err)
 }
 
 func TestBatchErrorBadComponentFormat(t *testing.T) {