diff --git a/keras_nlp/__init__.py b/keras_nlp/__init__.py
index e46172e307..30f8a53b16 100644
--- a/keras_nlp/__init__.py
+++ b/keras_nlp/__init__.py
@@ -26,4 +26,5 @@
 from keras_nlp import samplers
 from keras_nlp import tokenizers
 from keras_nlp import utils
-from keras_nlp.version import __version__
+from keras_nlp.version_utils import __version__
+from keras_nlp.version_utils import version
diff --git a/keras_nlp/version.py b/keras_nlp/version_utils.py
similarity index 100%
rename from keras_nlp/version.py
rename to keras_nlp/version_utils.py
diff --git a/pip_build.py b/pip_build.py
index 6bf576df66..3655e6fb63 100644
--- a/pip_build.py
+++ b/pip_build.py
@@ -63,11 +63,21 @@ def export_version_string(version, is_nightly=False):
             )
             f.write(setup_contents)
 
-    # Make sure to export the __version__ string
+    # Overwrite the version string with our package version.
+    with open(os.path.join(package, "src", "version_utils.py")) as f:
+        version_contents = f.readlines()
+    with open(os.path.join(package, "src", "version_utils.py"), "w") as f:
+        for line in version_contents:
+            if line.startswith("__version__"):
+                f.write(f'__version__ = "{version}"\n')
+            else:
+                f.write(line)
+    # Make sure to export the __version__ string.
     with open(os.path.join(package, "__init__.py")) as f:
         init_contents = f.read()
     with open(os.path.join(package, "__init__.py"), "w") as f:
-        f.write(init_contents + "\n\n" + f'__version__ = "{version}"\n')
+        f.write(init_contents)
+        f.write("from keras_nlp.src.version_utils import __version__\n")
 
 
 def build(root_path, is_nightly=False):
diff --git a/setup.py b/setup.py
index 52f5c84fdb..6acd5416d7 100644
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,10 @@ def get_version(rel_path):
 
 HERE = pathlib.Path(__file__).parent
 README = (HERE / "README.md").read_text()
-VERSION = get_version("keras_nlp/version.py")
+if os.path.exists("keras_nlp/version_utils.py"):
+    VERSION = get_version("keras_nlp/version_utils.py")
+else:
+    VERSION = get_version("keras_nlp/src/version_utils.py")
 
 setup(
     name="keras-nlp",