From a80b4caaa458da5a7b9703409e6357f1cda6ac19 Mon Sep 17 00:00:00 2001 From: "Florine W. Dekker" Date: Fri, 26 May 2023 18:37:54 +0200 Subject: [PATCH] Automove boring files, modernise code and messages Also update dependencies, and clean up a whole bunch of stuff. --- Files.py | 34 +++++ IO.py | 45 +++++++ README.md | 2 +- State.py | 46 +++++++ XEdit.py | 177 ++++++++++++++++++++++++++ config_default.py | 22 ++-- export.py | 311 ---------------------------------------------- fo76dumps.py | 152 ++++++++++++++++++++++ requirements.txt | 2 +- 9 files changed, 467 insertions(+), 324 deletions(-) create mode 100644 Files.py create mode 100644 IO.py create mode 100644 State.py create mode 100644 XEdit.py delete mode 100644 export.py create mode 100644 fo76dumps.py diff --git a/Files.py b/Files.py new file mode 100644 index 0000000..ca923cc --- /dev/null +++ b/Files.py @@ -0,0 +1,34 @@ +import shutil +from pathlib import Path + + +def delete(path: Path) -> None: + """ + Recursively deletes `path` without confirmation. + + :param path: the path to the file/directory to delete + :return: `None` + """ + + if not path.exists(): + return + + if path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + + +def move_into(path: Path, target: Path) -> None: + """ + Moves `path` into the directory `target`, retaining the name of `path`, and creating `target` and its parents if + they do not exist yet. + + :param path: the file/directory to move into `path` + :param target: the directory to move `path` into + :return: `None` + """ + + target.mkdir(exist_ok=True, parents=True) + delete(target / path.name) + shutil.move(path, target / path.name) diff --git a/IO.py b/IO.py new file mode 100644 index 0000000..582959c --- /dev/null +++ b/IO.py @@ -0,0 +1,45 @@ +import os +import subprocess +from pathlib import Path +from typing import List + +from State import cfg + + +def prompt_confirmation(message: str) -> bool: + """ + Prompts the user to confirm `message`, returning `True` if the user inputs `"y"`, returning `False` if the user + inputs `"n"`, and repeating the prompt otherwise. + + :param message: the message to present to the user for confirmation + :return: whether the user confirms the prompt + """ + + while True: + result = input(message).lower() + if result == "y": + return True + elif result == "n": + return False + else: + continue + + +def run_executable(args: List[str], compatdata_path: str, cwd: Path = Path.cwd()) -> None: + """ + Runs the command `args` inside `cwd`; on Windows the command is executed directly on the command line, but on Linux + the command is executed in a Proton instance using `compatdata_path`. + + :param args: the arguments of the command to execute + :param compatdata_path: the path to the compatdata directory for Proton; may be `None` for Windows + :param cwd: the directory to run the command in + :return: `None` + """ + + if cfg.windows: + subprocess.Popen(args, cwd=cwd, stdout=subprocess.DEVNULL).wait() + else: + subprocess.Popen([cfg.proton_path, "run"] + args, cwd=cwd, stdout=subprocess.DEVNULL, + env=dict(os.environ, + STEAM_COMPAT_CLIENT_INSTALL_PATH=cfg.steam_path, + STEAM_COMPAT_DATA_PATH=compatdata_path)).wait() diff --git a/README.md b/README.md index b8f925b..edd16b8 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ explanation. ## Development All dumps are created using a Python script that uses [xEdit](https://tes5edit.github.io/) and [ba2extract](https://f4se.silverlock.org/). -The main script is located at [`export.py`](https://github.com/FWDekker/fo76-dumps/blob/master/export.py). +The main script is located at [`facemation.py`](https://github.com/FWDekker/fo76-dumps/blob/master/export.py). The xEdit scripts are in the [`Edit scripts` directory](https://github.com/FWDekker/fo76-dumps/tree/master/Edit%20scripts). The wiki describes the [dumping process in more detail](https://github.com/FWDekker/fo76-dumps/wiki/Generating-dumps). diff --git a/State.py b/State.py new file mode 100644 index 0000000..60e9d90 --- /dev/null +++ b/State.py @@ -0,0 +1,46 @@ +from pathlib import Path +from types import SimpleNamespace + + +def load_config() -> SimpleNamespace: + """ + Loads configuration from config files and set globals. + + :return: the loaded configuration + """ + + from config_default import config as cfg_default + if Path("config.py").exists(): + from config import config as cfg_user + else: + cfg_user = {} + + my_cfg = cfg_default | cfg_user + if not my_cfg["windows"]: + if ("linux_settings" in cfg_default.keys()) and ("linux_settings" in cfg_user.keys()): + my_cfg["linux_settings"] = cfg_default["linux_settings"] | cfg_user["linux_settings"] + my_cfg = my_cfg | my_cfg["linux_settings"] + my_cfg = SimpleNamespace(**my_cfg) + + # Version of this script + my_cfg.script_version = "3.1.0" + # Path to scripts + my_cfg.script_root = my_cfg.game_root / "Edit scripts/" + # Path to exported dumps + my_cfg.dump_root = my_cfg.script_root / "dumps/" + # Path to store files that have been archived in + my_cfg.dump_archived = my_cfg.dump_root / "_archived/" + # Path to store dump parts in + my_cfg.dump_parts = my_cfg.dump_root / "_parts/" + # Path to store SQLite database at + my_cfg.db_path = my_cfg.dump_root / f"fo76-dumps-v{my_cfg.script_version}-v{my_cfg.game_version}.db" + # Path to `_done.txt` + my_cfg.done_path = my_cfg.dump_root / "_done.txt" + + return my_cfg + + +# Configuration +cfg = load_config() +# Unfinished subprocesses +subprocesses = {} diff --git a/XEdit.py b/XEdit.py new file mode 100644 index 0000000..fe33dd1 --- /dev/null +++ b/XEdit.py @@ -0,0 +1,177 @@ +import shutil +import sqlite3 +import subprocess +from pathlib import Path +from typing import List + +import pandas as pd + +import Files +from IO import prompt_confirmation, run_executable +from State import cfg, subprocesses + + +def xedit() -> None: + """ + Runs xEdit and waits until it closes. + + :return: `None` + """ + + print(f"> Running xEdit using '{cfg.xedit_path}'.\n" + f"> Be sure to double-check version information in the xEdit window!") + + # Check for existing files + if cfg.done_path.exists(): + if not prompt_confirmation(f"> WARNING: " + f"'{cfg.done_path.name}' already exists and must be deleted. " + f"Do you want to DELETE '{cfg.done_path}' and continue? (y/n) "): + exit() + Files.delete(cfg.done_path) + + # Create ini if it does not exist + config_dir = (Path.home() / "Documents/My Games/Fallout 76/" if cfg.windows + else cfg.xedit_compatdata_path / "pfx/drive_c/users/steamuser/Documents/My Games/Fallout 76/") + config_dir.mkdir(exist_ok=True, parents=True) + (config_dir / "Fallout76.ini").touch(exist_ok=True) + + # Store initial `_done.txt` modification time + done_time = cfg.done_path.stat().st_mtime if cfg.done_path.exists() else None + + # Actually run xEdit + run_executable(args=[cfg.xedit_path, f"-D:{cfg.game_root / 'Data/'}", "ExportAll.fo76pas"], + compatdata_path=cfg.xedit_compatdata_path if not cfg.windows else "", + cwd=cfg.script_root) + + # Check if `_done.txt` changed + new_done_time = cfg.done_path.stat().st_mtime if cfg.done_path.exists() else None + if new_done_time is None or done_time == new_done_time: + if not prompt_confirmation(f"> WARNING: " + f"xEdit did not create or update '{cfg.done_path.name}', indicating that the dump " + f"scripts may have failed. " + f"Continue anyway? (y/n) "): + exit() + + # Post-processing + prefix_outputs() + concat_parts() + create_db() + if cfg.enable_archive_xedit: + archive_start() + + print("> Done running xEdit.\n") + + +def concat_parts_of(input_paths: List[Path], target: Path) -> None: + """ + Concatenates the contents of all files in `input_paths` and writes the output to `target`. + + :return: `None` + """ + if len(input_paths) == 0: + return + + input_paths.sort() + + with target.open("wb") as f_out: + for input_path in input_paths: + with input_path.open("rb") as f_in: + shutil.copyfileobj(f_in, f_out) + + +def prefix_outputs() -> None: + """ + Prefixes the exported files with `"tabular."` or `"wiki."` depending on the file type, ignoring files that already + have the appropriate prefix. + + :return: `None` + """ + + print(">> Prefixing files.") + + for file in list(cfg.dump_root.glob("*.csv")) + list(cfg.dump_root.glob("*.wiki")): + prefix = "tabular." if file.suffix == ".csv" else "wiki." + + # Skip already-prefixed files + if not file.stem.startswith(prefix): + file.rename(file.parent / f"{prefix}{file.name}") + + print(">> Done prefixing files.") + + +def concat_parts() -> None: + """ + Concatenates files that have been dumped in parts by the xEdit script, and moves parts to the `"_parts"` + subdirectory. + + :return: `None` + """ + + print(">> Combining dumped CSV parts.") + + print(">>> Combining 'tabular.IDs.csv'.") + parts = list(cfg.dump_root.glob("IDs.csv.*")) + concat_parts_of(parts, cfg.dump_root / "tabular.IDs.csv") + [Files.move_into(part, cfg.dump_parts) for part in parts] + + print(">>> Combining 'wiki.TERM.wiki'.") + parts = list(cfg.dump_root.glob("TERM.wiki.*")) + concat_parts_of(parts, cfg.dump_root / "wiki.TERM.wiki") + [Files.move_into(part, cfg.dump_parts) for part in parts] + + print(">> Done combining dumped CSV parts.") + + +def create_db() -> None: + """ + Imports the dumped CSVs into an SQLite database. + + :return: `None` + """ + + print(f">> Importing CSVs into SQLite database at '{cfg.db_path}'.") + + # Check for existing files + if cfg.db_path.exists(): + if not prompt_confirmation(f">> WARNING: " + f"'{cfg.db_path.name}' already exists and must be deleted. " + f"Do you want to DELETE '{cfg.db_path}' and continue? (y/n) "): + exit() + Files.delete(cfg.db_path) + + # Import into database + with sqlite3.connect(cfg.db_path) as con: + for csv in list(cfg.dump_root.glob("*.csv")): + table_name = csv.stem.split('.', 1)[1] + print(f">>> Importing '{csv.name}' into table '{table_name}'.") + + df = pd.read_csv(csv, quotechar='"', doublequote=True, skipinitialspace=True, + dtype="string", encoding="iso-8859-1") + df.columns = df.columns.str.replace(" ", "_") + df.to_sql(table_name, con, index=False) + + print(f">> Done importing CSVs into SQLite database at '{cfg.db_path}'.") + + +def archive_start() -> None: + """ + Archives all xEdit dumps that are larger than 10MB, moving the files that are archived into `_archived`. + + :return: `None` + """ + + print(">> Archiving large xEdit dumps in the background.") + + for dump in (list(cfg.dump_root.glob("*.csv")) + + list(cfg.dump_root.glob("*.wiki")) + + list(cfg.dump_root.glob("*.db"))): + if dump.stat().st_size < 10000000: + continue + + print(f">>> Starting archiving of '{dump.name}'.") + process = subprocess.Popen([cfg.archiver_path, "a", "-mx9", "-mmt4", f"{dump.name}.7z", dump.name], + cwd=cfg.dump_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT) + subprocesses[dump.name] = {"process": process, + "post": lambda it=dump: Files.move_into(it, cfg.dump_archived)} diff --git a/config_default.py b/config_default.py index 96b2004..13680f9 100644 --- a/config_default.py +++ b/config_default.py @@ -42,13 +42,13 @@ ## Settings for Windows # Path to 7z executable - "archiver_path": r"C:\Program Files\7-Zip\7z.exe", + "archiver_path": Path(r"C:\Program Files\7-Zip\7z.exe"), # Path to game files - "game_root": r"C:\Program Files (x86)\Steam\steamapps\common\Fallout76", + "game_root": Path(r"C:\Program Files (x86)\Steam\steamapps\common\Fallout76"), # Path to xEdit executable - "xedit_path": r"C:\Program Files (x86)\Steam\steamapps\common\Fallout76\FO76Edit64.exe", + "xedit_path": Path(r"C:\Program Files (x86)\Steam\steamapps\common\Fallout76\FO76Edit64.exe"), # Path to ba2extract executable - "ba2extract_path": r"ba2extract.exe", + "ba2extract_path": Path(f"{Path.cwd()}/ba2extract.exe"), ## Settings for Linux @@ -56,20 +56,20 @@ # Path to 7z executable "archiver_path": "7z", # Path to game files - "game_root": f"{Path.home()}/.steam/steam/steamapps/common/Fallout76/", + "game_root": Path(f"{Path.home()}/.steam/steam/steamapps/common/Fallout76/"), # Path to Steam installation - "steam_path": f"{Path.home()}/.steam/steam/", + "steam_path": Path(f"{Path.home()}/.steam/steam/"), # Path to Proton installation - "proton_path": f"{Path.home()}/.local/share/Steam/steamapps/common/Proton - Experimental/proton", + "proton_path": Path(f"{Path.home()}/.local/share/Steam/steamapps/common/Proton - Experimental/proton"), # Path to xEdit executable - "xedit_path": f"{Path.home()}/.steam/steam/steamapps/common/Fallout76/FO76Edit64.exe", + "xedit_path": Path(f"{Path.home()}/.steam/steam/steamapps/common/Fallout76/FO76Edit64.exe"), # Path to xEdit compatdata - "xedit_compatdata_path": f"{Path.home()}/.steam/steam/steamapps/compatdata/INSERT NUMBER HERE/", + "xedit_compatdata_path": Path(f"{Path.home()}/.steam/steam/steamapps/compatdata/INSERT NUMBER HERE/"), # Path to ba2extract executable - "ba2extract_path": "./ba2extract.exe", + "ba2extract_path": Path(f"{Path.cwd()}/ba2extract.exe"), # Path to ba2extract compatdata - "ba2extract_compatdata_path": f"{Path.home()}/.steam/steam/steamapps/compatdata/INSERT NUMBER HERE/", + "ba2extract_compatdata_path": Path(f"{Path.home()}/.steam/steam/steamapps/compatdata/INSERT NUMBER HERE/"), } } diff --git a/export.py b/export.py deleted file mode 100644 index e0f997d..0000000 --- a/export.py +++ /dev/null @@ -1,311 +0,0 @@ -import glob -import os -import shutil -import sqlite3 -import subprocess -from pathlib import Path -from tempfile import TemporaryDirectory -from types import SimpleNamespace -from typing import List - -import pandas as pd - - -def load_config() -> SimpleNamespace: - """Load configuration from config files and set globals.""" - from config_default import config as cfg_default - if Path("config.py").exists(): - from config import config as cfg_user - else: - cfg_user = {} - - my_cfg = cfg_default | cfg_user - if not my_cfg["windows"]: - if ("linux_settings" in cfg_default.keys()) and ("linux_settings" in cfg_user.keys()): - my_cfg["linux_settings"] = cfg_default["linux_settings"] | cfg_user["linux_settings"] - my_cfg = my_cfg | my_cfg["linux_settings"] - my_cfg = SimpleNamespace(**my_cfg) - - # Version of this script - my_cfg.script_version = "3.0.0" - # Path to scripts - my_cfg.script_root = f"{my_cfg.game_root}/Edit scripts" - # Path to exported dumps - my_cfg.dump_root = f"{my_cfg.script_root}/dumps" - # Path to store SQLite database at - my_cfg.db_path = f"{my_cfg.dump_root}/fo76-dumps-v{my_cfg.script_version}-v{my_cfg.game_version}.db" - # Path to `_done.txt` - my_cfg.done_path = f"{my_cfg.dump_root}/_done.txt" - - return my_cfg - - -def prompt_confirmation(message: str) -> bool: - # Prompts the user to confirm `message`, returning `True` if the user inputs `"y"`, returning `False` if the user - # inputs `"n"`, and repeating the prompt otherwise. - while True: - result = input(message).lower() - if result == "y": - return True - elif result == "n": - return False - else: - continue - - -def run_executable(command: str, compatdata_path: str): - """Runs the executable with parameters defined in `command`. On Windows, the command is executed normally. On Linux, - the command is executed in a Proton instance using `compatdata_path`.""" - if cfg.windows: - subprocess.call(command, stdout=subprocess.DEVNULL) - else: - os.system( - f"STEAM_COMPAT_CLIENT_INSTALL_PATH='{cfg.steam_path}' " - f"STEAM_COMPAT_DATA_PATH='{compatdata_path}' " - f"'{cfg.proton_path}' run {command} " - f">/dev/null" - ) - - -def concat_parts_of(input_paths: List[str], output_path: str): - """Concatenates the contents of all files in `input_paths` and writes the output to `output_path`.""" - if len(input_paths) == 0: - return - - input_paths.sort() - - with open(output_path, "wb") as f_out: - for input_path in input_paths: - with open(input_path, "rb") as f_in: - shutil.copyfileobj(f_in, f_out) - - -def xedit(): - """Runs xEdit and waits until it closes. - - Tries to detect whether the script ran successfully by checking if `_done.txt` was created or modified.""" - print("> Running xEdit.\n> Be sure to double-check version information in the xEdit window!") - - # Check for existing files - if Path(cfg.done_path).exists(): - if not prompt_confirmation("> WARNING: '_done.txt' already exists, indicating a dump already exists in the " - "target folder. Continue anyway? (y/n) "): - print("") - return - os.remove(cfg.done_path) - - # Create ini if it does not exist - config_dir = \ - f"{Path.home()}/Documents/My Games/Fallout 76/" if cfg.windows \ - else f"{cfg.xedit_compatdata_path}/pfx/drive_c/users/steamuser/Documents/My Games/Fallout 76/" - Path(config_dir).mkdir(exist_ok=True, parents=True) - Path(f"{config_dir}/Fallout76.ini").touch(exist_ok=True) - - # Store initial `_done.txt` modification time - done_file = Path(cfg.done_path) - done_time = done_file.stat().st_mtime if done_file.exists() else None - - # Actually run xEdit - cwd = os.getcwd() - os.chdir(cfg.script_root) - run_executable(f"\"{cfg.xedit_path}\" -D:\"{cfg.game_root}/Data/\" ExportAll.fo76pas", - cfg.xedit_compatdata_path if not cfg.windows else "") - os.chdir(cwd) - - # Check if `_done.txt` changed - new_done_time = done_file.stat().st_mtime if done_file.exists() else None - if new_done_time is None or done_time == new_done_time: - if not prompt_confirmation("> WARNING: xEdit did not create or update '_done.txt'. Continue anyway? (y/n) "): - print() - return - - # Post-processing - xedit_prefix_outputs() - xedit_concat_parts() - xedit_create_db() - if cfg.enable_archive_xedit: - archive_xedit_start() - - print("> Done running xEdit.\n") - - -def xedit_prefix_outputs(): - """Renames the exported files so that they have a prefix `tabular.` or `wiki.` depending on the file type. - - Files that already have the appropriate prefix are unaffected.""" - print(">> Prefixing files.") - - for filename in glob.glob(f"{cfg.dump_root}/*.csv") + glob.glob(f"{cfg.dump_root}/*.wiki"): - path = Path(filename) - prefix = "tabular." if path.suffix == ".csv" else "wiki." - - if path.stem.startswith(prefix): - # Skip already-prefixed files - return - - path.rename(f"{path.parent}/{prefix}{path.name}") - - print(">> Done prefixing files.") - - -def xedit_concat_parts(): - """Concatenates files that have been dumped in parts by the xEdit script.""" - print(">> Combining dumped CSV parts.") - - print(">>> Combining 'tabular.IDs.csv'.") - concat_parts_of(glob.glob(f"{cfg.dump_root}/IDs.csv.*"), f"{cfg.dump_root}/tabular.IDs.csv") - - print(">>> Combining 'wiki.TERM.wiki'.") - concat_parts_of(glob.glob(f"{cfg.dump_root}/TERM.wiki.*"), f"{cfg.dump_root}/wiki.TERM.wiki") - - print(">> Done combining dumped CSV parts.") - - -def xedit_create_db(): - """Imports the dumped CSVs into an SQLite database.""" - print(f">> Importing CSVs into SQLite database at '{cfg.db_path}'.") - - # Check for existing files - if Path(cfg.db_path).exists(): - if not prompt_confirmation(f">> WARNING: '{Path(cfg.db_path).name}' already exists and will be deleted. " - f"Continue anyway? (y/n) "): - return - os.remove(cfg.db_path) - - # Import into database - with sqlite3.connect(cfg.db_path) as con: - for file in glob.glob(f"{cfg.dump_root}/*.csv"): - path = Path(file) - table_name = path.stem.split('.', 1)[1] - print(f">>> Importing '{path.name}' into table '{table_name}'.") - - df = pd.read_csv(file, quotechar='"', doublequote=True, skipinitialspace=True, dtype="string") - df.columns = df.columns.str.replace(" ", "_") - df.to_sql(table_name, con, index=False) - - print(f">> Done importing CSVs into SQLite database at '{cfg.db_path}'.") - - -def ba2extract(): - """Creates raw dumps using ba2extract.""" - print("> Extracting Bethesda archives.") - - # Extract archives - for target, files in cfg.ba2extract_targets.items(): - print(f">> Extracting {target}.") - temp_dir = TemporaryDirectory(prefix=f"fo76-dumps-{target}") - - run_executable(f"\"{cfg.ba2extract_path}\" \"{cfg.game_root}/Data/{target}\" \"{temp_dir.name}\"", - cfg.ba2extract_compatdata_path if not cfg.windows else "") - - for archive_path, desired_path in files.items(): - desired_path_abs = Path(f"{cfg.dump_root}/raw.{desired_path}") - - if desired_path_abs.exists() and desired_path_abs.is_dir(): - shutil.rmtree(desired_path_abs) - - shutil.move(f"{temp_dir.name}/{archive_path}", desired_path_abs) - - if cfg.ba2extract_zip_dirs and desired_path_abs.is_dir(): - shutil.make_archive(str(desired_path_abs), "zip", desired_path_abs) - - print(f">> Done extracting {target}.") - - print("> Done extracting Bethesda archives.\n") - - -def archive_esms_start(): - print("> Archiving ESMs in the background.") - cwd = os.getcwd() - os.chdir(cfg.dump_root) - - with open(os.devnull, "wb") as devnull: - archive_children["ESMs"] = subprocess.Popen([cfg.archiver_path, "a", "-mx9", "-mmt4", - f"SeventySix.esm.v{cfg.game_version}.7z", - f"{cfg.game_root}/Data/SeventySix.esm", - f"{cfg.game_root}/Data/NW.esm"], - stdout=devnull, stderr=subprocess.STDOUT) - - os.chdir(cwd) - print("") - - -def archive_xedit_start(): - """Archives all xEdit dumps that are larger than 10MB.""" - print(">> Archiving large xEdit dumps in the background.") - cwd = os.getcwd() - os.chdir(cfg.dump_root) - - with open(os.devnull, "wb") as devnull: - for dump in glob.glob(f"*.csv") + glob.glob(f"*.wiki") + glob.glob(f"*.db"): - csv_path = Path(dump) - - if csv_path.stat().st_size < 10000000: - continue - - print(f">> Starting archiving of '{csv_path.name}'.") - archive_children[f"'{csv_path.name}'"] = subprocess.Popen([cfg.archiver_path, "a", "-mx9", "-mmt4", - f"{csv_path.name}.7z", - csv_path.name], - stdout=devnull, stderr=subprocess.STDOUT) - - os.chdir(cwd) - - -def archive_join(): - print("> Waiting for background archiving processes.") - - for csv_path, child in archive_children.items(): - print(f">> Waiting for archiving of {csv_path}.") - child.wait() - - print("> Done waiting for background archiving processes.\n") - - -def main(): - """The main function.""" - print(f"Creating fo76-dumps using '{cfg.xedit_path}'.") - if cfg.game_version == "x.y.z.w": - if not prompt_confirmation("WARNING: The game version is set to 'x.y.z.w' in the configuration, which is " - "probably incorrect. If you continue, some dumps will have incorrect names. Do you " - "want to continue anyway? (y/n) "): - exit() - if not cfg.windows and ("INSERT NUMBER HERE" in cfg.xedit_compatdata_path or - "INSERT NUMBER HERE" in cfg.ba2extract_compatdata_path): - if not prompt_confirmation("WARNING: You did not adjust the compatdata path for xEdit or for ba2extract. This " - "might cause issues when launching xEdit or ba2extract. Check the dump scripts wiki " - "at https://github.com/FWDekker/fo76-dumps/wiki/Generating-dumps/ for more " - "information. Continue anyway? (y/n) "): - exit() - if Path(cfg.dump_root).exists() and len(os.listdir(cfg.dump_root)) != 0: - if prompt_confirmation("INFO: The dump output directory exists and is not empty. Do you want to remove the " - "directory and its contents? This is optional. (y/n) "): - shutil.rmtree(cfg.dump_root) - print("") - - # Create dumps output dir - Path(cfg.dump_root).mkdir(parents=True, exist_ok=True) - - # Archiving - if cfg.enable_archive_esms: - archive_esms_start() - - # xEdit - if cfg.enable_xedit: - xedit() - - # ba2extract - if cfg.enable_ba2extract: - ba2extract() - - # Archiving - archive_join() - - print("Done!") - - -if __name__ == "__main__": - archive_children = {} - cfg = load_config() - - main() diff --git a/fo76dumps.py b/fo76dumps.py new file mode 100644 index 0000000..214fb08 --- /dev/null +++ b/fo76dumps.py @@ -0,0 +1,152 @@ +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import Files +from IO import prompt_confirmation, run_executable +from State import cfg, subprocesses +from XEdit import xedit + + +def ba2extract() -> None: + """ + Creates raw dumps using ba2extract and archives directories. + + :return: `None` + """ + + print("> Extracting Bethesda archives.") + + for target, files in cfg.ba2extract_targets.items(): + print(f">> Extracting {target}.") + + with tempfile.TemporaryDirectory(prefix=f"fo76-dumps-{target}") as temp_dir: + temp_dir = Path(temp_dir) + + run_executable(args=[cfg.ba2extract_path, cfg.game_root / 'Data' / target, temp_dir], + compatdata_path=cfg.ba2extract_compatdata_path if not cfg.windows else "") + + for archive_path, desired_path in files.items(): + desired_path = cfg.dump_root / f"raw.{desired_path}" + + if desired_path.exists(): + Files.delete(desired_path) + + shutil.move(temp_dir / archive_path, desired_path) + + if cfg.ba2extract_zip_dirs and desired_path.is_dir(): + shutil.make_archive(str(desired_path), "zip", desired_path) + Files.move_into(desired_path, cfg.dump_archived) + + print(f">> Done extracting {target}.") + + print("> Done extracting Bethesda archives.\n") + + +def archive_esms_start() -> None: + """ + Archives ESMs in the background. + + :return: `None` + """ + + print("> Archiving ESMs in the background.") + subprocesses["ESMs"] = {"process": subprocess.Popen([cfg.archiver_path, "a", "-mx9", "-mmt4", + f"SeventySix.esm.v{cfg.game_version}.7z", + cfg.game_root / "Data/SeventySix.esm", + cfg.game_root / "Data/NW.esm"], + cwd=cfg.dump_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT), + "post": lambda *args: None} + print("") + + +def archive_join() -> None: + """ + Waits for background archiving processes. + + :return: `None` + """ + + print("> Waiting for background archiving processes.") + + for key, child in subprocesses.items(): + print(f">> Waiting for archiving of {key}.") + child["process"].wait() + child["post"]() + + print("> Done waiting for background archiving processes.\n") + + +def main() -> None: + """ + The main function. + + :return: `None` + """ + + if cfg.game_version == "x.y.z.w": + if not prompt_confirmation(f"WARNING: " + f"You did not adjust the game version in the configuration. " + f"The game version is currently set to '{cfg.game_version}'." + f"This may cause some dump files to have incorrect filenames. " + f"Check the dump scripts wiki at " + f"https://github.com/FWDekker/fo76-dumps/wiki/Generating-dumps/ for more " + f"information. " + f"Continue anyway? (y/n) "): + exit() + if not cfg.windows and "INSERT NUMBER HERE" in str(cfg.xedit_compatdata_path): + if not prompt_confirmation(f"WARNING: " + f"You did not adjust the compatdata path for xEdit in the configuration. " + f"The compatdata path is currently set to '{cfg.xedit_compatdata_path}'. " + f"This may cause issues when launching xEdit. " + f"Check the dump scripts wiki at " + f"https://github.com/FWDekker/fo76-dumps/wiki/Generating-dumps/ for more " + f"information. " + f"Continue anyway? (y/n) "): + exit() + if not cfg.windows and "INSERT NUMBER HERE" in str(cfg.ba2extract_compatdata_path): + if not prompt_confirmation(f"WARNING: " + f"You did not adjust the compatdata path for ba2extract in the configuration. " + f"The compatdata path is currently set to '{cfg.ba2extract_compatdata_path}'. " + f"This may cause issues when launching ba2extract. " + f"Check the dump scripts wiki at " + f"https://github.com/FWDekker/fo76-dumps/wiki/Generating-dumps/ for more " + f"information. " + f"Continue anyway? (y/n) "): + exit() + if cfg.dump_root.exists() and len(os.listdir(cfg.dump_root)) != 0: + if prompt_confirmation(f"INFO: " + f"The dump output directory '{cfg.dump_root}' exists and is not empty. " + f"It may be a good idea to delete this directory. " + f"Do you want to DELETE the directory and its contents? " + f"This is optional, the dump scripts will run after this either way. (y/n) "): + Files.delete(cfg.dump_root) + print("") + + # Create dumps output dir + cfg.dump_root.mkdir(exist_ok=True, parents=True) + + # Archiving + if cfg.enable_archive_esms: + archive_esms_start() + + # xEdit + if cfg.enable_xedit: + xedit() + + # ba2extract + if cfg.enable_ba2extract: + ba2extract() + + # Archiving + archive_join() + + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 0569a4b..8282ef2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pandas==1.2.4 +pandas==2.0.1