diff --git a/samples/vboxwrapper/Makefile b/samples/vboxwrapper/Makefile index 9275efc3f50..d859a06f149 100644 --- a/samples/vboxwrapper/Makefile +++ b/samples/vboxwrapper/Makefile @@ -77,4 +77,4 @@ vboxwrapper.o: vboxwrapper.cpp $(HEADERS) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c vboxwrapper.cpp vboxwrapper$(VBOXWRAPPER_RELEASE_SUFFIX): vboxwrapper.o vbox_common.o vbox_vboxmanage.o vboxcheckpoint.o vboxjob.o vboxlogging.o floppyio.o $(MAKEFILE_STDLIB) $(BOINC_LIB_DIR)/libboinc.a $(BOINC_API_DIR)/libboinc_api.a - $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) -o vboxwrapper$(VBOXWRAPPER_RELEASE_SUFFIX) vboxwrapper.o vbox_common.o vbox_vboxmanage.o vboxcheckpoint.o vboxjob.o vboxlogging.o floppyio.o $(MAKEFILE_LDFLAGS) -lboinc_api -lboinc $(STDCPPTC) + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) -o vboxwrapper$(VBOXWRAPPER_RELEASE_SUFFIX) vboxwrapper.o vbox_common.o vbox_vboxmanage.o vboxcheckpoint.o vboxjob.o vboxlogging.o floppyio.o $(MAKEFILE_LDFLAGS) -lboinc_api -lboinc -lrt $(STDCPPTC) diff --git a/samples/vboxwrapper/vbox_common.cpp b/samples/vboxwrapper/vbox_common.cpp index e63e910271d..fd048df7fa6 100644 --- a/samples/vboxwrapper/vbox_common.cpp +++ b/samples/vboxwrapper/vbox_common.cpp @@ -1171,7 +1171,10 @@ int VBOX_BASE::vbm_popen(string& command, string& output, const char* item, bool } // Timeout? - if (retry_count >= 5) break; + if (retry_count >= 5) { + retval = ERR_TIMEOUT; + break; + } retry_count++; boinc_sleep(sleep_interval); diff --git a/samples/vboxwrapper/vbox_vboxmanage.cpp b/samples/vboxwrapper/vbox_vboxmanage.cpp index 602db049d41..548c794cf87 100644 --- a/samples/vboxwrapper/vbox_vboxmanage.cpp +++ b/samples/vboxwrapper/vbox_vboxmanage.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include #endif using std::string; @@ -48,6 +50,7 @@ using std::string; #include "util.h" #include "error_numbers.h" #include "procinfo.h" +#include "md5_file.h" #include "network.h" #include "boinc_api.h" #include "floppyio.h" @@ -176,6 +179,7 @@ int VBOX_VM::create_vm() { bool disable_acceleration = false; char buf[256]; int retval; + int save_retval; vboxlog_msg("Create VM. (%s, slot#%d)", vm_master_name.c_str(), aid.slot); @@ -554,9 +558,22 @@ int VBOX_VM::create_vm() { vboxlog_msg("Adding virtual disk drive to VM. (%s)", multiattach_vdi_file.c_str()); +#ifdef _WIN32 + HANDLE fd_race_mitigator = NULL; +#else + int fd_race_mitigator = 0; +#endif int retry_count = 0; bool log_error = false; bool vbox_bug_mitigation = false; + string lock_name = ""; + + retval = set_race_mitigation_lock(fd_race_mitigator, lock_name, medium_file); + if (retval) { + save_retval = retval; + vboxlog_msg("Could not set race mitigation lock in 'create_vm'."); + return save_retval; + } do { string set_new_uuid = ""; @@ -568,6 +585,7 @@ int VBOX_VM::create_vm() { retval = vbm_popen(command, output, "check if parent hdd is registered", false); if (retval) { + save_retval = retval; // showhdinfo implicitly registers unregistered hdds. // Hence, this has to be handled first. // @@ -584,11 +602,12 @@ int VBOX_VM::create_vm() { ); } else { // other errors + remove_race_mitigation_lock(fd_race_mitigator, lock_name); vboxlog_msg("Error in check if parent hdd is registered.\nCommand:\n%s\nOutput:\n%s", command.c_str(), output.c_str() ); - return retval; + return save_retval; } } @@ -621,13 +640,21 @@ int VBOX_VM::create_vm() { command += set_new_uuid + "--medium \"" + medium_file + "\" "; retval = vbm_popen(command, output, "register parent vdi"); - if (retval) return retval; + if (retval) { + save_retval = retval; + remove_race_mitigation_lock(fd_race_mitigator, lock_name); + return save_retval; + } command = command_fix_part; command += "--medium none "; retval = vbm_popen(command, output, "detach parent vdi"); - if (retval) return retval; + if (retval) { + save_retval = retval; + remove_race_mitigation_lock(fd_race_mitigator, lock_name); + return save_retval; + } // the vdi file is now registered and ready // to be attached in multiattach mode } @@ -639,6 +666,7 @@ int VBOX_VM::create_vm() { retval = vbm_popen(command, output, "storage attach (fixed disk - multiattach mode)", log_error); if (retval) { + save_retval = retval; // VirtualBox occasionally writes the 'MultiAttach' // attribute to the disk entry in VirtualBox.xml // although this is not allowed there. @@ -655,29 +683,32 @@ int VBOX_VM::create_vm() { (output.find("MultiAttach") != string::npos) && (output.find("can only be attached to machines that were created with VirtualBox 4.0 or later") != string::npos)) { // try to deregister the medium from the global media store - command = "closemedium \"" + medium_file + "\" "; - - retval = vbm_popen(command, output, "deregister parent vdi"); - if (retval) return retval; + // + retval = remove_vbox_disk_orphans(medium_file.c_str()); + if (retval) { + save_retval = retval; + remove_race_mitigation_lock(fd_race_mitigator, lock_name); + return save_retval; + } retry_count++; log_error = true; - boinc_sleep(1.0); break; } if (retry_count >= 1) { // in case of other errors or if retry also failed + // + remove_race_mitigation_lock(fd_race_mitigator, lock_name); vboxlog_msg("Error in storage attach (fixed disk - multiattach mode).\nCommand:\n%s\nOutput:\n%s", command.c_str(), output.c_str() ); - return retval; + return save_retval; } retry_count++; log_error = true; - boinc_sleep(1.0); } else { vbox_bug_mitigation = true; @@ -687,6 +718,7 @@ int VBOX_VM::create_vm() { while (true); } while (!vbox_bug_mitigation); + remove_race_mitigation_lock(fd_race_mitigator, lock_name); } @@ -862,8 +894,15 @@ int VBOX_VM::register_vm() { } int VBOX_VM::deregister_vm(bool delete_media) { + int retval; string command; string output; + string lock_name = ""; +#ifdef _WIN32 + HANDLE fd_race_mitigator = NULL; +#else + int fd_race_mitigator = 0; +#endif vboxlog_msg("Deregistering VM. (%s, slot#%d)", vm_name.c_str(), aid.slot); @@ -890,12 +929,25 @@ int VBOX_VM::deregister_vm(bool delete_media) { } // Next, delete VM + // This automatically deletes child disk images connected to the VM. // + if (multiattach_vdi_file.size()) { + string medium_file = aid.project_dir; + medium_file += "/" + multiattach_vdi_file; + + retval = set_race_mitigation_lock(fd_race_mitigator, lock_name, medium_file); + if (retval) { + vboxlog_msg("Could not set race mitigation lock in 'deregister_vm'."); + vboxlog_msg("Warning: Will continue without a lock."); + } + } + vboxlog_msg("Removing VM from VirtualBox."); command = "unregistervm \"" + vm_name + "\" "; command += "--delete "; vbm_popen(command, output, "delete VM", false, false); + remove_race_mitigation_lock(fd_race_mitigator, lock_name); // Lastly delete medium(s) from Virtual Box Media Registry // @@ -1712,14 +1764,16 @@ bool VBOX_VM::is_disk_image_registered() { string command; string output; - command = "showhdinfo \"" + slot_dir_path + "/" + image_filename + "\" "; - if (vbm_popen(command, output, "hdd registration", false, false) == 0) { - if ((output.find("VBOX_E_FILE_ERROR") == string::npos) - && (output.find("VBOX_E_OBJECT_NOT_FOUND") == string::npos) - && (output.find("does not match the value") == string::npos) - ) { - // Error message not found in text - return true; + if (!multiattach_vdi_file.size()) { + command = "showhdinfo \"" + slot_dir_path + "/" + image_filename + "\" "; + if (vbm_popen(command, output, "hdd registration", false, false) == 0) { + if ((output.find("VBOX_E_FILE_ERROR") == string::npos) + && (output.find("VBOX_E_OBJECT_NOT_FOUND") == string::npos) + && (output.find("does not match the value") == string::npos) + ) { + // Error message not found in text + return true; + } } } @@ -1846,6 +1900,7 @@ int VBOX_VM::get_version_information(string& version_raw, string& version_displa ); version_display = buf; } else { + vboxlog_msg("VBoxManage version raw: %s", output.c_str()); version_raw = "Unknown"; version_display = "VirtualBox VboxManage Interface (Version: Unknown)"; } @@ -2236,3 +2291,268 @@ bool VBOX_VM::is_hostrtc_set_to_utc() { return true; #endif } + +#ifdef _WIN32 +void VBOX_VM::remove_race_mitigation_lock(HANDLE& fd_race_mitigator, string& lock_name) { + DWORD err = BOINC_SUCCESS; + bool retval; + + if (fd_race_mitigator) { + retval = CloseHandle(fd_race_mitigator); + err = GetLastError(); + if (!retval) { + vboxlog_msg("Could not remove race mitigation lock."); + vboxlog_msg("Lockname: %s", lock_name.c_str()); + vboxlog_msg("Error: %d, %s", err, strerror(err)); + } + } +} +#else +void VBOX_VM::remove_race_mitigation_lock(int& fd_race_mitigator, string& lock_name) { + int err = BOINC_SUCCESS; + int retval; + + if (fd_race_mitigator > 0) { + retval = shm_unlink(lock_name.c_str()); + err = errno; + if (retval) { + vboxlog_msg("Could not remove race mitigation lock."); + vboxlog_msg("Lockname: %s", lock_name.c_str()); + vboxlog_msg("Error: %d, %s", err, strerror(err)); + } + } +} +#endif + +#ifdef _WIN32 +int VBOX_VM::set_race_mitigation_lock(HANDLE& fd_race_mitigator, string& lock_name, const string& medium_file) { +#else +int VBOX_VM::set_race_mitigation_lock(int& fd_race_mitigator, string& lock_name, const string& medium_file) { +#endif + int attempts = 1; + double timeout = 0.0; + double sleep_low = 0.7; + double sleep_high = 2.4; + + // The lock ensures that only 1 vboxwrapper instance can + // modify a given virtual disk entry at a given time. + // This is a must for multiattach disks since some modifications + // need more than 1 call to VBoxManage. + // + // lock_name should be derived from the full path of the disk's filename. + // + lock_name = "boinc_vboxwrapper_lock_"; + lock_name += md5_string(medium_file).substr(0, 16); + + // Tests with Linux on a 16c/32t computer + // typically register 30 VMs in less than 8 s. + // + timeout = dtime() + 90; + +#ifdef _WIN32 + DWORD err = BOINC_SUCCESS; + + while (1) { + // Parameter #5 (size) must not be 0 if INVALID_HANDLE_VALUE is used. + // + fd_race_mitigator = CreateFileMapping( + INVALID_HANDLE_VALUE, + NULL, + PAGE_READONLY, + NULL, + 1, + (LPTSTR)lock_name.c_str() + ); + err = GetLastError(); + + if (!err) { + // Successfully set a fresh lock. + // No error implies we also have a valid handle. + // + if (attempts > 1) { + vboxlog_msg("Attempts: %d", attempts); + } + break; + } else { + // If we got a handle, it must not be used. + // Close it immediately. + // + if (fd_race_mitigator) { + CloseHandle(fd_race_mitigator); + fd_race_mitigator = NULL; + } + + if (err == ERROR_ALREADY_EXISTS) { + // a lock exists, most likely set by another vboxwrapper + // + if (dtime() >= timeout) { + // Either the lock is stale + // or far too many VMs are starting concurrently. + // + vboxlog_msg("Could not set race mitigation lock."); + vboxlog_msg("Lockname: '%s'", lock_name.c_str()); + vboxlog_msg("Error: ERR_TIMEOUT"); + vboxlog_msg("Attempts: %d", attempts); + + return ERR_TIMEOUT; + } + boinc_sleep(sleep_low + sleep_high * drand()); + attempts++; + } else { + vboxlog_msg("Could not set race mitigation lock."); + vboxlog_msg("Lockname: '%s'", lock_name.c_str()); + vboxlog_msg("Error: %d, %s", err, strerror(err)); + vboxlog_msg("Attempts: %d", attempts); + + return ERR_FOPEN; + } + } + } +#else + int err = BOINC_SUCCESS; + lock_name = "/" + lock_name; + + while (1) { + fd_race_mitigator = shm_open(lock_name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600); + err = errno; + + if (fd_race_mitigator > 0) { + // Successfully set a fresh lock. + // + if (attempts > 1) { + vboxlog_msg("Attempts: %d", attempts); + } + break; + } else { + if ((fd_race_mitigator == -1) + && (err == EEXIST)) { + // a lock exists, most likely set by another vboxwrapper + // + if (dtime() >= timeout) { + // Either the lock is stale + // or far too many VMs are starting concurrently. + // + vboxlog_msg("Could not set race mitigation lock."); + vboxlog_msg("Lockname: '%s'", lock_name.c_str()); + vboxlog_msg("Error: ERR_TIMEOUT"); + vboxlog_msg("Attempts: %d", attempts); + + fd_race_mitigator = 0; + return ERR_TIMEOUT; + } + boinc_sleep(sleep_low + sleep_high * drand()); + attempts++; + } else { + vboxlog_msg("Could not set race mitigation lock."); + vboxlog_msg("Lockname: '%s'", lock_name.c_str()); + vboxlog_msg("Error: %d, %s", err, strerror(err)); + vboxlog_msg("Attempts: %d", attempts); + + fd_race_mitigator = 0; + return ERR_FOPEN; + } + } + } +#endif + return BOINC_SUCCESS; +} + +// To remove the parent disk entry use 'closemedium disk ""' instead of 'closemedium disk ""' +// since the latter sometimes returns an error like this: +// +// VBoxManage closemedium disk "1d9935fd-37c9-4c34-946b-f6d252c6a1af" +// VBoxManage: error: The given path '1d9935fd-37c9-4c34-946b-f6d252c6a1af' is not fully qualified +// VBoxManage: error: Details: code VBOX_E_FILE_ERROR (0x80bb0004), component MediumWrap, interface IMedium, callee nsISupports +// VBoxManage: error: Context: "OpenMedium(Bstr(pszFilenameOrUuid).raw(), enmDevType, enmAccessMode, fForceNewUuidOnOpen, pMedium.asOutParam())" at line 197 of file VBoxManageDisk.cpp +// +// Output of 'VBoxManage showhdinfo "/path to/disk/parent_disk_name.vdi"' usually looks like this: +// +// UUID: f81c0950-57ee-462e-b931-051193700d76 +// Parent UUID: base +// State: created +// Type: multiattach +// Location: /path to/disk/parent_disk_name.vdi +// Storage format: VDI +// Format variant: dynamic default +// Capacity: 51200 MBytes +// Size on disk: 2 MBytes +// Encryption: disabled +// Property: AllocationBlockSize=1048576 +// Child UUIDs: 67e34269-e52e-4957-a813-c85f72084fba +// 5fa7905e-72e4-4f85-8405-dc6401417720 +// 7262b5e5-2b32-482a-8c6e-2ef49548740b +// +int VBOX_VM::remove_vbox_disk_orphans(string vbox_disk) { + int retval; + string command; + string output; + string needle; + string childlist; + string child_uuid; + size_t childlist_start; + string loc_line; + size_t loc_start; + size_t loc_end; + + command = "showhdinfo \"" + vbox_disk + "\" "; + retval = vbm_popen(command, output, "hdd registration", false, false); + if (retval) { + vboxlog_msg("Could not get disk details in 'remove_vbox_disk_orphans'."); + return retval; + } + + needle = "\nChild UUIDs:"; + childlist_start = output.find(needle.c_str()) + needle.size(); + + if (childlist_start != string::npos) { + size_t pos = 0; + size_t uuid_length = 36; + + childlist = output.substr(childlist_start, string::npos - childlist_start); + + needle = " "; + pos = childlist.find(needle); + while (pos != string::npos) { + childlist.replace(pos, 1, ""); + pos = childlist.find(needle, pos); + } + + needle = "\n"; + pos = childlist.find(needle); + while (pos != string::npos) { + childlist.replace(pos, 1, ""); + pos = childlist.find(needle, pos); + } + + pos = 0; + while (pos < childlist.size()) { + child_uuid = childlist.substr(pos, uuid_length); + // recursively process child disks + // + retval = remove_vbox_disk_orphans(child_uuid); + if (retval) { + vboxlog_msg("Could not remove child disk '%s'.", child_uuid.c_str()); + return retval; + } + pos += uuid_length; + } + } + + // if no child disks are left, remove the disk itself + // + needle = "\nLocation:"; + if ((loc_start = output.find(needle.c_str())) != string::npos) { + loc_start += needle.size(); + loc_start = output.find_first_not_of(" ", loc_start); + loc_end = output.find("\n", loc_start) - loc_start; + loc_line = output.substr(loc_start, loc_end); + + command = "closemedium disk \"" + loc_line + "\" "; + retval = vbm_popen(command, output, "remove virtual disk", false, false); + if (retval) { + vboxlog_msg("Could not remove parent disk '%s'.", loc_line.c_str()); + return retval; + } + } + return BOINC_SUCCESS; +} diff --git a/samples/vboxwrapper/vbox_vboxmanage.h b/samples/vboxwrapper/vbox_vboxmanage.h index 855db3c92ef..81106f5d6c0 100644 --- a/samples/vboxwrapper/vbox_vboxmanage.h +++ b/samples/vboxwrapper/vbox_vboxmanage.h @@ -84,6 +84,14 @@ struct VBOX_VM : VBOX_BASE { double bytes_sent, double bytes_received ); +#ifdef _WIN32 + int set_race_mitigation_lock(HANDLE& fd_race_mitigator, string& lock_name, const string& medium_file); + void remove_race_mitigation_lock(HANDLE& fd_race_mitigator, string& lock_name); +#else + int set_race_mitigation_lock(int& fd_race_mitigator, string& lock_name, const string& medium_file); + void remove_race_mitigation_lock(int& fd_race_mitigator, string& lock_name); +#endif + int remove_vbox_disk_orphans(string vbox_disk); }; #endif diff --git a/samples/vboxwrapper/vboxwrapper.cpp b/samples/vboxwrapper/vboxwrapper.cpp index ff55681c02c..ddd38b715e1 100644 --- a/samples/vboxwrapper/vboxwrapper.cpp +++ b/samples/vboxwrapper/vboxwrapper.cpp @@ -440,6 +440,14 @@ int main(int argc, char** argv) { bool is_sporadic = false; bool register_only = false; + // seed random numbers + // +#ifdef _WIN32 + srand((unsigned int)GetCurrentProcessId()); +#else + srand((unsigned int)getpid()); +#endif + // Initialize diagnostics system // boinc_init_diagnostics(BOINC_DIAG_DEFAULTS); @@ -796,7 +804,6 @@ int main(int argc, char** argv) { // to stagger disk I/O. // if (!pVM->disable_automatic_checkpoints) { - srand((int)getpid()); random_checkpoint_factor = drand() * 600; vboxlog_msg(