diff --git a/src/gmt_init.c b/src/gmt_init.c index 53a4dc988de..4f5f4070a05 100644 --- a/src/gmt_init.c +++ b/src/gmt_init.c @@ -425,7 +425,7 @@ static struct GMT_HASH keys_hashnode[GMT_N_KEYS]; #include /*! . */ -GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) { OVERLAPPED over = { 0 }; HANDLE hand = (HANDLE)_get_osfhandle(fd); if (!LockFileEx(hand, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &over)) /* Will block until exclusive lock is acquired */ @@ -437,7 +437,7 @@ GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) { } /*! . */ -GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) { HANDLE hand = (HANDLE)_get_osfhandle(fd); if (!UnlockFile(hand, 0, 0, 0, 1)) { @@ -449,7 +449,7 @@ GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) { #elif defined (HAVE_FCNTL_H_) /* Use POSIX fcntl */ /*! . */ -GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) { int status; struct flock lock; lock.l_type = F_WRLCK; /* Lock for exclusive reading/writing */ @@ -466,7 +466,7 @@ GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) { } /*! . */ -GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) { int status; struct flock lock; lock.l_type = F_UNLCK; /* Release lock and close file */ @@ -484,13 +484,13 @@ GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) { #else /* Not Windows and fcntl not available */ /*! . */ -GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) { GMT_Report (GMT->parent, GMT_MSG_DEBUG, "File locking not supported.\n"); return false; } /*! . */ -GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) { +bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) { return false; } #endif @@ -2871,11 +2871,11 @@ GMT_LOCAL int gmtinit_get_history (struct GMT_CTRL *GMT) { return (GMT_NOERROR); /* OK to be unsuccessful in opening this file */ if (gmt_hash_init (GMT, unique_hashnode, GMT_unique_option, GMT_N_UNIQUE, GMT_N_UNIQUE)) { - return (GMT_NOERROR); /* Cannot do anything without the hash */ + return (GMT_NOERROR); /* Cannot do anything without the hash */ } /* When we get here the file exists */ - gmtinit_file_lock (GMT, fileno(fp)); + gmtlib_file_lock (GMT, fileno(fp)); /* Format of GMT gmt.history is as follow: * BEGIN GMT This is the start of parsable section * OPT ARG @@ -2914,7 +2914,7 @@ GMT_LOCAL int gmtinit_get_history (struct GMT_CTRL *GMT) { } /* Close the file */ - gmtinit_file_unlock (GMT, fileno(fp)); + gmtlib_file_unlock (GMT, fileno(fp)); fclose (fp); GMT_Report (GMT->parent, GMT_MSG_DEBUG, "Exit: gmtinit_get_history\n"); @@ -2967,7 +2967,7 @@ GMT_LOCAL int gmtinit_put_history (struct GMT_CTRL *GMT) { if ((fp = fopen (hfile, "w")) == NULL) return (-1); /* Not OK to be unsuccessful in creating this file */ /* When we get here the file is open */ - if (!gmtinit_file_lock (GMT, fileno(fp))) + if (!gmtlib_file_lock (GMT, fileno(fp))) GMT_Report (GMT->parent, GMT_MSG_WARNING, "Directory %s is not locked for exclusive access. Multiple gmt processes running at once could corrupt history file.\n", hfile); fprintf (fp, "# GMT %d Session common arguments shelf\n", GMT_MAJOR_VERSION); @@ -2983,7 +2983,7 @@ GMT_LOCAL int gmtinit_put_history (struct GMT_CTRL *GMT) { fprintf (fp, "END\n"); /* Close the file */ - gmtinit_file_unlock (GMT, fileno(fp)); + gmtlib_file_unlock (GMT, fileno(fp)); fclose (fp); return (GMT_NOERROR); @@ -14338,7 +14338,7 @@ void gmt_end_module (struct GMT_CTRL *GMT, struct GMT_CTRL *Ccopy) { gmt_M_memset (&GMT->current.gdal_read_out, 1, struct GMT_GDALREAD_OUT_CTRL); gmt_M_memset (&GMT->current.gdal_write, 1, struct GMT_GDALWRITE_CTRL); #endif - + GMT->parent->cache = false; /* Otherwise gdalread from externals on Windows would mingle CACHEDIR in fnames */ gmt_M_str_free (Ccopy); /* Good riddance */ @@ -17337,15 +17337,15 @@ void gmt_add_legend_item (struct GMTAPI_CTRL *API, struct GMT_SYMBOL *S, bool do fprintf (fp, "# LEGEND_JUSTIFICATION: %s\n", justcode); fprintf (fp, "# LEGEND_SCALING: %g\n", item->scale); fprintf (fp, "# LEGEND_FRAME: "); - if (item->pen[GMT_LEGEND_PEN_P][0]) + if (item->pen[GMT_LEGEND_PEN_P][0]) fprintf (fp, " %s", item->pen[GMT_LEGEND_PEN_P]); else fprintf (fp, " 1p"); - if (item->fill[0]) + if (item->fill[0]) fprintf (fp, " %s", item->fill); else fprintf (fp, " white"); - if (item->off[0]) + if (item->off[0]) fprintf (fp, " %s\n", item->off); else fprintf (fp, " 0.2c\n"); diff --git a/src/gmt_internals.h b/src/gmt_internals.h index 187a819ea42..8862fb9828e 100644 --- a/src/gmt_internals.h +++ b/src/gmt_internals.h @@ -53,6 +53,8 @@ EXTERN_MSC char *dlerror (void); #endif EXTERN_MSC int gmtlib_delete_virtualfile (void *API, const char *string); +EXTERN_MSC bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd); +EXTERN_MSC bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd); EXTERN_MSC int gmtlib_file_is_jpeg2000_tile (struct GMTAPI_CTRL *API, char *file); EXTERN_MSC int gmtlib_download_remote_file (struct GMT_CTRL *GMT, const char* file_name, char *path, int k_data, unsigned int mode); EXTERN_MSC int gmtlib_get_serverfile_index (struct GMTAPI_CTRL *API, const char *file); diff --git a/src/gmt_remote.c b/src/gmt_remote.c index 069da852d88..88b5e299a30 100644 --- a/src/gmt_remote.c +++ b/src/gmt_remote.c @@ -197,12 +197,16 @@ GMT_LOCAL struct GMT_DATA_INFO *gmtremote_data_load (struct GMTAPI_CTRL *API, in if (fgets (line, GMT_LEN256, fp) == NULL) { /* Try to get first record */ fclose (fp); GMT_Report (API, GMT_MSG_ERROR, "Read error first record in file %s\n", file); + GMT_Report (API, GMT_MSG_ERROR, "Deleting %s so it can get regenerated - please try again\n", file); + gmt_remove_file (GMT, file); return NULL; } *n = atoi (line); /* Number of non-commented records to follow */ if (*n <= 0 || *n > GMT_BIG_CHUNK) { /* Probably not a good value */ fclose (fp); GMT_Report (API, GMT_MSG_ERROR, "Bad record counter in file %s\n", file); + GMT_Report (API, GMT_MSG_ERROR, "Deleting %s so it can get regenerated - please try again\n", file); + gmt_remove_file (GMT, file); return NULL; } if (fgets (line, GMT_LEN256, fp) == NULL) { /* Try to get second record */ @@ -453,7 +457,20 @@ GMT_LOCAL int gmtremote_find_and_give_data_attribution (struct GMTAPI_CTRL *API, return (match); } -GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char* URL, size_t limit) { +GMT_LOCAL char *gmtremote_lockfile (struct GMTAPI_CTRL *API, char *file) { + /* Create a dummy file in temp with extension .download and use as a lock file */ + char *c = strrchr (file, '/'); + char Lfile[PATH_MAX] = {""}; + if (c) /* Found the last slash, skip it */ + c++; + else /* No path, just point to file */ + c = file; + if (c[0] == '@') c++; /* Skip any leading @ sign */ + sprintf (Lfile, "%s/%s.download", API->tmp_dir, c); + return (strdup (Lfile)); +} + +GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char * URL, size_t limit) { /* Get the remote file's size and if too large we refuse to download */ CURL *curl = NULL; CURLcode res; @@ -503,12 +520,25 @@ GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char* URL, si #define GMT_HASH_TIME_OUT 10L /* Not waiting longer than this to time out on getting the hash file */ GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, char *orig, unsigned int index) { + bool query = gmt_M_file_is_query (url); int curl_err = 0; long time_spent; + char *Lfile = NULL; + FILE *fp = NULL; CURL *Curl = NULL; struct FtpFile urlfile = {NULL, NULL}; + struct GMTAPI_CTRL *API = GMT->parent; time_t begin, end; + if (!query) { /* Only make a filename if not a query */ + Lfile = gmtremote_lockfile (API, file); + if ((fp = fopen (Lfile, "w")) == NULL) { + GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile); + return 1; + } + gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */ + } + if ((Curl = curl_easy_init ()) == NULL) { GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to initiate curl - cannot obtain %s\n", url); return 1; @@ -573,6 +603,14 @@ GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, ch curl_easy_cleanup (Curl); if (urlfile.fp) /* close the local file */ fclose (urlfile.fp); + + if (!query) { /* Remove lock file after successful download */ + gmtlib_file_unlock (GMT, fileno(fp)); + fclose(fp); + gmt_remove_file (GMT, Lfile); + gmt_M_str_free (Lfile); + } + gmtremote_turn_off_ctrl_C_check (); return 0; } @@ -1073,9 +1111,12 @@ int gmtlib_file_is_jpeg2000_tile (struct GMTAPI_CTRL *API, char *file) { } int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *localfile, bool be_fussy) { + bool query = gmt_M_file_is_query (url); int curl_err, error; size_t fsize; + char *Lfile = NULL; CURL *Curl = NULL; + FILE *fp = NULL; struct FtpFile urlfile = {NULL, NULL}; struct GMTAPI_CTRL *API = GMT->parent; @@ -1096,6 +1137,15 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char * /* Here we will try to download a file */ + if (!query) { /* Only make a filename if not a query */ + Lfile = gmtremote_lockfile (API, (char *)name); + if ((fp = fopen (Lfile, "w")) == NULL) { + GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile); + return 1; + } + gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */ + } + if ((Curl = curl_easy_init ()) == NULL) { GMT_Report (API, GMT_MSG_ERROR, "Failed to initiate curl\n"); return 1; @@ -1149,6 +1199,14 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char * curl_easy_cleanup (Curl); if (urlfile.fp) /* close the local file */ fclose (urlfile.fp); + + if (!query) { /* Remove lock file after successful download */ + gmtlib_file_unlock (GMT, fileno(fp)); + fclose(fp); + gmt_remove_file (GMT, Lfile); + gmt_M_str_free (Lfile); + } + gmtremote_turn_off_ctrl_C_check (); error = gmtremote_convert_jp2_to_nc (API, localfile);