Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Directory listing performance improvement while file transfers are going on #41

Merged
merged 1 commit into from
Aug 31, 2019
Merged

Directory listing performance improvement while file transfers are going on #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,10 @@ OpenSSL 1.1 do not.

### Debugging Mutexes
By default the debugging output associated with mutexes are not compiled. To
enable them, compile the program using the following command:
enable them, compile the program with the ``-DCACHE_LOCK_DEBUG``, the
``-DNETWORK_LOCK_DEBUG`` and/or the ``-DLINK_LOCK_DEBUG`` CPPFLAGS, e.g.

make CPPFLAGS=-DLOCK_DEBUG
make CPPFLAGS=-DCACHE_LOCK_DEBUG

## The Technical Details
This program downloads the HTML web pages/files using
Expand Down
20 changes: 11 additions & 9 deletions src/cache.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,7 +755,7 @@ cf->content_length: %ld, Data_size(fn): %ld.\n", fn, cf->content_length,

void Cache_close(Cache *cf)
{
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Must wait for the background download thread to stop */
fprintf(stderr, "Cache_close(): locking bgt_lock;\n");
pthread_mutex_lock(&cf->bgt_lock);
Expand Down Expand Up @@ -815,7 +815,7 @@ static void Seg_set(Cache *cf, off_t offset, int i)
static void *Cache_bgdl(void *arg)
{
Cache *cf = (Cache *) arg;
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: locking rw_lock;\n",
pthread_self());
#endif
Expand All @@ -834,12 +834,12 @@ static void *Cache_bgdl(void *arg)
"Cache_bgdl(): received %ld, possible network error.\n", recv);
}
free(recv_buf);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: unlocking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
Expand Down Expand Up @@ -874,18 +874,18 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
goto bgdl;
} else {

#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Wait for the background download thread to finish */
fprintf(stderr, "Cache_read(): thread %lu: locking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Wait for any other download thread to finish*/
fprintf(stderr, "Cache_read(): thread %lu: locking rw_lock;\n",
pthread_self());
Expand All @@ -895,7 +895,7 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
/* The segment already exists - it was downloaded by other
* download thread. Send it off and unlock the I/O */
send = Data_read(cf, (uint8_t *) output_buf, len, offset);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
Expand Down Expand Up @@ -931,7 +931,7 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
"Cache_read(): received %ld, possible network error.\n", recv);
}
free(recv_buf);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
Expand All @@ -945,8 +945,10 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
cf->next_offset < cf->content_length ){
/* Stop the spawning of multiple background pthreads */
if(!pthread_mutex_trylock(&cf->bgt_lock)) {
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: trylocked bgt_lock;\n",
pthread_self());
#endif
if (pthread_create(&cf->bgt, NULL, Cache_bgdl, cf)) {
fprintf(stderr,
"Cache_read(): Error creating background download thread\n"
Expand Down
41 changes: 41 additions & 0 deletions src/link.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,23 @@
LinkTable *ROOT_LINK_TBL = NULL;
int ROOT_LINK_OFFSET = 0;

/* ----------------- Static variable ----------------------- */
/**
* \brief LinkTable generation priority lock
* \details This allows LinkTable generation to be run exclusively. This
* effectively gives LinkTable generation priority over file transfer.
*/
static pthread_mutex_t link_lock;

void link_system_init()
{
if (pthread_mutex_init(&link_lock, NULL) != 0) {
fprintf(stderr,
"link_system_init(): link_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
}

static void LinkTable_add(LinkTable *linktbl, Link *link)
{
linktbl->num++;
Expand Down Expand Up @@ -266,6 +283,12 @@ static void LinkTable_print(LinkTable *linktbl)

LinkTable *LinkTable_new(const char *url)
{
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"LinkTable_new(): thread %lu: locking link_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&link_lock);
LinkTable *linktbl = calloc(1, sizeof(LinkTable));
if (!linktbl) {
fprintf(stderr, "LinkTable_new(): calloc failure!\n");
Expand Down Expand Up @@ -349,6 +372,12 @@ HTTP %ld\n", url, http_resp);
curl_easy_cleanup(c);

LinkTable_print(linktbl);
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"LinkTable_new(): thread %lu: unlocking link_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&link_lock);
return linktbl;
}

Expand Down Expand Up @@ -553,6 +582,18 @@ long path_download(const char *path, char *output_buf, size_t size,
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);

#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"path_download(): thread %lu: locking link_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&link_lock);
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"path_download(): thread %lu: unlocking link_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&link_lock);
transfer_blocking(curl);

long http_resp;
Expand Down
5 changes: 5 additions & 0 deletions src/link.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ extern LinkTable *ROOT_LINK_TBL;
*/
extern int ROOT_LINK_OFFSET;

/**
* \brief initialise link sub-system.
*/
void link_system_init();

/**
* \brief
*/
Expand Down
3 changes: 3 additions & 0 deletions src/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ int main(int argc, char **argv)
/*--- FUSE expects the first initialisation to be the program's name ---*/
add_arg(&fuse_argv, &fuse_argc, argv[0]);

/* initialise link subsystem */
link_system_init();

/* initialise network configuration struct */
network_config_init();

Expand Down
34 changes: 31 additions & 3 deletions src/network.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl,
*/
int curl_multi_perform_once()
{
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"curl_multi_perform_once(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
/* Get curl multi interface to perform pending tasks */
int n_running_curl;
Expand Down Expand Up @@ -217,6 +222,11 @@ int curl_multi_perform_once()
while((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
}
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"curl_multi_perform_once(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);
return n_running_curl;
}
Expand Down Expand Up @@ -254,8 +264,7 @@ LinkTable *network_init(const char *url)
curl_share_setopt(CURL_SHARE, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);

if (pthread_mutex_init(&curl_lock, NULL) != 0) {
printf(
"network_init(): curl_lock initialisation failed!\n");
fprintf(stderr, "network_init(): curl_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
curl_share_setopt(CURL_SHARE, CURLSHOPT_LOCKFUNC, curl_callback_lock);
Expand Down Expand Up @@ -327,9 +336,18 @@ void transfer_blocking(CURL *curl)
transfer.type = DATA;
transfer.transferring = 1;
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);

#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_blocking(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_blocking(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);

if(res > 0) {
Expand All @@ -345,8 +363,18 @@ void transfer_blocking(CURL *curl)

void transfer_nonblocking(CURL *curl)
{
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_nonblocking(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_nonblocking(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);

if(res > 0) {
Expand Down