Merge pull request #9 from qingshui/paddlebox

add cgpu and file parser block bug fix
jack603047588 · Jul 2, 2021 · e49656a · e49656a
2 parents 30b445d + f7bd5f0
commit e49656a
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 16 deletions.
diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc
@@ -2815,19 +2815,17 @@ void SlotPaddleBoxDataFeed::LoadIntoMemoryByFile(void) {
     timeline.Start();
 
     int lines = 0;
-    bool is_error = false;
+    bool is_ok = true;
     do {
       if (BoxWrapper::GetInstance()->UseAfsApi() && pipe_command_.empty()) {
         while (reader->open(filename) < 0) {
           sleep(1);
         }
-        if (!parser->ParseFileInstance(
-                [this, reader](char* buf, int len) {
-                  return reader->read(buf, len);
-                },
-                pull_record_func, lines)) {
-          is_error = true;
-        }
+        is_ok = parser->ParseFileInstance(
+            [this, reader](char* buf, int len) {
+              return reader->read(buf, len);
+            },
+            pull_record_func, lines);
         reader->close();
       } else {
         if (BoxWrapper::GetInstance()->UseAfsApi()) {
@@ -2839,15 +2837,17 @@ void SlotPaddleBoxDataFeed::LoadIntoMemoryByFile(void) {
         }
         CHECK(this->fp_ != nullptr);
         __fsetlocking(&*(this->fp_), FSETLOCKING_BYCALLER);
-        if (!parser->ParseFileInstance(
-                [this](char* buf, int len) {
-                  return fread(buf, sizeof(char), len, this->fp_.get());
-                },
-                pull_record_func, lines)) {
-          is_error = true;
-        }
+        is_ok = parser->ParseFileInstance(
+            [this](char* buf, int len) {
+              return fread(buf, sizeof(char), len, this->fp_.get());
+            },
+            pull_record_func, lines);
+      }
+      if (!is_ok) {
+        LOG(WARNING) << "parser error, filename=" << filename
+                     << ", lines=" << lines;
       }
-    } while (is_error);
+    } while (!is_ok);
     timeline.Pause();
     VLOG(3) << "LoadIntoMemoryByLib() read all file, file=" << filename
             << ", cost time=" << timeline.ElapsedSec()

diff --git a/paddle/fluid/platform/gpu_info.cc b/paddle/fluid/platform/gpu_info.cc
@@ -19,6 +19,7 @@ limitations under the License. */
 
 #include "gflags/gflags.h"
 #include "paddle/fluid/platform/cuda_device_guard.h"
+#include "paddle/fluid/platform/dynload/cuda_driver.h"
 #include "paddle/fluid/platform/dynload/cudnn.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/lock_guard_ptr.h"
@@ -488,5 +489,16 @@ bool IsCudaMallocRecorded(int dev_id) {
   return RecordedCudaMallocHelper::Instance(dev_id)->NeedRecord();
 }
 
+//! cgpu acquire
+bool CGPUMemBfcTimeSharingAcquire(void) {
+  int acquire = -1001;
+  return platform::dynload::cuDriverGetVersion(&acquire) == CUDA_SUCCESS;
+}
+//! cgpu release
+bool CGPUMemBfcTimeSharingRelease(void) {
+  int release = -1002;
+  return platform::dynload::cuDriverGetVersion(&release) == CUDA_SUCCESS;
+}
+
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/gpu_info.h b/paddle/fluid/platform/gpu_info.h
@@ -120,6 +120,10 @@ bool RecordedCudaMemGetInfo(size_t *avail, size_t *total, size_t *actual_avail,
 uint64_t RecordedCudaMallocSize(int dev_id);
 
 bool IsCudaMallocRecorded(int dev_id);
+//! cgpu acquire
+bool CGPUMemBfcTimeSharingAcquire(void);
+//! cgpu release
+bool CGPUMemBfcTimeSharingRelease(void);
 
 }  // namespace platform
 }  // namespace paddle

diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
@@ -374,6 +374,8 @@ PYBIND11_MODULE(core_noavx, m) {
 
 #ifdef PADDLE_WITH_CUDA
   m.def("cudnn_version", &platform::CudnnVersion);
+  m.def("cgpu_mem_acquire", &platform::CGPUMemBfcTimeSharingAcquire);
+  m.def("cgpu_mem_release", &platform::CGPUMemBfcTimeSharingRelease);
 #endif
 
   m.def("from_dlpack", [](py::capsule *dltensor) {