From 2fa718e413615334b1e6030c7dc4d0943e268a10 Mon Sep 17 00:00:00 2001 From: utk14 Date: Mon, 25 Oct 2021 14:36:41 +0530 Subject: [PATCH 1/6] Issue SB-24793 feat: Assessment archived data:: getArchivalBlobUrl method implementation --- .../analytics/framework/util/CommonUtil.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index 37873054..5f2f1d72 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -748,4 +748,19 @@ object CommonUtil { connProperties.setProperty("password", pass) connProperties } + + def getArchivalBlobUrl(store: String, filePath: String, bucket:String, batchId: Any, year: Any, weekNum: Any, format: String): String = { + store match { + case "local" => + filePath + s"${batchId}/${year}-${weekNum}-*.${format}" + // $COVERAGE-OFF$ for azure testing + case "azure" => + val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" + getAzureFile(bucket,file) + case "s3" => + val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" + getS3File(bucket, file) + // $COVERAGE-ON$ + } + } } \ No newline at end of file From 12036b50faf67cd6baa87d2a49b96fce31e8a65c Mon Sep 17 00:00:00 2001 From: utk14 Date: Mon, 25 Oct 2021 15:01:54 +0530 Subject: [PATCH 2/6] Issue SB-24793 feat: Assessment archived data:: getArchivalBlobUrl method implementation --- .../org/ekstep/analytics/framework/util/CommonUtil.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index 5f2f1d72..db84f351 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -748,8 +748,8 @@ object CommonUtil { connProperties.setProperty("password", pass) connProperties } - - def getArchivalBlobUrl(store: String, filePath: String, bucket:String, batchId: Any, year: Any, weekNum: Any, format: String): String = { + + def getArchivalBlobUrl(store: String, filePath: String, bucket:String, batchId: String, year: Any, weekNum: String, format: String): String = { store match { case "local" => filePath + s"${batchId}/${year}-${weekNum}-*.${format}" From 8caef132d499f63aacd701cce3bbbd08feb35211 Mon Sep 17 00:00:00 2001 From: utk14 Date: Mon, 25 Oct 2021 15:43:25 +0530 Subject: [PATCH 3/6] Issue SB-24793 feat: Assessment archived data:: Testcases added --- .../ekstep/analytics/framework/util/CommonUtil.scala | 4 +--- .../analytics/framework/util/TestCommonUtil.scala | 11 ++++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index db84f351..abec1951 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -748,19 +748,17 @@ object CommonUtil { connProperties.setProperty("password", pass) connProperties } - + def getArchivalBlobUrl(store: String, filePath: String, bucket:String, batchId: String, year: Any, weekNum: String, format: String): String = { store match { case "local" => filePath + s"${batchId}/${year}-${weekNum}-*.${format}" - // $COVERAGE-OFF$ for azure testing case "azure" => val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" getAzureFile(bucket,file) case "s3" => val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" getS3File(bucket, file) - // $COVERAGE-ON$ } } } \ No newline at end of file diff --git a/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala b/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala index 685cd749..34b07d91 100644 --- a/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala +++ b/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala @@ -324,7 +324,16 @@ class TestCommonUtil extends BaseSpec { val copiedFile = fileUtil.copy("src/test/resources/sample_telemetry.log", "src/test/resources/sample_telemetry.json", sc.hadoopConfiguration) sc.textFile(copiedFile, 1).count() should be (7437) fileUtil.delete(sc.hadoopConfiguration, copiedFile) - + + val localUrl = CommonUtil.getArchivalBlobUrl("local", "src/test/resources/", "local", "batch-001", "2021", "21", "csv.gz") + localUrl should be ("src/test/resources/batch-001/2021-21-*.csv.gz") + + val azureUrl = CommonUtil.getArchivalBlobUrl("azure", "report/archival-data/", "telemetry-data-store", "batch-001", "2021", "21", "csv.gz") + azureUrl should be ("wasb://telemetry-data-store@azure-test-key.blob.core.windows.net/report/archival-data/batch-001/2021-21-*.csv.gz") + + val s3Url = CommonUtil.getArchivalBlobUrl("s3", "report/archival-data/", "telemetry-data-store", "batch-001", "2021", "21", "csv.gz") + s3Url should be ("s3n://telemetry-data-store/report/archival-data/batch-001/2021-21-*.csv.gz") + sc.stop() } From 0864001660321d0b58d681519e421462564158c3 Mon Sep 17 00:00:00 2001 From: utk14 Date: Mon, 25 Oct 2021 17:30:11 +0530 Subject: [PATCH 4/6] Issue SB-24793 feat: Assessment archived data:: gcp case added --- .../org/ekstep/analytics/framework/util/CommonUtil.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index abec1951..11cbebb1 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -759,6 +759,11 @@ object CommonUtil { case "s3" => val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" getS3File(bucket, file) + // $COVERAGE-OFF$ for azure testing + case "gcp" => + //TODO - Need to support the S3 As well. + throw new Exception("gcp is currently not supported.") + // $COVERAGE-ON$ } } } \ No newline at end of file From 492c0913fa4c8ac956248df5024873cc43b8b2ea Mon Sep 17 00:00:00 2001 From: utk14 Date: Mon, 25 Oct 2021 17:31:13 +0530 Subject: [PATCH 5/6] Issue SB-24793 feat: Assessment archived data:: gcp case added --- .../org/ekstep/analytics/framework/util/CommonUtil.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index 11cbebb1..d5af06aa 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -761,9 +761,9 @@ object CommonUtil { getS3File(bucket, file) // $COVERAGE-OFF$ for azure testing case "gcp" => - //TODO - Need to support the S3 As well. + //TODO - Need to support the GCP As well. throw new Exception("gcp is currently not supported.") - // $COVERAGE-ON$ + // $COVERAGE-ON$ } } } \ No newline at end of file From fbb4d1fddd43db5447f5e0bf208b43c612241343 Mon Sep 17 00:00:00 2001 From: utk14 Date: Tue, 26 Oct 2021 10:11:55 +0530 Subject: [PATCH 6/6] Issue SB-24793 feat: Assessment archived data:: review comments resolved --- .../ekstep/analytics/framework/util/CommonUtil.scala | 10 ++++------ .../analytics/framework/util/TestCommonUtil.scala | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index d5af06aa..54b47f83 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -749,16 +749,14 @@ object CommonUtil { connProperties } - def getArchivalBlobUrl(store: String, filePath: String, bucket:String, batchId: String, year: Any, weekNum: String, format: String): String = { + def getBlobUrl(store: String, filePath: String, bucket:String): String = { store match { case "local" => - filePath + s"${batchId}/${year}-${weekNum}-*.${format}" + filePath case "azure" => - val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" - getAzureFile(bucket,file) + getAzureFile(bucket,filePath) case "s3" => - val file: String = s"${filePath}${batchId}/${year}-${weekNum}-*.${format}" - getS3File(bucket, file) + getS3File(bucket, filePath) // $COVERAGE-OFF$ for azure testing case "gcp" => //TODO - Need to support the GCP As well. diff --git a/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala b/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala index 34b07d91..04838e01 100644 --- a/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala +++ b/analytics-core/src/test/scala/org/ekstep/analytics/framework/util/TestCommonUtil.scala @@ -325,13 +325,13 @@ class TestCommonUtil extends BaseSpec { sc.textFile(copiedFile, 1).count() should be (7437) fileUtil.delete(sc.hadoopConfiguration, copiedFile) - val localUrl = CommonUtil.getArchivalBlobUrl("local", "src/test/resources/", "local", "batch-001", "2021", "21", "csv.gz") + val localUrl = CommonUtil.getBlobUrl("local", "src/test/resources/batch-001/2021-21-*.csv.gz", "local") localUrl should be ("src/test/resources/batch-001/2021-21-*.csv.gz") - val azureUrl = CommonUtil.getArchivalBlobUrl("azure", "report/archival-data/", "telemetry-data-store", "batch-001", "2021", "21", "csv.gz") + val azureUrl = CommonUtil.getBlobUrl("azure", "report/archival-data/batch-001/2021-21-*.csv.gz", "telemetry-data-store") azureUrl should be ("wasb://telemetry-data-store@azure-test-key.blob.core.windows.net/report/archival-data/batch-001/2021-21-*.csv.gz") - val s3Url = CommonUtil.getArchivalBlobUrl("s3", "report/archival-data/", "telemetry-data-store", "batch-001", "2021", "21", "csv.gz") + val s3Url = CommonUtil.getBlobUrl("s3", "report/archival-data/batch-001/2021-21-*.csv.gz", "telemetry-data-store") s3Url should be ("s3n://telemetry-data-store/report/archival-data/batch-001/2021-21-*.csv.gz") sc.stop()