From 7860144e49f0744f11181bf12ea3d43fa63a3fb5 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 28 Feb 2018 09:53:58 -0500 Subject: [PATCH 01/14] use cumulus circleci image for testing --- .circleci/config.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c7c14d7fd9..f6cf4d3483e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 jobs: build_and_test: docker: - - image: circleci/node:6.10 + - image: cumuluss/circleci:node-6.10 - name: localstack image: localstack/localstack working_directory: ~/project @@ -18,6 +18,20 @@ jobs: - run: name: Installing Dependencies command: | + # start ftp + sudo rm -rf /home/vsftpd + sudo ln -s /home/circeci/project/.tmp-test-data /home/vsftpd + sudo service vsftpd start + + # start http service + sudo rm -rf /var/www/html + sudo ln -s /home/circeci/project/.tmp-test-data /var/www/html + sudo service apache2 start + + # start sftp service + sudo bash /usr/sbin/sftpd.sh + sudo ln -s /home/circeci/project/.tmp-test-data /home/user + yarn install yarn bootstrap-no-build From 0406a4317a4d848865631e2c7684b1be564c21e9 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 28 Feb 2018 09:54:08 -0500 Subject: [PATCH 02/14] change http port to 3030 --- cumulus/tasks/discover-granules/tests/index.js | 2 +- cumulus/tasks/discover-pdrs/tests/index.js | 2 +- cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js | 2 +- cumulus/tasks/sync-granule/tests/sync_granule_test.js | 6 +++--- docker-compose.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cumulus/tasks/discover-granules/tests/index.js b/cumulus/tasks/discover-granules/tests/index.js index 3ebd4f4d4a6..ea32312c396 100644 --- a/cumulus/tasks/discover-granules/tests/index.js +++ b/cumulus/tasks/discover-granules/tests/index.js @@ -119,7 +119,7 @@ test('discover granules using HTTP', async (t) => { event.config.provider = { id: 'MODAPS', protocol: 'http', - host: 'http://localhost:8080' + host: 'http://localhost:3030' }; await validateConfig(t, event.config); diff --git a/cumulus/tasks/discover-pdrs/tests/index.js b/cumulus/tasks/discover-pdrs/tests/index.js index d12d9a91527..ce9a3f804d0 100644 --- a/cumulus/tasks/discover-pdrs/tests/index.js +++ b/cumulus/tasks/discover-pdrs/tests/index.js @@ -182,7 +182,7 @@ test('test pdr discovery with HTTP assuming some PDRs are new', async (t) => { event.config.provider = { id: 'MODAPS', protocol: 'http', - host: 'http://localhost:8080' + host: 'http://localhost:3030' }; event.config.collection.provider_path = providerPath; event.input = {}; diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js index faa41c50ecd..a3b48eb97a5 100644 --- a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -79,7 +79,7 @@ test('parse PDR from HTTP endpoint', async (t) => { newPayload.config.provider = { id: 'MODAPS', protocol: 'http', - host: 'http://localhost:8080' + host: 'http://localhost:3030' }; newPayload.input = { pdr: { diff --git a/cumulus/tasks/sync-granule/tests/sync_granule_test.js b/cumulus/tasks/sync-granule/tests/sync_granule_test.js index 82c651a8c5e..37713290786 100644 --- a/cumulus/tasks/sync-granule/tests/sync_granule_test.js +++ b/cumulus/tasks/sync-granule/tests/sync_granule_test.js @@ -96,7 +96,7 @@ test('download Granule from HTTP endpoint', async (t) => { t.context.event.config.provider = { id: 'MODAPS', protocol: 'http', - host: 'http://localhost:8080' + host: 'http://localhost:3030' }; t.context.event.input.granules[0].files[0].path = `/${granulePath}`; @@ -238,7 +238,7 @@ test('download granule with checksum in file from an HTTP endpoint', async (t) = event.config.provider = { id: 'MODAPS', protocol: 'http', - host: 'http://localhost:8080' + host: 'http://localhost:3030' }; const granulePath = randomString(); @@ -288,7 +288,7 @@ test('download granule with checksum in file from an HTTP endpoint', async (t) = // const provider = { // id: 'MODAPS', // protocol: 'http', -// host: 'http://localhost:8080' +// host: 'http://localhost:3030' // }; // sinon.stub(S3, 'fileExists').callsFake(() => true); // const uploaded = sinon.stub(S3, 'upload').callsFake(() => '/test/test.hd'); diff --git a/docker-compose.yml b/docker-compose.yml index dafe9723fcf..aa5910d0674 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: http: image: centos/httpd ports: - - "8080:80" + - "3030:80" volumes: - "./.tmp-test-data:/var/www/html:ro" From 6bc4dd7df37830b827794d263e4d6cf8c5aa325a Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 28 Feb 2018 10:01:47 -0500 Subject: [PATCH 03/14] ignore ftp warning --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6cf4d3483e..e1a713ddc48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: # start ftp sudo rm -rf /home/vsftpd sudo ln -s /home/circeci/project/.tmp-test-data /home/vsftpd - sudo service vsftpd start + sudo service vsftpd start || true # start http service sudo rm -rf /var/www/html From 6c5eed1c82f19d7fd19d58cbb0f85f2dbe1bc53a Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 28 Feb 2018 10:06:44 -0500 Subject: [PATCH 04/14] fix a typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1a713ddc48..6e0a44fcb48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: sudo service apache2 start # start sftp service - sudo bash /usr/sbin/sftpd.sh + sudo bash /usr/sbin/sftp.sh sudo ln -s /home/circeci/project/.tmp-test-data /home/user yarn install From ce9907db245445a6a0c0860e126c0dc546a34cd1 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 28 Feb 2018 10:10:27 -0500 Subject: [PATCH 05/14] add user/password --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e0a44fcb48..eb3b1152172 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: sudo service apache2 start # start sftp service - sudo bash /usr/sbin/sftp.sh + sudo bash /usr/sbin/sftp.sh user:password sudo ln -s /home/circeci/project/.tmp-test-data /home/user yarn install From d9a3545eb2219f5c36b5b8389ed6f0ea8295aa90 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 10:16:42 -0500 Subject: [PATCH 06/14] comment out sftp --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb3b1152172..994c1b50395 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,17 +20,17 @@ jobs: command: | # start ftp sudo rm -rf /home/vsftpd - sudo ln -s /home/circeci/project/.tmp-test-data /home/vsftpd + sudo ln -s /home/circleci/project/.tmp-test-data /home/vsftpd sudo service vsftpd start || true # start http service sudo rm -rf /var/www/html - sudo ln -s /home/circeci/project/.tmp-test-data /var/www/html + sudo ln -s /home/circleci/project/.tmp-test-data /var/www/html sudo service apache2 start # start sftp service - sudo bash /usr/sbin/sftp.sh user:password - sudo ln -s /home/circeci/project/.tmp-test-data /home/user + # sudo bash /usr/sbin/sftp.sh user:password + # sudo ln -s /home/circleci/project/.tmp-test-data /home/user yarn install yarn bootstrap-no-build From 716f648cb87bc4e538778b6fd95fdd5e99e8bf59 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:20:48 -0500 Subject: [PATCH 07/14] update docker-compose to include localstack --- README.md | 4 +--- docker-compose.yml | 49 ++++++++++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1ad58cae238..7314ebad3a6 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,7 @@ Build and watch packages: The LocalStack repository has [installation instructions](https://github.com/localstack/localstack#installing). -Before running tests, start the LocalStack servers: - - $ localstack start +Localstack is included in the docker-compose file. You only need to run the docker-compose command in the next section in order to use it with your tests. ### Docker containers diff --git a/docker-compose.yml b/docker-compose.yml index aa5910d0674..07092cdb87e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,43 @@ -version: '2' +version: '3' services: local: - image: bogem/ftp + image: cumuluss/circleci:node-6.10 + command: start volumes: - - "./packages/test-data:/home/vsftpd" + - "./.tmp-test-data:/home/vsftpd:ro" + - "./.tmp-test-data:/var/www/html:ro" + - "./.tmp-test-data:/home/user" ports: - "20:20" - "21:21" - "47400-47470:47400-47470" - environment: - - FTP_USER=testuser - - FTP_PASS=testpass - - PASV_ADDRESS=127.0.0.1 + - "3030:3030" + - "2222:2222" + links: + - localstack depends_on: - - http - - sftp + - localstack - http: - image: centos/httpd - ports: - - "3030:80" + test: + image: cumuluss/circleci:node-6.10 volumes: + - ".:/home/circleci/project" + - "./.tmp-test-data:/home/vsftpd:ro" - "./.tmp-test-data:/var/www/html:ro" + - "./.tmp-test-data:/home/user" + ports: + - "20:20" + - "21:21" + - "47400-47470:47400-47470" + - "3030:3030" + - "2222:2222" + links: + - localstack + depends_on: + - localstack - sftp: - image: atmoz/sftp + localstack: + image: localstack/localstack ports: - - "2222:22" - volumes: - - "./.tmp-test-data:/home/user/test-data:ro" - command: user:password + - 4567-4582:4567-4582 + - 8080:8080 \ No newline at end of file From 2fdc145700207b45f2fe41018591b1122cfdf9d3 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:35:28 -0500 Subject: [PATCH 08/14] remove test-data path from sftp test folder --- cumulus/tasks/discover-granules/tests/index.js | 4 +--- cumulus/tasks/discover-pdrs/tests/index.js | 4 +--- cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js | 4 +--- cumulus/tasks/sync-granule/tests/sync_granule_test.js | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cumulus/tasks/discover-granules/tests/index.js b/cumulus/tasks/discover-granules/tests/index.js index ea32312c396..3f4d724bf01 100644 --- a/cumulus/tasks/discover-granules/tests/index.js +++ b/cumulus/tasks/discover-granules/tests/index.js @@ -57,9 +57,7 @@ test('discover granules using SFTP', async (t) => { fs.outputFile(path.join(providerPathDirectory, file), `This is ${file}`))); const event = cloneDeep(mur); - // The test-data prefix is required in the provider_path because of the way - // that the sftp container is configured in docker-compose.yml. - event.config.collection.provider_path = `test-data/${providerPath}`; + event.config.collection.provider_path = providerPath; event.config.provider = { id: 'MODAPS', protocol: 'sftp', diff --git a/cumulus/tasks/discover-pdrs/tests/index.js b/cumulus/tasks/discover-pdrs/tests/index.js index ce9a3f804d0..fdc3fb9fd46 100644 --- a/cumulus/tasks/discover-pdrs/tests/index.js +++ b/cumulus/tasks/discover-pdrs/tests/index.js @@ -258,9 +258,7 @@ test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => { username: 'user', password: 'password' }; - // The test-data prefix is required because of the way that the sftp - // container is configured in docker-compose.yml. - event.config.collection.provider_path = `test-data/${providerPath}`; + event.config.collection.provider_path = providerPath; event.input = {}; // Mark one of the PDRs as not new diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js index a3b48eb97a5..9a76c4c18fb 100644 --- a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -139,9 +139,7 @@ test('parse PDR from SFTP endpoint', async (t) => { newPayload.input = { pdr: { name: pdrName, - // The test-data prefix is required because of the way that the sftp - // container is configured in docker-compose.yml. - path: `/test-data/${providerPath}` + path: providerPath } }; diff --git a/cumulus/tasks/sync-granule/tests/sync_granule_test.js b/cumulus/tasks/sync-granule/tests/sync_granule_test.js index 37713290786..a274db1c3ea 100644 --- a/cumulus/tasks/sync-granule/tests/sync_granule_test.js +++ b/cumulus/tasks/sync-granule/tests/sync_granule_test.js @@ -148,9 +148,7 @@ test('download Granule from SFTP endpoint', async (t) => { password: 'password' }; - // The test-data prefix is required because of the way that the sftp - // container is configured in docker-compose.yml. - t.context.event.input.granules[0].files[0].path = `test-data/${granulePath}`; + t.context.event.input.granules[0].files[0].path = granulePath; validateConfig(t, t.context.event.config); validateInput(t, t.context.event.input); From a12284a79f82b8e2563ee425d33c6cf63d412880 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:35:37 -0500 Subject: [PATCH 09/14] uset test-data package for ftp tests --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 07092cdb87e..4300898e458 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: image: cumuluss/circleci:node-6.10 command: start volumes: - - "./.tmp-test-data:/home/vsftpd:ro" + - "./packages/test-data:/home/vsftpd:ro" - "./.tmp-test-data:/var/www/html:ro" - "./.tmp-test-data:/home/user" ports: From 261ffcf8d225fa572ba9523e33c31218e8031bd1 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:35:48 -0500 Subject: [PATCH 10/14] update symlink for ftp service --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 994c1b50395..16cb78eb87f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: command: | # start ftp sudo rm -rf /home/vsftpd - sudo ln -s /home/circleci/project/.tmp-test-data /home/vsftpd + sudo ln -s /home/circleci/project/packages/test-data /home/vsftpd sudo service vsftpd start || true # start http service From d993ca8fd1d532d301cc801d584e4105e0d0e81a Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:47:15 -0500 Subject: [PATCH 11/14] update test volume --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4300898e458..115d35fe924 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: image: cumuluss/circleci:node-6.10 volumes: - ".:/home/circleci/project" - - "./.tmp-test-data:/home/vsftpd:ro" + - "./packages/test-data:/home/vsftpd:ro" - "./.tmp-test-data:/var/www/html:ro" - "./.tmp-test-data:/home/user" ports: From cf8c3d736350b4e4c2d363f25822932b19350bbe Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 14:47:33 -0500 Subject: [PATCH 12/14] add command to test --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 115d35fe924..56e5c9e11f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: test: image: cumuluss/circleci:node-6.10 + command: /bin/bash volumes: - ".:/home/circleci/project" - "./packages/test-data:/home/vsftpd:ro" From 67d1bc5fab71b952a7cb0b84e8f8f82ec5622dbb Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 15:37:45 -0500 Subject: [PATCH 13/14] add sftp support to circleci --- .circleci/config.yml | 4 +-- .../tasks/discover-granules/tests/index.js | 25 +++---------------- cumulus/tasks/discover-pdrs/tests/index.js | 20 +++------------ .../tasks/parse-pdr/tests/parse_pdrs_test.js | 20 +++------------ .../sync-granule/tests/sync_granule_test.js | 15 +---------- docker-compose.yml | 4 +-- 6 files changed, 15 insertions(+), 73 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16cb78eb87f..82024144bfc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,8 +29,8 @@ jobs: sudo service apache2 start # start sftp service - # sudo bash /usr/sbin/sftp.sh user:password - # sudo ln -s /home/circleci/project/.tmp-test-data /home/user + sudo bash /usr/sbin/sftp.sh user:password + sudo cp -r /home/circleci/project/packages/test-data/* /home/user/ yarn install yarn bootstrap-no-build diff --git a/cumulus/tasks/discover-granules/tests/index.js b/cumulus/tasks/discover-granules/tests/index.js index 3f4d724bf01..fd6f16100ab 100644 --- a/cumulus/tasks/discover-granules/tests/index.js +++ b/cumulus/tasks/discover-granules/tests/index.js @@ -36,28 +36,12 @@ test('discover granules using FTP', async (t) => { test('discover granules using SFTP', async (t) => { const internalBucketName = randomString(); - const providerPath = randomString(); - - // Figure out the directory paths that we're working with - const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath); // Create providerPathDirectory and internal bucket - await Promise.all([ - fs.ensureDir(providerPathDirectory), - s3().createBucket({ Bucket: internalBucketName }).promise() - ]); - - // State sample files - const files = [ - 'granule-1.nc', 'granule-1.nc.md5', - 'granule-2.nc', 'granule-2.nc.md5', - 'granule-3.nc', 'granule-3.nc.md5' - ]; - await Promise.all(files.map((file) => - fs.outputFile(path.join(providerPathDirectory, file), `This is ${file}`))); + await s3().createBucket({ Bucket: internalBucketName }).promise(); const event = cloneDeep(mur); - event.config.collection.provider_path = providerPath; + event.config.collection.provider_path = 'granules/fake_granules'; event.config.provider = { id: 'MODAPS', protocol: 'sftp', @@ -83,10 +67,7 @@ test('discover granules using SFTP', async (t) => { } finally { // Clean up - await Promise.all([ - recursivelyDeleteS3Bucket(internalBucketName), - fs.remove(providerPathDirectory) - ]); + await recursivelyDeleteS3Bucket(internalBucketName); } }); diff --git a/cumulus/tasks/discover-pdrs/tests/index.js b/cumulus/tasks/discover-pdrs/tests/index.js index fdc3fb9fd46..6c5bd31bd50 100644 --- a/cumulus/tasks/discover-pdrs/tests/index.js +++ b/cumulus/tasks/discover-pdrs/tests/index.js @@ -224,17 +224,12 @@ test('test pdr discovery with HTTP assuming some PDRs are new', async (t) => { test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => { const internalBucketName = randomString(); - const providerPath = randomString(); // Figure out the directory paths that we're working with const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs'); - const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath); // Create providerPathDirectory and internal bucket - await Promise.all([ - fs.ensureDir(providerPathDirectory), - s3().createBucket({ Bucket: internalBucketName }).promise() - ]); + await s3().createBucket({ Bucket: internalBucketName }).promise(); try { // Copy the PDRs to the SFTP directory @@ -243,10 +238,6 @@ test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => { const oldPdr = pdrFilenames[0]; const newPdrs = pdrFilenames.slice(1); - await Promise.all(pdrFilenames.map((pdrFilename) => fs.copy( - path.join(testDataDirectory, pdrFilename), - path.join(providerPathDirectory, pdrFilename)))); - // Build the event const event = cloneDeep(input); event.config.bucket = internalBucketName; @@ -258,13 +249,13 @@ test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => { username: 'user', password: 'password' }; - event.config.collection.provider_path = providerPath; + event.config.collection.provider_path = 'pdrs'; event.input = {}; // Mark one of the PDRs as not new await s3().putObject({ Bucket: internalBucketName, - // 'pdrs' is the default 'folder' value in the Discover contructor + // 'pdrs' is the default 'folder' value in the Discover constructor Key: `${event.config.stack}/pdrs/${oldPdr}`, Body: 'Pretend this is a PDR' }).promise(); @@ -289,9 +280,6 @@ test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => { } finally { // Clean up - await Promise.all([ - recursivelyDeleteS3Bucket(internalBucketName), - fs.remove(providerPathDirectory) - ]); + await recursivelyDeleteS3Bucket(internalBucketName); } }); diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js index 9a76c4c18fb..a15399750cf 100644 --- a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -115,14 +115,9 @@ test('parse PDR from HTTP endpoint', async (t) => { test('parse PDR from SFTP endpoint', async (t) => { const internalBucketName = randomString(); - const providerPath = randomString(); - const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath); // Create providerPathDirectory and internal bucket - await Promise.all([ - fs.ensureDir(providerPathDirectory), - s3().createBucket({ Bucket: internalBucketName }).promise() - ]); + await s3().createBucket({ Bucket: internalBucketName }).promise(); const pdrName = 'MOD09GQ.PDR'; @@ -139,7 +134,7 @@ test('parse PDR from SFTP endpoint', async (t) => { newPayload.input = { pdr: { name: pdrName, - path: providerPath + path: 'pdrs' } }; @@ -147,12 +142,6 @@ test('parse PDR from SFTP endpoint', async (t) => { await validateConfig(t, newPayload.config); try { - // Stage the file to be downloaded - const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs'); - await fs.copy( - path.join(testDataDirectory, pdrName), - path.join(providerPathDirectory, pdrName)); - const output = await parsePdr(newPayload); await validateOutput(t, output); @@ -168,10 +157,7 @@ test('parse PDR from SFTP endpoint', async (t) => { } finally { // Clean up - await Promise.all([ - recursivelyDeleteS3Bucket(internalBucketName), - fs.remove(providerPathDirectory) - ]); + await recursivelyDeleteS3Bucket(internalBucketName); } }); diff --git a/cumulus/tasks/sync-granule/tests/sync_granule_test.js b/cumulus/tasks/sync-granule/tests/sync_granule_test.js index a274db1c3ea..d4050ec6110 100644 --- a/cumulus/tasks/sync-granule/tests/sync_granule_test.js +++ b/cumulus/tasks/sync-granule/tests/sync_granule_test.js @@ -136,9 +136,6 @@ test('download Granule from HTTP endpoint', async (t) => { }); test('download Granule from SFTP endpoint', async (t) => { - const granulePath = randomString(); - const localGranulePath = path.join(await findTmpTestDataDirectory(), granulePath); - t.context.event.config.provider = t.context.event.config.provider = { id: 'MODAPS', protocol: 'sftp', @@ -148,21 +145,14 @@ test('download Granule from SFTP endpoint', async (t) => { password: 'password' }; - t.context.event.input.granules[0].files[0].path = granulePath; + t.context.event.input.granules[0].files[0].path = '/granules'; validateConfig(t, t.context.event.config); validateInput(t, t.context.event.input); - await fs.mkdir(localGranulePath); try { const granuleFilename = t.context.event.input.granules[0].files[0].name; - // Stage the file to be downloaded - await fs.copy( - path.join(await findTestDataDirectory(), 'granules', granuleFilename), - path.join(localGranulePath, granuleFilename) - ); - const output = await syncGranule(t.context.event); validateOutput(t, output); @@ -180,9 +170,6 @@ test('download Granule from SFTP endpoint', async (t) => { } else throw e; } - finally { - fs.remove(localGranulePath); - } }); test('download granule from S3 provider', async (t) => { diff --git a/docker-compose.yml b/docker-compose.yml index 56e5c9e11f0..fa52f30adfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: volumes: - "./packages/test-data:/home/vsftpd:ro" - "./.tmp-test-data:/var/www/html:ro" - - "./.tmp-test-data:/home/user" + - "./packages/test-data:/home/user" ports: - "20:20" - "21:21" @@ -25,7 +25,7 @@ services: - ".:/home/circleci/project" - "./packages/test-data:/home/vsftpd:ro" - "./.tmp-test-data:/var/www/html:ro" - - "./.tmp-test-data:/home/user" + - "./packages/test-data:/home/user" ports: - "20:20" - "21:21" From b2b828facc9c6fb7fa9f0c242252dc2986674353 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 1 Mar 2018 15:54:27 -0500 Subject: [PATCH 14/14] add missing fake granules --- packages/test-data/granules/fake_granules/granule-1.nc | 1 + packages/test-data/granules/fake_granules/granule-1.nc.md5 | 1 + packages/test-data/granules/fake_granules/granule-2.nc | 1 + packages/test-data/granules/fake_granules/granule-2.nc.md5 | 1 + packages/test-data/granules/fake_granules/granule-3.nc | 1 + packages/test-data/granules/fake_granules/granule-3.nc.md5 | 1 + 6 files changed, 6 insertions(+) create mode 100644 packages/test-data/granules/fake_granules/granule-1.nc create mode 100644 packages/test-data/granules/fake_granules/granule-1.nc.md5 create mode 100644 packages/test-data/granules/fake_granules/granule-2.nc create mode 100644 packages/test-data/granules/fake_granules/granule-2.nc.md5 create mode 100644 packages/test-data/granules/fake_granules/granule-3.nc create mode 100644 packages/test-data/granules/fake_granules/granule-3.nc.md5 diff --git a/packages/test-data/granules/fake_granules/granule-1.nc b/packages/test-data/granules/fake_granules/granule-1.nc new file mode 100644 index 00000000000..5825481ecb4 --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-1.nc @@ -0,0 +1 @@ +This is granule-1.nc \ No newline at end of file diff --git a/packages/test-data/granules/fake_granules/granule-1.nc.md5 b/packages/test-data/granules/fake_granules/granule-1.nc.md5 new file mode 100644 index 00000000000..ab1f7294701 --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-1.nc.md5 @@ -0,0 +1 @@ +This is granule-1.nc.md5 \ No newline at end of file diff --git a/packages/test-data/granules/fake_granules/granule-2.nc b/packages/test-data/granules/fake_granules/granule-2.nc new file mode 100644 index 00000000000..b2e6b3b122d --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-2.nc @@ -0,0 +1 @@ +This is granule-2.nc \ No newline at end of file diff --git a/packages/test-data/granules/fake_granules/granule-2.nc.md5 b/packages/test-data/granules/fake_granules/granule-2.nc.md5 new file mode 100644 index 00000000000..bdbdaa2a6bb --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-2.nc.md5 @@ -0,0 +1 @@ +This is granule-2.nc.md5 \ No newline at end of file diff --git a/packages/test-data/granules/fake_granules/granule-3.nc b/packages/test-data/granules/fake_granules/granule-3.nc new file mode 100644 index 00000000000..c3fcb07ba8d --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-3.nc @@ -0,0 +1 @@ +This is granule-3.nc \ No newline at end of file diff --git a/packages/test-data/granules/fake_granules/granule-3.nc.md5 b/packages/test-data/granules/fake_granules/granule-3.nc.md5 new file mode 100644 index 00000000000..abf43b928a8 --- /dev/null +++ b/packages/test-data/granules/fake_granules/granule-3.nc.md5 @@ -0,0 +1 @@ +This is granule-3.nc.md5 \ No newline at end of file