Merge pull request #424 from jamhall/list-objects-pagination

Fully implement List Objects V2 with pagination response elements

lib/controllers/bucket.js

@@ -1,5 +1,8 @@
 "use strict";
 
+const crypto = require("crypto");
+
+const { DUMMY_ACCOUNT } = require("../models/account");
 const S3Error = require("../models/error");
 const {
   S3CorsConfiguration,
@@ -16,6 +19,34 @@ async function utf8BodyParser(ctx) {
   });
 }
 
+function generateContinuationToken(bucket, keyName, region) {
+  const key = Buffer.alloc(8, "S3RVER", "utf8");
+  const iv = crypto.randomBytes(8);
+  // ensure the first byte of IV lies between [212, 216)
+  iv[0] = (iv[0] & 0b00000011) | 0b11010100;
+  // use DES for its 8-byte block size
+  // (real S3 has blocks of lengths [9,8,7] repeating)
+  const cipher = crypto.createCipheriv("des", key, iv);
+  return Buffer.concat([
+    iv,
+    cipher.update(`${region}/${bucket}/${keyName}`, "utf8"),
+    cipher.final()
+  ]).toString("base64");
+}
+
+function decipherContinuationToken(token) {
+  const buf = Buffer.from(token, "base64");
+  if (buf.length < 8) return "";
+  const key = Buffer.alloc(8, "S3RVER", "utf8");
+  const iv = buf.slice(0, 8);
+  const decipher = crypto.createDecipheriv("des", key, iv);
+  const ciphertext = buf.slice(8);
+  return Buffer.concat([
+    decipher.update(ciphertext),
+    decipher.final()
+  ]).toString("utf8");
+}
+
 exports.bucketExists = async function bucketExists(ctx, next) {
   const bucketName = ctx.params.bucket;
   const bucket = await ctx.app.store.getBucket(bucketName);
@@ -112,44 +143,150 @@ exports.deleteBucketWebsite = async function deleteBucketWebsite(ctx) {
  */
 exports.getBucket = async function getBucket(ctx) {
   const options = {
-    delimiter: ctx.query["delimiter"],
-    marker: ctx.query["marker"],
-    maxKeys: Math.min(1000, Number(ctx.query["max-keys"]) || Infinity),
-    prefix: ctx.query["prefix"]
+    delimiter: ctx.query["delimiter"] || undefined,
+    encodingType: ctx.query["encoding-type"], // currently unimplemented
+    maxKeys: 1000,
+    startAfter: undefined,
+    prefix: ctx.query["prefix"] || undefined,
+    fetchOwner: undefined
   };
+  if (ctx.query["max-keys"]) {
+    if (!ctx.query["max-keys"].match(/^-?\d+$/)) {
+      throw new S3Error(
+        "InvalidArgument",
+        "Provided max-keys not an integer or within integer range",
+        {
+          ArgumentName: "max-keys",
+          ArgumentValue: ctx.query["max-keys"]
+        }
+      );
+    }
+    const maxKeys = Number(ctx.query["max-keys"]);
+    if (maxKeys < 0 || 2147483647 < maxKeys) {
+      throw new S3Error(
+        "InvalidArgument",
+        "Argument maxKeys must be an integer between 0 and 2147483647",
+        {
+          ArgumentName: "maxKeys",
+          ArgumentValue: maxKeys
+        }
+      );
+    }
+    options.maxKeys = Math.min(1000, maxKeys);
+  }
+  switch (ctx.query["list-type"]) {
+    case "2":
+      if ("marker" in ctx.query) {
+        throw new S3Error(
+          "InvalidArgument",
+          "Marker unsupported with REST.GET.BUCKET in list-type=2",
+          { ArgumentName: "marker" }
+        );
+      }
+      if (ctx.query["continuation-token"]) {
+        const token = decipherContinuationToken(
+          ctx.query["continuation-token"]
+        );
+        const [, region, bucket, startAfter] =
+          /([\w-.]+)\/([\w-.]+)\/(.+)/.exec(token) || [];
+        if (region !== "us-east-1" || bucket !== ctx.params.bucket) {
+          throw new S3Error(
+            "InvalidArgument",
+            "The continuation token provided is incorrect",
+            { ArgumentName: "continuation-token" }
+          );
+        }
+        options.startAfter = startAfter;
+      } else {
+        options.startAfter = ctx.query["start-after"];
+      }
+      options.fetchOwner = ctx.query["fetch-owner"] === "true";
+      break;
+    default:
+      // fall back to version 1
+      if ("continuation-token" in ctx.query) {
+        throw new S3Error(
+          "InvalidArgument",
+          "continuation-token only supported in REST.GET.BUCKET with list-type=2",
+          { ArgumentName: "continuation-token" }
+        );
+      }
+      if ("start-after" in ctx.query) {
+        throw new S3Error(
+          "InvalidArgument",
+          // yes, for some reason they decided to camelCase the start-after argument in this error message
+          "startAfter only supported in REST.GET.BUCKET with list-type=2",
+          { ArgumentName: "start-after" }
+        );
+      }
+      options.fetchOwner = true;
+      options.startAfter = ctx.query["marker"];
+      break;
+  }
   ctx.logger.info(
     'Fetched bucket "%s" with options %s',
     ctx.params.bucket,
     options
   );
   try {
-    const results = await ctx.store.listObjects(ctx.params.bucket, options);
+    const result =
+      options.maxKeys === 0
+        ? {
+            objects: [],
+            commonPrefixes: [],
+            isTruncated: false
+          }
+        : await ctx.store.listObjects(ctx.params.bucket, options);
     ctx.logger.info(
       'Found %d objects for bucket "%s"',
-      results.objects.length,
+      result.objects.length,
       ctx.params.bucket
     );
     ctx.body = {
       ListBucketResult: {
         "@": { xmlns: "http://doc.s3.amazonaws.com/2006-03-01/" },
-        IsTruncated: results.isTruncated || false,
-        Marker: options.marker || "",
         Name: ctx.params.bucket,
-        Prefix: options.prefix || "",
-        MaxKeys: options.maxKeys,
-        CommonPrefixes: results.commonPrefixes.map(prefix => ({
-          Prefix: prefix
-        })),
-        Contents: results.objects.map(object => ({
+        Prefix: options.prefix || "", // never omit
+        ...(ctx.query["list-type"] === "2"
+          ? {
+              StartAfter: ctx.query["continuation-token"]
+                ? undefined
+                : options.startAfter,
+              ContinuationToken: ctx.query["continuation-token"] || undefined,
+              NextContinuationToken: result.isTruncated
+                ? generateContinuationToken(
+                    ctx.params.bucket,
+                    result.objects[result.objects.length - 1].key,
+                    "us-east-1"
+                  )
+                : undefined,
+              KeyCount: result.objects.length
+            }
+          : {
+              Marker: options.startAfter || "", // never omit
+              NextMarker:
+                options.delimiter && result.isTruncated
+                  ? result.objects[result.objects.length - 1].key
+                  : undefined
+            }),
+        MaxKeys: ctx.query["max-keys"] || 1000, // S3 has a hard limit at 1000 but will still echo back the original input
+        Delimiter: options.delimiter || undefined, // omit when "" or undefined
+        IsTruncated: result.isTruncated || false,
+        Contents: result.objects.map(object => ({
           Key: object.key,
           LastModified: object.lastModifiedDate.toISOString(),
           ETag: object.metadata["etag"],
           Size: object.size,
-          StorageClass: "STANDARD",
-          Owner: {
-            ID: 123,
-            DisplayName: "S3rver"
-          }
+          Owner: options.fetchOwner
+            ? {
+                ID: DUMMY_ACCOUNT.id,
+                DisplayName: DUMMY_ACCOUNT.displayName
+              }
+            : undefined,
+          StorageClass: "STANDARD"
+        })),
+        CommonPrefixes: result.commonPrefixes.map(prefix => ({
+          Prefix: prefix
         }))
       }
     };

lib/controllers/object.js

@@ -2,7 +2,9 @@
 
 const crypto = require("crypto");
 const xmlParser = require("fast-xml-parser");
+const he = require("he");
 
+const { DUMMY_ACCOUNT } = require("../models/account");
 const S3Error = require("../models/error");
 const S3Event = require("../models/event");
 const S3Object = require("../models/object");
@@ -23,7 +25,9 @@ async function xmlBodyParser(ctx) {
         "our published schema."
     );
   }
-  ctx.request.body = xmlParser.parse(xmlString);
+  ctx.request.body = xmlParser.parse(xmlString, {
+    tagValueProcessor: a => he.decode(a)
+  });
 }
 
 function triggerS3Event(ctx, eventData) {
@@ -205,8 +209,8 @@ exports.getObjectAcl = async function getObjectAcl(ctx) {
     AccessControlPolicy: {
       "@": { xmlns: "http://doc.s3.amazonaws.com/2006-03-01/" },
       Owner: {
-        ID: 123,
-        DisplayName: "S3rver"
+        ID: DUMMY_ACCOUNT.id,
+        DisplayName: DUMMY_ACCOUNT.displayName
       },
       AccessControlList: {
         Grant: {

lib/controllers/service.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const { DUMMY_ACCOUNT } = require("../models/account");
+
 /*
  * Operations on Buckets
  * The following methods correspond to operations you can perform on the Amazon S3 service.
@@ -19,8 +21,8 @@ exports.getService = async function getService(ctx) {
     ListAllMyBucketsResult: {
       "@": { xmlns: "http://doc.s3.amazonaws.com/2006-03-01/" },
       Owner: {
-        ID: 123,
-        DisplayName: "S3rver"
+        ID: DUMMY_ACCOUNT.id,
+        DisplayName: DUMMY_ACCOUNT.displayName
       },
       Buckets: {
         Bucket: buckets.map(bucket => ({

lib/middleware/authentication.js

@@ -4,19 +4,11 @@ const { createHmac } = require("crypto");
 const { mapKeys, pickBy } = require("lodash");
 const querystring = require("querystring");
 
+const AWSAccount = require("../models/account");
 const S3Error = require("../models/error");
 const { RESPONSE_HEADERS } = require("./response-header-override");
 const { parseDate, parseISO8601String } = require("../utils");
 
-/**
- * Hardcoded dummy user used for authenticated requests.
- */
-const DUMMY_USER = {
-  accountId: 123456789000,
-  accessKeyId: "S3RVER",
-  secretAccessKey: "S3RVER"
-};
-
 const SUBRESOURCES = {
   acl: 1,
   accelerate: 1,
@@ -151,27 +143,33 @@ module.exports = () =>
     }
 
     let stringToSign;
+    const account = AWSAccount.registry.get(signature.accessKeyId);
+    if (!account) {
+      throw new S3Error(
+        "InvalidAccessKeyId",
+        "The AWS Access Key Id you provided does not exist in our records.",
+        { AWSAccessKeyId: signature.accessKeyId }
+      );
+    }
+
     if (signature.version === 2) {
       stringToSign = getStringToSign(canonicalRequest);
 
-      const keys = enumerateSecretAccessKeys(signature.accessKeyId);
-      for (const signingKey of keys) {
-        const calculatedSignature = calculateSignature(
-          stringToSign,
-          signingKey,
-          signature.algorithm
-        );
-        if (signatureProvided === calculatedSignature) {
-          ctx.state.accountId = getAccountId(signature.accessKeyId);
-          break;
-        }
+      const signingKey = account.accessKeys.get(signature.accessKeyId);
+      const calculatedSignature = calculateSignature(
+        stringToSign,
+        signingKey,
+        signature.algorithm
+      );
+      if (signatureProvided === calculatedSignature) {
+        ctx.state.account = account;
       }
     } else if (signature.version === 4) {
       // Signature version 4 calculation is unimplemeneted
-      ctx.state.accountId = getAccountId(signature.accessKeyId);
+      ctx.state.account = account;
     }
 
-    if (!ctx.state.accountId) {
+    if (!ctx.state.account) {
       throw new S3Error(
         "SignatureDoesNotMatch",
         "The request signature we calculated does not match the signature you provided. Check " +
@@ -445,34 +443,6 @@ function parseQueryV4(query) {
   };
 }
 
-/**
- * Generator that looks up secret keys for a given access key ID.
- *
- * @param {String} accessKeyId
- */
-function* enumerateSecretAccessKeys(accessKeyId) {
-  if (accessKeyId === DUMMY_USER.accessKeyId) {
-    yield DUMMY_USER.secretAccessKey;
-  } else {
-    throw new S3Error(
-      "InvalidAccessKeyId",
-      "The AWS Access Key Id you provided does not exist in our records.",
-      { AWSAccessKeyId: accessKeyId }
-    );
-  }
-}
-
-/**
- * Looks up an AWS account ID from an access key ID.
- *
- * @param {String} accessKeyId
- */
-function getAccountId(accessKeyId) {
-  if (accessKeyId === DUMMY_USER.accessKeyId) {
-    return DUMMY_USER.accountId;
-  }
-}
-
 /**
  * Generates a string to be signed for the specified signature version.
  *

lib/middleware/response-header-override.js

@@ -51,7 +51,7 @@ exports = module.exports = () =>
     switch (ctx.method) {
       case "HEAD":
       case "GET":
-        if (!isEmpty(overrides) && !ctx.state.accountId) {
+        if (!isEmpty(overrides) && !ctx.state.account) {
           throw new S3Error(
             "InvalidRequest",
             "Request specific response headers cannot be used for anonymous " +