[PATCH]Read synchronously for larger files while concatenating

concat.js

@@ -55,6 +55,8 @@ class Concat extends Plugin {
     this.footer = options.footer;
     this.footerFiles = options.footerFiles;
     this.separator = (options.separator != null) ? options.separator : '\n';
+    this.contentLimit = options.contentLimit || 10000;
+    this.inputNode = inputNode;
 
     ensureNoGlob('headerFiles', this.headerFiles);
     ensureNoGlob('footerFiles', this.footerFiles);
@@ -110,7 +112,9 @@ class Concat extends Plugin {
         header: this.header,
         headerFiles: this.headerFiles,
         footerFiles: this.footerFiles,
-        footer: this.footer
+        footer: this.footer,
+        contentLimit: this.contentLimit,
+        baseDir: this.inputNode
       }));
     }
 

lib/strategies/simple.js

@@ -1,8 +1,13 @@
 'use strict';
 
 var findIndex = require('find-index');
+var fs = require('fs');
+var baseDir;
 
 function contentMapper(entry) {
+  if (entry.isFileContentLarge) {
+    entry.content = fs.readFileSync(baseDir + entry.file, 'utf-8');
+  }
   return entry.content;
 }
 
@@ -18,6 +23,8 @@ class SimpleConcat {
     this.headerFiles = attrs.headerFiles || [];
     this.footerFiles = attrs.footerFiles || [];
     this.footer = attrs.footer;
+    this.contentLimit = attrs.contentLimit;
+    baseDir = attrs.baseDir ? attrs.baseDir + '/' : '';
 
     // Internally, we represent the concatenation as a series of entries. These
     // entries have a 'file' attribute for lookup/sorting and a 'content' property
@@ -37,6 +44,14 @@ class SimpleConcat {
     return findIndex(this._internal, function(entry) { return entry.file === file; });
   }
 
+  /**
+   * Check if the content of the file is too large to be stored in memory
+   */
+  _isContentLarge(content) {
+    var contentLimit = this.contentLimit;
+    return contentLimit && content.length > contentLimit;
+  }
+
   /**
    * Updates the contents of a header file.
    */
@@ -77,10 +92,16 @@ class SimpleConcat {
     }
 
     var entry = {
-      file: file,
-      content: content
+      file: file
     };
 
+    if (this._isContentLarge(content)) {
+      entry.content = '';
+      entry.isFileContentLarge = true;
+    } else {
+      entry.content = content;
+    }
+
     var index = findIndex(this._internal, function(entry) { return entry.file > file; });
     if (index === -1) {
       this._internal.push(entry);
@@ -100,7 +121,13 @@ class SimpleConcat {
       throw new Error('Trying to update ' + file + ' but it has not been read before');
     }
 
-    this._internal[index].content = content;
+    if (this._isContentLarge(content)) {
+      this._internal[index].content = '';
+      this._internal[index].isFileContentLarge = true;
+    } else {
+      this._internal[index].content = content;
+    }
+
   }
 
   removeFile(file) {

test/simple-concat-test.js

@@ -44,6 +44,18 @@ describe('simple-concat', function() {
     expectFile('no-sourcemap.map').notIn(result);
   }));
 
+  it('concatenates files with content higher than limit', co.wrap(function* () {
+    var node = concat(firstFixture, {
+      outputFile: '/all-inner.js',
+      inputFiles: ['inner/*.js'],
+      contentLimit: 10,
+      sourceMapConfig: { enabled: false }
+    });
+    builder = new broccoli.Builder(node);
+    let result = yield builder.build();
+    expectFile('all-inner.js').withoutSrcURL().in(result);
+  }));
+
   /**
    * Tests below here should appear for both simple-concat and sourcemap-concat.
    */