Merge pull request #89 from v79/86-move-generated-folder-to-new-s3-bucket

86 move generated folder to new s3 bucket

Author: v79

.gitignore

@@ -48,3 +48,6 @@ ImageProcessor/.gradle/buildOutputCleanup/buildOutputCleanup.lock
 ImageProcessor/.gradle/buildOutputCleanup/cache.properties
 ImageProcessor/.gradle/buildOutputCleanup/outputFiles.bin
 ImageProcessor/.gradle/vcs-1/gc.properties
+Web/build/*
+Web/node_modules/*
+/DevDiary/*

.idea/.gitignore

@@ -10,7 +10,7 @@ plugins {
 }
 
 group = "org.liamjd.cantilever"
-version = "0.0.11"
+version = "0.0.12"
 
 repositories {
     mavenCentral()

.idea/codeStyles/Project.xml

@@ -4,15 +4,12 @@ import com.charleskorn.kaml.Yaml
 import kotlinx.serialization.json.Json
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
-import org.liamjd.cantilever.api.models.APIResult
 import org.liamjd.cantilever.common.MimeType
-import org.liamjd.cantilever.common.S3_KEY
 import org.liamjd.cantilever.models.CantileverProject
 import org.liamjd.cantilever.models.ContentTree
-import org.liamjd.cantilever.routing.ResponseEntity
 import org.liamjd.cantilever.services.S3Service
 
-abstract class APIController(val sourceBucket: String) : KoinComponent {
+abstract class APIController(val sourceBucket: String, val generationBucket: String) : KoinComponent {
 
     val s3Service: S3Service by inject()
     val contentTree: ContentTree = ContentTree()
@@ -21,43 +18,52 @@ abstract class APIController(val sourceBucket: String) : KoinComponent {
     /**
      * Load the content tree from the S3 bucket
      */
-    fun loadContentTree(): Boolean {
-        if (s3Service.objectExists(S3_KEY.metadataKey, sourceBucket)) {
-            info("Reading metadata.json from bucket $sourceBucket")
-            contentTree.clear()
-            val metadata = s3Service.getObjectAsString(S3_KEY.metadataKey, sourceBucket)
-            val newTree = Json.decodeFromString(ContentTree.serializer(), metadata)
-            contentTree.items.addAll(newTree.items)
-            contentTree.templates.addAll(newTree.templates)
-            contentTree.statics.addAll(newTree.statics)
-            contentTree.images.addAll(newTree.images)
-            return true
-        } else {
-            warn("No '${S3_KEY.metadataKey}' file found in bucket $sourceBucket; please regenerate new empty tree")
+    fun loadContentTree(domain: String): Boolean {
+        val metadataKey = "$domain/metadata.json"
+        try {
+            if (s3Service.objectExists(metadataKey, generationBucket)) {
+                info("Reading $metadataKey from bucket $generationBucket")
+                contentTree.clear()
+                val metadata = s3Service.getObjectAsString(metadataKey, generationBucket)
+                val newTree = Json.decodeFromString(ContentTree.serializer(), metadata)
+                contentTree.items.addAll(newTree.items)
+                contentTree.templates.addAll(newTree.templates)
+                contentTree.statics.addAll(newTree.statics)
+                contentTree.images.addAll(newTree.images)
+                return true
+            } else {
+                warn("No '$metadataKey' file found in bucket $generationBucket; please regenerate new empty tree")
+                return false
+            }
+        } catch (e: Exception) {
+            error("Error reading $metadataKey from bucket $generationBucket: ${e.message}")
             return false
         }
     }
 
     /**
      * Save the content tree to the S3 bucket after a change
      */
-    fun saveContentTree() {
-        info("Saving content tree to bucket $sourceBucket")
+    fun saveContentTree(domain: String) {
+        val metadataKey =
+            "$domain/metadata.json"
+        info("Saving content tree $metadataKey to bucket $generationBucket")
         val json = Json { prettyPrint = true }
         val metadata = json.encodeToString(ContentTree.serializer(), contentTree)
-        s3Service.putObjectAsString(S3_KEY.metadataKey, sourceBucket, metadata, MimeType.json.toString())
+        s3Service.putObjectAsString(metadataKey, generationBucket, metadata, MimeType.json.toString())
     }
 
     /**
      * Load the project definition  'cantilever.yaml' from the S3 bucket
      */
-    fun loadProjectDefinition() {
-        if (s3Service.objectExists(S3_KEY.projectKey, sourceBucket)) {
-            info("Reading cantilever.yaml from bucket $sourceBucket")
-            val projectYaml = s3Service.getObjectAsString(S3_KEY.projectKey, sourceBucket)
+    fun loadProjectDefinition(domain: String) {
+        val projectKey = "$domain.yaml"
+        if (s3Service.objectExists(projectKey, sourceBucket)) {
+            info("Reading $projectKey from bucket $sourceBucket")
+            val projectYaml = s3Service.getObjectAsString(projectKey, sourceBucket)
             project = Yaml.default.decodeFromString(CantileverProject.serializer(), projectYaml)
         } else {
-            error("No '${S3_KEY.projectKey}' file found in bucket $sourceBucket!")
+            error("No '$projectKey' file found in bucket $sourceBucket!")
         }
     }
 

API/build.gradle.kts

@@ -20,7 +20,7 @@ import java.nio.charset.Charset
 /**
  * Handle routes relating to document generation. Mostly this will be done by sending messages to the appropriate queues.
  */
-class GeneratorController(sourceBucket: String) : KoinComponent, APIController(sourceBucket) {
+class GeneratorController(sourceBucket: String, generationBucket: String) : KoinComponent, APIController(sourceBucket, generationBucket) {
 
     companion object {
         const val error_NO_RESPONSE = "No response received for message"
@@ -40,16 +40,17 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
         val requestKey = request.pathParameters["srcKey"]
             ?: return ResponseEntity.badRequest(body = APIResult.Error("No srcKey provided"))
         var srcKey = ""
+        val projectKeyHeader = request.headers["cantilever-project-domain"]!!
         try {
-            loadContentTree()
+            loadContentTree(projectKeyHeader)
             if (requestKey == "*") {
                 info("GeneratorController: Received request to regenerate all pages")
                 val pages = contentTree.items.filterIsInstance<ContentNode.PageNode>()
                 var count = 0
                 if (pages.isNotEmpty()) {
                     pages.forEach { page ->
                         val sourceString = s3Service.getObjectAsString(page.srcKey, sourceBucket)
-                        val msgResponse = queuePageRegeneration(page.srcKey, sourceString)
+                        val msgResponse = queuePageRegeneration(projectKeyHeader, page.srcKey, sourceString)
                         if (msgResponse != null) {
                             count++
                         } else {
@@ -65,12 +66,12 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
                 srcKey = URLDecoder.decode(requestKey, Charset.defaultCharset())
                 info("GeneratorController: Received request to regenerate page '$srcKey'")
                 val sourceString = s3Service.getObjectAsString(srcKey, sourceBucket)
-                queuePageRegeneration(srcKey, sourceString)
+                queuePageRegeneration(projectKeyHeader, srcKey, sourceString)
                 return ResponseEntity.ok(body = APIResult.Success(value = "Regenerated page '$requestKey'"))
             }
         } catch (nske: NoSuchKeyException) {
             error("${nske.message} for key $srcKey")
-            return ResponseEntity.notFound(body = APIResult.Error(message = "Could not find page with key '$srcKey'"))
+            return ResponseEntity.notFound(body = APIResult.Error(statusText = "Could not find page with key '$srcKey'"))
         } catch (e: Exception) {
             error("Error generating page: ${e.message}")
             return ResponseEntity.serverError(body = APIResult.Error("Error generating page: ${e.message}"))
@@ -86,8 +87,9 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
     fun generatePost(request: Request<Unit>): ResponseEntity<APIResult<String>> {
         val requestKey = request.pathParameters["srcKey"]
         var srcKey = ""
+        val projectKeyHeader = request.headers["cantilever-project-domain"]!!
         try {
-            loadContentTree()
+            loadContentTree(projectKeyHeader)
             if (requestKey == "*") {
                 info("GeneratorController: Received request to regenerate all posts")
                 val posts = contentTree.items.filterIsInstance<ContentNode.PostNode>()
@@ -96,7 +98,11 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
                     posts.forEach { post ->
                         val sourceString = s3Service.getObjectAsString(post.srcKey, sourceBucket)
                         val postSrcKey = post.srcKey.removePrefix(postsPrefix)
-                        val msgResponse = queuePostRegeneration(postSrcKey, sourceString)
+                        val msgResponse = queuePostRegeneration(
+                            postSrcKey = postSrcKey,
+                            sourceString = sourceString,
+                            projectDomain = projectKeyHeader
+                        )
                         if (msgResponse != null) {
                             count++
                         } else {
@@ -113,13 +119,18 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
                 info("GeneratorController: Received request to regenerate post '$srcKey'")
                 val sourceString = s3Service.getObjectAsString(srcKey, sourceBucket)
                 val postSrcKey = srcKey.removePrefix(postsPrefix)
-                queuePostRegeneration(postSrcKey, sourceString)
+                queuePostRegeneration(
+                    postSrcKey = postSrcKey,
+                    sourceString = sourceString,
+                    projectDomain = projectKeyHeader
+                )
                 return ResponseEntity.ok(body = APIResult.Success(value = "Regenerated post '$requestKey'"))
             }
         } catch (nske: NoSuchKeyException) {
             error("${nske.message} for key $srcKey")
-            return ResponseEntity.notFound(body = APIResult.Error(message = "Could not find post with key '$srcKey'"))
+            return ResponseEntity.notFound(body = APIResult.Error(statusText = "Could not find post with key '$srcKey'"))
         } catch (e: Exception) {
+            error("Error generating post: ${e.javaClass}: ${e.message}")
             return ResponseEntity.serverError(body = APIResult.Error("Error generating post: ${e.message}"))
         }
     }
@@ -136,23 +147,30 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
                 body = APIResult.Error("Regeneration of all templates is not supported.")
             )
         }
-        info("ENCODED: GeneratorController received request to regenerate pages based on template '$requestKey'")
+        val projectKeyHeader = request.headers["cantilever-project-domain"]!!
         // TODO: https://github.com/v79/Cantilever/issues/26 this only works for HTML handlebars templates, i.e. those whose file names end in ".index.html" in the "templates" folder.
         // Also, annoying that I have to double-decode this.
+        // And I need to strip off the domain from the requestKey
         val templateKey =
-            URLDecoder.decode(URLDecoder.decode(requestKey, Charset.defaultCharset()), Charset.defaultCharset())
+            URLDecoder.decode(URLDecoder.decode(requestKey, Charset.defaultCharset()), Charset.defaultCharset()).substringAfter(
+                "$projectKeyHeader/"
+            )
         info(
-            "DOUBLE DECODED: GeneratorController received request to regenerate pages based on template '$templateKey'"
+            "Received request to regenerate pages based on template '$templateKey'"
         )
         var count = 0
 
         // We don't know if the template is for a Page or a Post. This is less than ideal as I have to check both. But I could short-circuit the second check if the first one succeeds?
         try {
-            loadContentTree()
+            loadContentTree(projectKeyHeader)
             contentTree.getPagesForTemplate(templateKey).forEach { page ->
                 info("Regenerating page ${page.srcKey} because it has template ${page.templateKey}")
                 val pageSource = s3Service.getObjectAsString(page.srcKey, sourceBucket)
-                val response = queuePageRegeneration(page.srcKey, pageSource)
+                val response = queuePageRegeneration(
+                    pageSrcKey = page.srcKey,
+                    sourceString = pageSource,
+                    projectDomain = projectKeyHeader
+                )
                 if (response != null) {
                     count++
                 } else {
@@ -163,7 +181,11 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
             contentTree.getPostsForTemplate(templateKey).forEach { post ->
                 info("Regenerating post ${post.srcKey} because it has template ${post.templateKey}")
                 val postSource = s3Service.getObjectAsString(post.srcKey, sourceBucket)
-                val msgResponse = queuePostRegeneration(post.srcKey, postSource)
+                val msgResponse = queuePostRegeneration(
+                    postSrcKey = post.srcKey,
+                    sourceString = postSource,
+                    projectDomain = projectKeyHeader
+                )
                 if (msgResponse != null) {
                     count++
                 } else {
@@ -194,10 +216,18 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
     /**
      * Send a message to the markdown queue for a Page
      */
-    private fun queuePageRegeneration(pageSrcKey: String, sourceString: String): SendMessageResponse? {
+    private fun queuePageRegeneration(
+        pageSrcKey: String,
+        sourceString: String,
+        projectDomain: String
+    ): SendMessageResponse? {
         // extract page model
         val pageMode = ContentMetaDataBuilder.PageBuilder.buildFromSourceString(sourceString, pageSrcKey)
-        val msgBody = MarkdownSQSMessage.PageUploadMsg(pageMode, sourceString)
+        val msgBody = MarkdownSQSMessage.PageUploadMsg(
+            projectDomain = projectDomain,
+            metadata = pageMode,
+            markdownText = sourceString
+        )
         return sqsService.sendMarkdownMessage(
             toQueue = markdownQueue, body = msgBody,
             messageAttributes = createStringAttribute("sourceType", SOURCE_TYPE.Pages.folder)
@@ -207,14 +237,22 @@ class GeneratorController(sourceBucket: String) : KoinComponent, APIController(s
     /**
      * Send a message to the markdown queue for a Post
      */
-    private fun queuePostRegeneration(postSrcKey: String, sourceString: String): SendMessageResponse? {
+    private fun queuePostRegeneration(
+        postSrcKey: String,
+        sourceString: String,
+        projectDomain: String
+    ): SendMessageResponse? {
         // extract post model
         val metadata =
             ContentMetaDataBuilder.PostBuilder.buildFromSourceString(sourceString.getFrontMatter(), postSrcKey)
         // extract body
         val markdownBody = sourceString.stripFrontMatter()
 
-        val msgBody = MarkdownSQSMessage.PostUploadMsg(metadata, markdownBody)
+        val msgBody = MarkdownSQSMessage.PostUploadMsg(
+            projectDomain = projectDomain,
+            metadata = metadata,
+            markdownText = markdownBody
+        )
         return sqsService.sendMarkdownMessage(
             toQueue = markdownQueue, body = msgBody,
             messageAttributes = createStringAttribute("sourceType", SOURCE_TYPE.Posts.folder)