Skip to content

Commit

Permalink
Add api linter gradle plugin (#44)
Browse files Browse the repository at this point in the history
* Add api linter gradle plugin

* Optimized code

* Optimize code

* Optimize code

Co-authored-by: Kanro <[email protected]>
  • Loading branch information
GuoDuanLZ and devkanro authored Jul 13, 2020
1 parent ba3aaab commit 9cf8b72
Show file tree
Hide file tree
Showing 11 changed files with 204 additions and 13 deletions.
1 change: 1 addition & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ include("lib:sisyphus-grpc")
include("tools:sisyphus-protoc")
include("tools:sisyphus-project-gradle-plugin")
include("tools:sisyphus-protobuf-gradle-plugin")
include("tools:sisyphus-api-linter-runner")

include("middleware:sisyphus-configuration-artifact")
include("middleware:sisyphus-jdbc")
Expand Down
9 changes: 9 additions & 0 deletions tools/sisyphus-api-linter-runner/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
tools

plugins {
`java-library`
`java-gradle-plugin`
id("com.gradle.plugin-publish")
}

description = "Runner and executable manager for Google API Linter on java"
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package com.bybutter.sisyphus.apilinter

import java.io.BufferedReader
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.attribute.PosixFilePermission

class ApiLinterRunner {

fun runApiLinter(args: List<String>, version: String): String? {
val apiLinterTemp = extractApiLinter(version)
return executeCmd(apiLinterTemp.toString(), args)
}

private fun executeCmd(cmd: String, args: List<String>): String {
val apiLinterCmd = mutableListOf(cmd)
for (arg in args) {
apiLinterCmd.add(arg)
}
println("api-linter executing: $apiLinterCmd")
val process = ProcessBuilder(apiLinterCmd).start()
val result = process.text()
process.waitFor()
return result
}

private fun extractApiLinter(version: String): Path {
val osName = System.getProperties().getProperty("os.name").normalize()
val platform = detectPlatform(osName)
val srcFilePath = Paths.get(version, "api-linter-$version-$platform.exe").toString()
val srcFile = this.javaClass.classLoader.getResource(srcFilePath)
?: throw UnsupportedOperationException("Unsupported api linter version $version or platform $osName.")
val executable = createTempBinDir().resolve("apilinter.exe")
srcFile.openStream().use {
Files.copy(it, executable)
}
Files.setPosixFilePermissions(executable, setOf(PosixFilePermission.OWNER_EXECUTE))
return executable.also {
it.toFile().deleteOnExit()
}
}

private fun detectPlatform(osName: String): String {
return when {
(osName.startsWith("macosx") || osName.startsWith("osx")) -> "darwin"
osName.startsWith("linux") -> "linux"
osName.startsWith("windows") -> "windows"
else -> "unknown"
}
}

private fun String.normalize(): String {
return this.toLowerCase().replace("[^a-z0-9]+".toRegex(), "")
}

private fun createTempBinDir(): Path {
return Files.createTempDirectory("apilinterrun").also {
it.toFile().deleteOnExit()
}
}

private fun Process.text(): String {
return this.inputStream.bufferedReader().use(BufferedReader::readText)
}

companion object {
const val API_LINTER_DEFAULT_VERSION = "1.1.0"
}
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
1 change: 1 addition & 0 deletions tools/sisyphus-protobuf-gradle-plugin/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ dependencies {
implementation(project(":lib:sisyphus-grpc"))
implementation(project(":lib:sisyphus-jackson"))
implementation(project(":tools:sisyphus-protoc"))
implementation(project(":tools:sisyphus-api-linter-runner"))

implementation(Dependencies.Kotlin.reflect)
implementation(Dependencies.Kotlin.plugin)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.bybutter.sisyphus.protobuf.gradle
import com.bybutter.sisyphus.io.toUnixPath
import com.bybutter.sisyphus.protobuf.compiler.ProtocRunner
import java.io.File
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.FileVisitResult
import java.nio.file.Files
Expand Down Expand Up @@ -39,6 +40,7 @@ open class ExtractProtoTask : SourceTask() {

private val scannedMapping = mutableMapOf<String, String>()
private val sourceProtos = mutableSetOf<String>()
private val sourceFileMapping = mutableMapOf<String, String>()

private fun addProto(file: File) {
addProto(file.toPath())
Expand All @@ -65,7 +67,7 @@ open class ExtractProtoTask : SourceTask() {
Files.walkFileTree(dir, object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
if (file.fileName.toString().endsWith(".proto")) {
addProtoInternal(dir.relativize(file).toString(), Files.readAllBytes(file), source)
addProtoInternal(dir.relativize(file).toString(), Files.readAllBytes(file), file, source)
}
if (file.endsWith("protomap")) {
scannedMapping += Files.readAllLines(file).mapNotNull {
Expand All @@ -82,9 +84,11 @@ open class ExtractProtoTask : SourceTask() {
})
}

private fun addProtoInternal(name: String, value: ByteArray, source: Boolean) {
private fun addProtoInternal(name: String, value: ByteArray, file: Path, source: Boolean) {
if (source) {
sourceProtos.add(name.toUnixPath())
val protoName = name.toUnixPath()
sourceProtos.add(protoName)
sourceFileMapping[protoName] = URI(file.toUri().toURL().path).path
}
val targetFile = protoPath.toPath().resolve(name)
Files.createDirectories(targetFile.parent)
Expand Down Expand Up @@ -133,6 +137,7 @@ open class ExtractProtoTask : SourceTask() {
Files.write(Paths.get(protoPath.toPath().toString(), "protodesc.pb"), desc.toByteArray())
Files.write(Paths.get(protoPath.toPath().toString(), "protomap"), scannedMapping.map { "${it.key}=${it.value}" })
Files.write(Paths.get(protoPath.toPath().toString(), "protosrc"), sourceProtos)
Files.write(Paths.get(protoPath.toPath().toString(), "protofile"), sourceFileMapping.map { "${it.key}=${it.value}" })

val enableServices = protobuf.service?.apis?.map { it.name }?.toSet()
val releaseProtos = mutableSetOf<String>()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,89 @@
package com.bybutter.sisyphus.protobuf.gradle

import com.bybutter.sisyphus.apilinter.ApiLinterRunner
import com.bybutter.sisyphus.jackson.parseJson
import com.fasterxml.jackson.annotation.JsonProperty
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.SourceTask
import org.gradle.api.tasks.TaskAction

open class ProtobufApiLintTask : SourceTask()
open class ProtobufApiLintTask : SourceTask() {

@get:InputDirectory
lateinit var protoPath: File

@get:Internal
lateinit var protobuf: ProtobufExtension

@TaskAction
fun apiLinter() {
val cmd = mutableListOf<String>()
val apiLinterConfig = protobuf.linter
ruleHandle(cmd, apiLinterConfig)
val path = protoPath.toPath().toString()
cmd.addAll(listOf(OUTPUT_FORMAT, "json", PROTO_PATH, path))
val excludeFiles = apiLinterConfig.excludeFiles
Paths.get(path, "protosrc").toFile().bufferedReader().forEachLine {
if (excludeFiles.isEmpty() || !excludeFiles.contains(it)) {
cmd.add(it)
}
}
val protoPathMapping = mutableMapOf<String, String>()
Paths.get(path, "protofile").toFile().bufferedReader().forEachLine {
val mapping = it.split("=", limit = 2)
protoPathMapping[mapping[0]] = mapping[1]
}
val version = apiLinterConfig.version ?: ApiLinterRunner.API_LINTER_DEFAULT_VERSION
val message = ApiLinterRunner().runApiLinter(cmd, version) ?: return
printMessage(message.parseJson(), protoPathMapping)
outputMessageToFile(message)
}

private fun outputMessageToFile(message: String) {
val outPutDirectory = project.layout.buildDirectory.file(project.provider {
"reports/apilinter"
}).get().asFile
if (!outPutDirectory.exists()) Files.createDirectories(outPutDirectory.toPath())
Files.write(Paths.get(outPutDirectory.toPath().toString(), "apilinter.json"), message.toByteArray())
}

private fun ruleHandle(cmd: MutableList<String>, apiLinterConfig: ApiLinterConfig) {
val enableRules = apiLinterConfig.enableRules
if (enableRules.isNotEmpty()) {
cmd.add(ENABLE_RULE)
enableRules.forEach { cmd.add(it) }
}
val disableRules = apiLinterConfig.disableRules
if (disableRules.isNotEmpty()) {
cmd.add(DISABLE_RULE)
disableRules.forEach { cmd.add(it) }
}
}

private fun printMessage(linterResponseList: List<LinterResponse>, protoPathMapping: Map<String, String>) {
for (linterResponse in linterResponseList) {
linterResponse.problems.forEach {
println("api-linter: ${protoPathMapping[linterResponse.filePath]}:${it.location.startPosition.lineNumber}:${it.location.startPosition.columnNumber} ${it.message} rule detail in ${it.ruleDocUri}")
}
}
}

companion object {
private const val DISABLE_RULE = "--disable-rule"
private const val ENABLE_RULE = "--enable-rule"
private const val PROTO_PATH = "--proto-path"
private const val OUTPUT_FORMAT = "--output-format"
}
}

data class LinterResponse(@JsonProperty("file_path") var filePath: String, @JsonProperty("problems") var problems: List<Problem>)

data class Problem(var message: String, var location: Location, @JsonProperty("rule_id") var ruleId: String, @JsonProperty("rule_doc_uri") var ruleDocUri: String)

data class Location(@JsonProperty("start_position") val startPosition: FilePosition, @JsonProperty("end_position") val endPosition: FilePosition)

data class FilePosition(@JsonProperty("line_number") var lineNumber: Int, @JsonProperty("column_number") var columnNumber: Int)
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ open class ProtobufExtension {

val service get() = serviceConfig

val linter = ApiLinterConfig()

var autoGenerating = true

fun sourceSet(name: String, block: ProtoGeneratingConfig.() -> Unit = {}): ProtoGeneratingConfig {
Expand All @@ -25,6 +27,10 @@ open class ProtobufExtension {
serviceConfig = serviceConfig?.invoke(block) ?: Service(block)
}

fun linter(block: ApiLinterConfig.() -> Unit) {
linter.block()
}

fun packageMapping(proto: String, kotlin: String) {
packageMapping[proto] = kotlin
}
Expand All @@ -34,4 +40,6 @@ open class ProtobufExtension {
}
}

data class ApiLinterConfig(var version: String? = null, val enableRules: MutableSet<String> = mutableSetOf(), val disableRules: MutableSet<String> = mutableSetOf(), val excludeFiles: MutableSet<String> = mutableSetOf())

data class ProtoGeneratingConfig(var inputDir: String? = null, var outputDir: String? = null, var implDir: String? = null, var resourceOutputDir: String? = null)
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class ProtobufPlugin : Plugin<Project> {
val generateTask = registerGenerateProto(target, extension, sourceSet, extractTask)
registerPackageProto(target, extension, sourceSet, extractTask)
registerApiCompileProto(target, extension, sourceSet, extractTask)
registerApiLintProto(target, extension, sourceSet, extractTask)
generateProtos.dependsOn(generateTask)

target.afterEvaluate {
Expand Down Expand Up @@ -82,11 +83,11 @@ class ProtobufPlugin : Plugin<Project> {

private fun registerExtractProto(target: Project, extension: ProtobufExtension, sourceSet: SourceSet): ExtractProtoTask {
val inputDir = target.file(extension.sourceSet(sourceSet.name).inputDir
?: sourceSet.protoSourcePath)
?: sourceSet.protoSourcePath)
val resourceOutputDir = target.file(extension.sourceSet(sourceSet.name).resourceOutputDir
?: sourceSet.protoResourceCompileOutputPath)
?: sourceSet.protoResourceCompileOutputPath)
val protoDir = target.file(extension.sourceSet(sourceSet.name).resourceOutputDir
?: sourceSet.protoTempCompileOutputPath)
?: sourceSet.protoTempCompileOutputPath)

Files.createDirectories(resourceOutputDir.toPath())
Files.createDirectories(protoDir.toPath())
Expand All @@ -99,7 +100,7 @@ class ProtobufPlugin : Plugin<Project> {
return target.tasks.register("extract ${sourceSet.name} protos".toCamelCase(), ExtractProtoTask::class.java) {
if (sourceSet.isTestSourceSet) {
val mainInputDir = target.file(extension.sourceSet("main").inputDir
?: target.sourceSets.main!!.protoSourcePath)
?: target.sourceSets.main!!.protoSourcePath)
it.input = target.layout.files(inputDir, mainInputDir)
} else {
it.input = target.layout.files(inputDir)
Expand All @@ -121,11 +122,11 @@ class ProtobufPlugin : Plugin<Project> {

private fun registerGenerateProto(target: Project, extension: ProtobufExtension, sourceSet: SourceSet, extractTask: ExtractProtoTask): ProtoGenerateTask {
val outputDir = target.file(extension.sourceSet(sourceSet.name).outputDir
?: sourceSet.protoCompileOutputPath)
?: sourceSet.protoCompileOutputPath)
val implOutputDir = target.file(extension.sourceSet(sourceSet.name).implDir
?: sourceSet.protoInternalCompileOutputPath)
?: sourceSet.protoInternalCompileOutputPath)
val resourceOutputDir = target.file(extension.sourceSet(sourceSet.name).resourceOutputDir
?: sourceSet.protoResourceCompileOutputPath)
?: sourceSet.protoResourceCompileOutputPath)

Files.createDirectories(outputDir.toPath())
Files.createDirectories(implOutputDir.toPath())
Expand Down Expand Up @@ -171,7 +172,7 @@ class ProtobufPlugin : Plugin<Project> {
if (!sourceSet.isMainSourceSet) return null

val protoDir = target.file(extension.sourceSet(sourceSet.name).resourceOutputDir
?: sourceSet.protoTempCompileOutputPath)
?: sourceSet.protoTempCompileOutputPath)

return target.tasks.register("protoZip", Zip::class.java) {
it.from(protoDir)
Expand All @@ -183,7 +184,7 @@ class ProtobufPlugin : Plugin<Project> {

private fun registerApiCompileProto(target: Project, extension: ProtobufExtension, sourceSet: SourceSet, extractTask: ExtractProtoTask): ProtobufApiCompileTask {
val resourceOutputDir = File(extension.sourceSet(sourceSet.name).resourceOutputDir
?: "${target.buildDir}/generated/resources/proto-meta/${sourceSet.name}")
?: "${target.buildDir}/generated/resources/proto-meta/${sourceSet.name}")

Files.createDirectories(resourceOutputDir.toPath())

Expand All @@ -199,6 +200,18 @@ class ProtobufPlugin : Plugin<Project> {
}.get()
}

private fun registerApiLintProto(target: Project, extension: ProtobufExtension, sourceSet: SourceSet, extractTask: ExtractProtoTask): ProtobufApiLintTask {
return target.tasks.register("${sourceSet.name} api lint".toCamelCase(), ProtobufApiLintTask::class.java) {
it.protoPath = extractTask.protoPath
it.protobuf = extension
it.group = "api"
it.description = "Apilint for '${sourceSet.name}' source set."

it.source(extractTask.protoPath)
it.dependsOn(extractTask)
}.get()
}

private fun configTest(target: Project) {
val testSourceSet = target.sourceSets.test ?: return
val mainSourceSet = target.sourceSets.main ?: return
Expand Down

0 comments on commit 9cf8b72

Please sign in to comment.