Fuzz impl v2

build.gradle.kts

@@ -2,6 +2,7 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
+
 buildscript {
     repositories {
         mavenCentral()
@@ -14,6 +15,7 @@ buildscript {
     }
 }
 
+
 allprojects {
     val allowLocalDeps: String by project
     repositories {
@@ -23,8 +25,10 @@ allprojects {
         mavenCentral()
         google()
     }
+
 }
 
+
 val ktlint by configurations.creating
 val ktlintVersion: String by project
 

codegen-client/build.gradle.kts

@@ -18,6 +18,7 @@ group = "software.amazon.smithy.rust.codegen"
 version = "0.1.0"
 
 val smithyVersion: String by project
+val smithyRsVersion: String by project
 
 dependencies {
     implementation(project(":codegen-core"))

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/generators/CargoTomlGenerator.kt

@@ -113,6 +113,7 @@ class CargoTomlGenerator(
                 "features" to cargoFeatures.toMap(),
             ).deepMergeWith(manifestCustomizations)
 
+        println(cargoToml)
         writer.writeWithNoFormatting(TomlWriter().write(cargoToml))
     }
 }

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/testutil/Rust.kt

@@ -360,6 +360,7 @@ fun RustCrate.testModule(block: Writable) =
     }
 
 fun FileManifest.printGeneratedFiles() {
+    println("Generated files:")
     this.files.forEach { path ->
         println("file:///$path")
     }

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/util/Exec.kt

@@ -16,15 +16,16 @@ fun String.runCommand(
     workdir: Path? = null,
     environment: Map<String, String> = mapOf(),
     timeout: Long = 3600,
+    redirect: ProcessBuilder.Redirect = ProcessBuilder.Redirect.PIPE,
 ): String {
     val logger = Logger.getLogger("RunCommand")
     logger.fine("Invoking comment $this in `$workdir` with env $environment")
     val start = System.currentTimeMillis()
     val parts = this.split("\\s".toRegex())
     val builder =
         ProcessBuilder(*parts.toTypedArray())
-            .redirectOutput(ProcessBuilder.Redirect.PIPE)
-            .redirectError(ProcessBuilder.Redirect.PIPE)
+            .redirectOutput(redirect)
+            .redirectError(redirect)
             .letIf(workdir != null) {
                 it.directory(workdir?.toFile())
             }

fuzzgen/build.gradle.kts

@@ -0,0 +1,111 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+
+plugins {
+    kotlin("jvm")
+    `maven-publish`
+}
+
+description = "Plugin to generate a fuzz harness"
+extra["displayName"] = "Smithy :: Rust :: Fuzzer Generation"
+extra["moduleName"] = "software.amazon.smithy.rust.codegen.client"
+
+group = "software.amazon.smithy.rust.codegen.serde"
+version = "0.1.0"
+
+val smithyVersion: String by project
+
+dependencies {
+    implementation(project(":codegen-core"))
+    implementation(project(":codegen-client"))
+    implementation(project(":codegen-server"))
+    implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion")
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+}
+
+tasks.compileKotlin {
+    kotlinOptions.jvmTarget = "11"
+}
+
+// Reusable license copySpec
+val licenseSpec = copySpec {
+    from("${project.rootDir}/LICENSE")
+    from("${project.rootDir}/NOTICE")
+}
+
+// Configure jars to include license related info
+tasks.jar {
+    metaInf.with(licenseSpec)
+    inputs.property("moduleName", project.name)
+    manifest {
+        attributes["Automatic-Module-Name"] = project.name
+    }
+}
+
+val sourcesJar by tasks.creating(Jar::class) {
+    group = "publishing"
+    description = "Assembles Kotlin sources jar"
+    archiveClassifier.set("sources")
+    from(sourceSets.getByName("main").allSource)
+}
+
+val isTestingEnabled: String by project
+if (isTestingEnabled.toBoolean()) {
+    val kotestVersion: String by project
+
+    dependencies {
+        runtimeOnly(project(":rust-runtime"))
+        testImplementation("org.junit.jupiter:junit-jupiter:5.6.1")
+        testImplementation("software.amazon.smithy:smithy-validation-model:$smithyVersion")
+        testImplementation("software.amazon.smithy:smithy-aws-protocol-tests:$smithyVersion")
+        testImplementation("io.kotest:kotest-assertions-core-jvm:$kotestVersion")
+    }
+
+    tasks.compileTestKotlin {
+        kotlinOptions.jvmTarget = "11"
+    }
+
+    tasks.register("generateClasspath") {
+        doLast {
+            // Get the runtime classpath
+            val runtimeClasspath = sourceSets["main"].runtimeClasspath
+
+            // Add the 'libs' directory to the classpath
+            val libsDir = file(layout.buildDirectory.dir("libs"))
+            val fullClasspath = runtimeClasspath + files(libsDir.listFiles())
+
+            // Convert to classpath string
+            val classpath = fullClasspath.asPath
+        }
+    }
+
+
+    tasks.test {
+        useJUnitPlatform()
+        testLogging {
+            events("failed")
+            exceptionFormat = TestExceptionFormat.FULL
+            showCauses = true
+            showExceptions = true
+            showStackTraces = true
+        }
+    }
+}
+
+publishing {
+    publications {
+        create<MavenPublication>("default") {
+            from(components["java"])
+            artifact(sourcesJar)
+        }
+    }
+    repositories { maven { url = uri(layout.buildDirectory.dir("repository")) } }
+}

fuzzgen/src/main/kotlin/software/amazon/smithy/rust/codegen/fuzz/FuzzHarnessBuildPlugin.kt

@@ -0,0 +1,212 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.rust.codegen.fuzz
+
+import software.amazon.smithy.build.FileManifest
+import software.amazon.smithy.build.PluginContext
+import software.amazon.smithy.build.SmithyBuildPlugin
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.knowledge.TopDownIndex
+import software.amazon.smithy.model.neighbor.Walker
+import software.amazon.smithy.model.node.ArrayNode
+import software.amazon.smithy.model.node.Node
+import software.amazon.smithy.model.node.NumberNode
+import software.amazon.smithy.model.node.ObjectNode
+import software.amazon.smithy.model.node.StringNode
+import software.amazon.smithy.model.shapes.MemberShape
+import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.model.shapes.Shape
+import software.amazon.smithy.model.shapes.ShapeId
+import software.amazon.smithy.model.traits.HttpPrefixHeadersTrait
+import software.amazon.smithy.model.traits.HttpQueryTrait
+import software.amazon.smithy.model.traits.HttpTrait
+import software.amazon.smithy.model.traits.JsonNameTrait
+import software.amazon.smithy.model.traits.XmlNameTrait
+import software.amazon.smithy.protocoltests.traits.HttpRequestTestsTrait
+import software.amazon.smithy.rust.codegen.core.rustlang.RustModule
+import software.amazon.smithy.rust.codegen.core.rustlang.Writable
+import software.amazon.smithy.rust.codegen.core.smithy.ModuleDocProvider
+import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig
+import software.amazon.smithy.rust.codegen.core.smithy.transformers.OperationNormalizer
+import software.amazon.smithy.rust.codegen.core.util.getTrait
+import software.amazon.smithy.rust.codegen.core.util.orNull
+import software.amazon.smithy.rust.codegen.server.smithy.transformers.AttachValidationExceptionToConstrainedOperationInputsInAllowList
+import java.nio.file.Path
+import java.util.Base64
+import kotlin.streams.toList
+
+/**
+ * Metadata for a TargetCrate: A code generated smithy-rs server for a given model
+ */
+data class TargetCrate(
+    /** The name of the Fuzz target */
+    val name: String,
+    /** Where the server implementation of this target is */
+    val relativePath: String,
+) {
+    companion object {
+        fun fromNode(node: ObjectNode): TargetCrate {
+            val name = node.expectStringMember("name").value
+            val relativePath = node.expectStringMember("relativePath").value
+            return TargetCrate(name, relativePath)
+        }
+    }
+
+    /** The name of the actual `package` from Cargo's perspective.
+     *
+     *  We need this to make a dependency on it
+     * */
+    fun targetPackage(): String {
+        val path = Path.of(relativePath)
+        val cargoToml = path.resolve("Cargo.toml").toFile()
+        val packageSection = cargoToml.readLines().dropWhile { it.trim() != "[package]" }
+        return packageSection.firstOrNull { it.startsWith("name =") }?.let { it.split("=")[1].trim() }?.trim('"')
+            ?: throw Exception("no package name")
+    }
+}
+
+data class FuzzSettings(
+    val targetServers: List<TargetCrate>,
+    val service: ShapeId,
+    val runtimeConfig: RuntimeConfig,
+) {
+    companion object {
+        fun fromNode(node: ObjectNode): FuzzSettings {
+            val targetCrates =
+                node.expectArrayMember("targetCrates")
+                    .map { TargetCrate.fromNode(it.expectObjectNode()) }
+            val service = ShapeId.fromNode(node.expectStringMember("service"))
+            val runtimeConfig = RuntimeConfig.fromNode(node.getObjectMember("runtimeConfig"))
+            return FuzzSettings(targetCrates, service, runtimeConfig)
+        }
+    }
+}
+
+/**
+ * Build plugin for generating a fuzz harness and lexicon from a smithy model and a set of smithy-rs versions
+ *
+ * This is used by `aws-smithy-fuzz` which contains most of the usage docs
+ */
+class FuzzHarnessBuildPlugin : SmithyBuildPlugin {
+    override fun getName(): String = "fuzz-harness"
+
+    override fun execute(context: PluginContext) {
+        val fuzzSettings = FuzzSettings.fromNode(context.settings)
+
+        val model =
+            context.model.let(OperationNormalizer::transform)
+                .let(AttachValidationExceptionToConstrainedOperationInputsInAllowList::transform)
+        val targets =
+            fuzzSettings.targetServers.map { target ->
+                val targetContext = createFuzzTarget(target, context.fileManifest, fuzzSettings, model)
+                println("Creating a fuzz targret for $targetContext")
+                FuzzTargetGenerator(targetContext).generateFuzzTarget()
+                targetContext
+            }
+
+        println("creating the driver...")
+        createDriver(model, context.fileManifest, fuzzSettings)
+
+        targets.forEach {
+            context.fileManifest.addAllFiles(it.finalize())
+        }
+    }
+}
+
+/**
+ * Generate a corpus of words used within the model to see the dictionary
+ */
+fun corpus(
+    model: Model,
+    fuzzSettings: FuzzSettings,
+): ArrayNode {
+    val operations = TopDownIndex.of(model).getContainedOperations(fuzzSettings.service)
+    val protocolTests = operations.flatMap { it.getTrait<HttpRequestTestsTrait>()?.testCases ?: listOf() }
+    val out = ArrayNode.builder()
+    protocolTests.forEach { testCase ->
+        val body: List<NumberNode> =
+            when (testCase.bodyMediaType.orNull()) {
+                "application/cbor" -> {
+                    println("base64 decoding first (v2)")
+                    Base64.getDecoder().decode(testCase.body.orNull())?.map { NumberNode.from(it.toUByte().toInt()) }
+                }
+
+                else -> testCase.body.orNull()?.chars()?.toList()?.map { c -> NumberNode.from(c) }
+            } ?: listOf()
+        out.withValue(
+            ObjectNode.objectNode()
+                .withMember("uri", testCase.uri)
+                .withMember("method", testCase.method)
+                .withMember(
+                    "headers",
+                    ObjectNode.objectNode(
+                        testCase.headers.map { (k, v) ->
+                            (StringNode.from(k) to ArrayNode.fromStrings(v))
+                        }.toMap(),
+                    ),
+                )
+                .withMember("trailers", ObjectNode.objectNode())
+                .withMember(
+                    "body",
+                    ArrayNode.fromNodes(body),
+                ),
+        )
+    }
+    return out.build()
+}
+
+fun createDriver(
+    model: Model,
+    baseManifest: FileManifest,
+    fuzzSettings: FuzzSettings,
+) {
+    val fuzzLexicon =
+        ObjectNode.objectNode()
+            .withMember("corpus", corpus(model, fuzzSettings))
+            .withMember("dictionary", dictionary(model, fuzzSettings))
+    baseManifest.writeFile("lexicon.json", Node.prettyPrintJson(fuzzLexicon))
+}
+
+fun dictionary(
+    model: Model,
+    fuzzSettings: FuzzSettings,
+): ArrayNode {
+    val operations = TopDownIndex.of(model).getContainedOperations(fuzzSettings.service)
+    val walker = Walker(model)
+    val dictionary = mutableSetOf<String>()
+    operations.forEach {
+        walker.iterateShapes(it).forEach { shape ->
+            dictionary.addAll(getTraitBasedNames(shape))
+            dictionary.add(shape.id.name)
+            when (shape) {
+                is MemberShape -> dictionary.add(shape.memberName)
+                is OperationShape -> dictionary.add(shape.id.toString())
+                else -> {}
+            }
+        }
+    }
+    return ArrayNode.fromStrings(dictionary.toList().sorted())
+}
+
+fun getTraitBasedNames(shape: Shape): List<String> {
+    return listOfNotNull(
+        shape.getTrait<JsonNameTrait>()?.value,
+        shape.getTrait<XmlNameTrait>()?.value,
+        shape.getTrait<HttpQueryTrait>()?.value,
+        shape.getTrait<HttpTrait>()?.method,
+        *(
+            shape.getTrait<HttpTrait>()?.uri?.queryLiterals?.flatMap { (k, v) -> listOf(k, v) }
+                ?: listOf()
+        ).toTypedArray(),
+        shape.getTrait<HttpPrefixHeadersTrait>()?.value,
+    )
+}
+
+class NoOpDocProvider : ModuleDocProvider {
+    override fun docsWriter(module: RustModule.LeafModule): Writable? {
+        return null
+    }
+}