Output Coverage Metadata

Add two flags to `main` to output metadata related to coverage, one for all fault points and one for fault points that are covered by the output test vectors.
AUCOHL · Jan 14, 2024 · 23fb51a · 23fb51a
1 parent a1a68af
commit 23fb51a
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 4 deletions.
diff --git a/Sources/Fault/Entries/main.swift b/Sources/Fault/Entries/main.swift
@@ -19,6 +19,7 @@ import CoreFoundation // Not automatically imported on Linux
 import Defile
 import Foundation
 import PythonKit
+import Yams
 
 let VERSION = "0.6.1"
 
@@ -116,6 +117,18 @@ func main(arguments: [String]) -> Int32 {
     )
     cli.addOptions(svfFilePath)
 
+    let faultPointFilePath = StringOption(
+        longFlag: "output-fault-points",
+        helpMessage: "Path to the output yml file listing all generated fault points. (Default: nil)"
+    )
+    cli.addOptions(faultPointFilePath)
+
+    let coverageMetaFilePath = StringOption(
+        longFlag: "output-covered",
+        helpMessage: "Path to the output yml file listing coverage metadata, i.e., ratio and fault points covered. (Default: nil)"
+    )
+    cli.addOptions(coverageMetaFilePath)
+
     let cellsOption = StringOption(
         shortFlag: "c",
         longFlag: "cellModel",
@@ -477,7 +490,7 @@ func main(arguments: [String]) -> Int32 {
         let timeElapsed = CFAbsoluteTimeGetCurrent() - startTime
         print("Time elapsed: \(String(format: "%.2f", timeElapsed))s.")
 
-        print("Simulations concluded: Coverage \(result.coverage * 100)%")
+        print("Simulations concluded: Coverage \(result.coverageMeta.ratio * 100)%")
 
         let encoder = JSONEncoder()
         encoder.outputFormatting = .prettyPrinted
@@ -504,6 +517,18 @@ func main(arguments: [String]) -> Int32 {
             try $0.print(svfString)
         }
 
+        if let faultPointOutput = faultPointFilePath.value {
+            try File.open(faultPointOutput, mode: .write) {
+                try $0.write(string: YAMLEncoder().encode(faultPoints.sorted()))
+            }
+        }
+
+        if let coverageMetaFilePath = coverageMetaFilePath.value {
+            try File.open(coverageMetaFilePath, mode: .write) {
+                try $0.write(string: YAMLEncoder().encode(result.coverageMeta))
+            }
+        }
+
     } catch {
         Stderr.print("Internal error: \(error)")
         return EX_SOFTWARE

diff --git a/Sources/Fault/Simulation.swift b/Sources/Fault/Simulation.swift
@@ -17,6 +17,18 @@ import Defile
 import Foundation
 import PythonKit
 
+class CoverageMeta: Codable {
+    let ratio: Float
+    let sa0Covered: [String]
+    let sa1Covered: [String]
+
+    init(ratio: Float, sa0Covered: [String], sa1Covered: [String]) {
+        self.ratio = ratio
+        self.sa0Covered = sa0Covered
+        self.sa1Covered = sa1Covered
+    }
+}
+
 enum Simulator {
     enum Behavior: Int {
         case holdHigh = 1
@@ -226,7 +238,7 @@ enum Simulator {
         defines: Set<String> = [],
         using iverilogExecutable: String,
         with vvpExecutable: String
-    ) throws -> (coverageList: [TVCPair], coverage: Float) {
+    ) throws -> (coverageList: [TVCPair], coverageMeta: CoverageMeta) {
         var testVectorHash: Set<TestVector> = []
 
         var coverageList: [TVCPair] = []
@@ -362,7 +374,11 @@ enum Simulator {
 
         return (
             coverageList: coverageList,
-            coverage: coverage
+            coverageMeta: CoverageMeta(
+                ratio: coverage,
+                sa0Covered: sa0Covered.sorted(),
+                sa1Covered: sa1Covered.sorted()
+            )
         )
     }
 

diff --git a/Tests/FaultTests/FaultTests.swift b/Tests/FaultTests/FaultTests.swift
@@ -114,6 +114,8 @@ final class FaultTests: XCTestCase {
         let fileSynth = base + ".nl.v"
         let fileCut = base + ".cut.v"
         let fileJson = base + ".tv.json"
+        let faultPointsYML = base + ".fault_points.yml"
+        let coverageYml = base + ".coverage_meta.yml"
         let fileChained = base + ".chained.v"
         let fileAsmVec = fileJson + ".vec.bin"
         let fileAsmOut = fileJson + ".out.bin"
@@ -125,7 +127,7 @@ final class FaultTests: XCTestCase {
         try run(steps: [
             ["synth", "-l", liberty, "-t", topModule, "-o", fileSynth, "--blackboxModel", "Tests/RTL/integration/buffered_inverter.v", fileName],
             ["cut", "-o", fileCut, "--blackbox", "BufferedInverter", "--blackboxModel", "Tests/RTL/integration/buffered_inverter.v", "--ignoring", "clk,rst,rstn", fileSynth],
-            ["-c", models, "-i", reset, "--clock", clock, "-o", fileJson, fileCut],
+            ["-c", models, "-i", reset, "--clock", clock, "-o", fileJson, "--output-fault-points", faultPointsYML, "--output-covered", coverageYml, fileCut],
             ["chain", "-c", models, "-l", liberty, "-o", fileChained, "--clock", clock, "--reset", reset, "--activeLow", "-i", ignoredInputs, fileSynth, "--blackbox", "BufferedInverter", "--blackboxModel", "Tests/RTL/integration/buffered_inverter.v"],
             ["asm", fileJson, fileChained],
             ["compact", "-o", "/dev/null", fileJson],