Add Code Coverage Report. (#7357)

Create coverage reports for PRs when PRs are updated.

scripts/code_coverage_report/generate_code_coverage_report/Package.swift

@@ -0,0 +1,45 @@
+// swift-tools-version:5.3
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PackageDescription
+
+let package = Package(
+  name: "CoverageReportGenerator",
+  products: [
+    // Products define the executables and libraries a package produces, and make them visible to other packages.
+    .executable(
+      name: "CoverageReportGenerator",
+      targets: ["CoverageReportGenerator"]
+    ),
+  ],
+  dependencies: [
+    // Dependencies declare other packages that this package depends on.
+    // .package(url: /* package url */, from: "1.0.0"),
+    .package(url: "https://github.com/apple/swift-argument-parser", from: "0.2.0"),
+  ],
+  targets: [
+    // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+    // Targets can depend on other targets in this package, and on products in packages this package depends on.
+    .target(
+      name: "CoverageReportGenerator",
+      dependencies: [
+        .product(name: "ArgumentParser", package: "swift-argument-parser"),
+      ]
+    ),
+  ]
+)

scripts/code_coverage_report/generate_code_coverage_report/README.md

@@ -0,0 +1,60 @@
+# coverage_report_parser
+
+This is a tool to read test coverages of xcresult bundle and generate a json report. This json
+report will be sent to Metrics Service to create a coverage report as a comment in a PR or to update
+the coverage database.
+
+## Usage
+
+This is tool will be used for both pull_request and merge. Common flags are shown below.
+
+```
+swift run CoverageReportGenerator --presubmit "${REPO}" --commit "${GITHUB_SHA}" --token "${TOKEN}" \
+--xcresult-dir "${XCRESULT_DIR}" --log-link "${}" --pull-request-num "${PULL_REQUEST_NUM}" \
+--base-commit "${BASE_COMMIT}" --branch "${BRANCH}"
+```
+Common parameters for both pull_request and merge:
+- `presubmit/merge`: A required flag to know if the request is for pull requests or merge.
+- `REPO`: A required argument for a repo where coverage data belong.
+- `commit`: The current commit sha.
+- `token`: A token to access a service account of Metrics Service
+- `xcresult-dir`: A directory containing all xcresult bundles.
+
+### Create a report in a pull request
+
+In a workflow, this will run for each pull request update. The command below will generate a report
+in a PR. After a workflow of test coverage is done, a new coverage report will be posted on a
+comment of a pull request. If such comment has existed, this comment will be overriden by the latest
+report.
+
+Since the flag is `presubmit` here, the following options are required for a PR request:
+- `log-link`: Log link to unit tests. This is generally a actions/runs/ link in Github Actions.
+- `pull-request-num`: A report will be posted in this pull request.
+- `base-commit`: The commit sha used to compare the diff of the current`commit`.
+
+An example in a Github Actions workflow:
+```
+swift run CoverageReportGenerator --presubmit "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" \
+--token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" \
+--log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" \
+--pull-request-num ${{github.event.pull_request.number}} --base-commit "$base_commit"
+
+```
+
+### Add new coverage data to the storage of Metrics Service
+
+In a workflow, this will run in merge events or postsubmit tests. After each merge, all pod tests
+will run to add a new commit and its corresponding coverage data.
+```
+swift run CoverageReportGenerator --merge "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" \
+--token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" \
+--log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" --branch \
+"${GITHUB_REF##*/}"
+```
+- `branch`: this is for merge and the new commit with coverage data will be linked with the branch
+in the database of Metrics Service.
+
+### Details
+
+More details in go/firebase-ios-sdk-test-coverage-metrics. Can also run
+`swift run CoverageReportGenerator -h` for help info.

scripts/code_coverage_report/generate_code_coverage_report/Sources/CoverageReportGenerator/CoverageReportParser.swift

@@ -0,0 +1,177 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+// This will contain code coverage result from a xcresult bundle.
+struct CoverageReportSource: Codable {
+  let coveredLines: Int
+  let lineCoverage: Double
+  let targets: [Target]
+
+  struct Target: Codable {
+    let name: String
+    let lineCoverage: Double
+    let files: [File]
+    struct File: Codable {
+      let coveredLines: Int
+      let lineCoverage: Double
+      let path: String
+      let name: String
+    }
+  }
+}
+
+// This will contains data that will be eventually transferred to a json file
+// sent to the Metrics Service.
+struct CoverageReportRequestData: Codable {
+  var metric: String
+  var results: [FileCoverage]
+  var log: String
+
+  struct FileCoverage: Codable {
+    let sdk: String
+    let type: String
+    let value: Double
+  }
+}
+
+// In the tool here, this will contain add all CoverageReportSource objects from
+// different xcresult bundles.
+extension CoverageReportRequestData {
+  init() {
+    metric = "Coverage"
+    results = []
+    log = ""
+  }
+
+  mutating func addCoverageData(from source: CoverageReportSource, resultBundle: String) {
+    for target in source.targets {
+      // Get sdk name. resultBundle is like ${SDK}-${platform}. E.g. FirebaseDatabase-ios.
+      // To display only sdk related tests and exclude non related testing, e.g.
+      // FirebaseDatabase-ios-GoogleDataTransport.framework,
+      // FirebaseDatabase-ios-FirebaseCore.framework, a regex pattern will be
+      // used to exclude results that are not related in terms of the target names.
+      let sdk_name = resultBundle.components(separatedBy: "-")[0]
+      let range = NSRange(location: 0, length: sdk_name.utf16.count)
+      let sdk_related_coverage_file_pattern = try! NSRegularExpression(
+        pattern: ".*\(sdk_name).*",
+        options: NSRegularExpression.Options(rawValue: 0)
+      )
+
+      if sdk_related_coverage_file_pattern.firstMatch(in: target.name, range: range) != nil {
+        results
+          .append(FileCoverage(sdk: resultBundle + "-" + target.name, type: "",
+                               value: target.lineCoverage))
+        for file in target.files {
+          results
+            // .append(FileCoverage(sdk: resultBundle + "-" + target.name + "(Coverage:\(String(format:"%.2f%%",  target.lineCoverage*100)))", type: file.name,
+            //                    value: file.lineCoverage))
+            .append(FileCoverage(sdk: resultBundle + "-" + target.name, type: file.name,
+                                 value: file.lineCoverage))
+          results
+            .append(FileCoverage(sdk: resultBundle + "-" + target.name, type: file.name,
+                                 value: file.lineCoverage))
+        }
+      }
+    }
+  }
+
+  mutating func addLogLink(_ logLink: String) {
+    log = logLink
+  }
+
+  func toData() -> Data {
+    let jsonData = try! JSONEncoder().encode(self)
+    return jsonData
+  }
+}
+
+struct Shell {
+  static let shared = Shell()
+  @discardableResult
+  func run(_ command: String, displayCommand: Bool = true,
+           displayFailureResult: Bool = true) -> Int32 {
+    let task = Process()
+    let pipe = Pipe()
+    task.standardOutput = pipe
+    task.launchPath = "/bin/zsh"
+    task.arguments = ["-c", command]
+    task.launch()
+    if displayCommand {
+      print("[CoverageReportParser] Command:\(command)\n")
+    }
+    task.waitUntilExit()
+    let data = pipe.fileHandleForReading.readDataToEndOfFile()
+    let log = String(data: data, encoding: .utf8)!
+    if displayFailureResult, task.terminationStatus != 0 {
+      print("-----Exit code: \(task.terminationStatus)")
+      print("-----Log:\n \(log)")
+    }
+    return task.terminationStatus
+  }
+}
+
+// Read json file and transfer to CoverageReportSource.
+func readLocalFile(forName name: String) -> CoverageReportSource? {
+  do {
+    let fileURL = URL(fileURLWithPath: FileManager().currentDirectoryPath)
+      .appendingPathComponent(name)
+    let data = try Data(contentsOf: fileURL)
+    let coverageReportSource = try JSONDecoder().decode(CoverageReportSource.self, from: data)
+    return coverageReportSource
+  } catch {
+    print("CoverageReportSource is not able to be generated. \(error)")
+  }
+
+  return nil
+}
+
+// Get in the dir, xcresultDirPathURL, which contains all xcresult bundles, and
+// create CoverageReportRequestData which will have all coverage data for in
+// the dir.
+func combineCodeCoverageResultBundles(from xcresultDirPathURL: URL,
+                                      log: String) throws -> CoverageReportRequestData? {
+  let fileManager = FileManager.default
+  do {
+    var coverageReportRequestData = CoverageReportRequestData()
+    coverageReportRequestData.addLogLink(log)
+    let fileURLs = try fileManager.contentsOfDirectory(
+      at: xcresultDirPathURL,
+      includingPropertiesForKeys: nil
+    )
+    let xcresultURLs = fileURLs.filter { $0.pathExtension == "xcresult" }
+    for xcresultURL in xcresultURLs {
+      let resultBundleName = xcresultURL.deletingPathExtension().lastPathComponent
+      let coverageSourceJSONFile = "\(resultBundleName).json"
+      try? fileManager.removeItem(atPath: coverageSourceJSONFile)
+      Shell()
+        .run("xcrun xccov view --report --json \(xcresultURL.path) >> \(coverageSourceJSONFile)")
+      if let coverageReportSource = readLocalFile(forName: "\(coverageSourceJSONFile)") {
+        coverageReportRequestData.addCoverageData(
+          from: coverageReportSource,
+          resultBundle: resultBundleName
+        )
+      }
+    }
+    return coverageReportRequestData
+  } catch {
+    print(
+      "Error while enuermating files \(xcresultDirPathURL): \(error.localizedDescription)"
+    )
+  }
+  return nil
+}

scripts/code_coverage_report/generate_code_coverage_report/Sources/CoverageReportGenerator/MetricsServiceRequest.swift

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+#if canImport(FoundationNetworking)
+  import FoundationNetworking
+#endif
+
+func sendMetricsServiceRequest(repo: String, commits: String, jsonContent: Data, token: String,
+                               is_presubmit: Bool, branch: String?, pullRequest: Int?,
+                               pullRequestNote: String?, baseCommit: String?) {
+  var request: URLRequest
+  var semaphore = DispatchSemaphore(value: 0)
+  let endpoint =
+    "https://sdk-metrics-service-tv5rmd4a6q-uc.a.run.app/repos/\(repo)/commits/\(commits)/reports?"
+  var pathPara: [String] = []
+  if is_presubmit {
+    guard let pr = pullRequest else {
+      print(
+        "The pull request number should be specified for an API pull-request request to the Metrics Service."
+      )
+      return
+    }
+    guard let bc = baseCommit else {
+      print(
+        "Base commit hash should be specified for an API pull-request request to the Metrics Service."
+      )
+      return
+    }
+    pathPara.append("pull_request=\(String(pr))")
+    if let note = pullRequestNote {
+      let compatible_url_format_note = note
+        .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)!
+      pathPara.append("note=\(compatible_url_format_note))")
+    }
+    pathPara.append("base_commit=\(bc)")
+  } else {
+    guard let branch = branch else {
+      print("Targeted merged branch should be specified.")
+      return
+    }
+    pathPara.append("branch=\(branch)")
+  }
+
+  let webURL = endpoint + pathPara.joined(separator: "&")
+  guard let metricsServiceURL = URL(string: webURL) else {
+    print("URL Path \(webURL) is not valid.")
+    return
+  }
+  request = URLRequest(url: metricsServiceURL, timeoutInterval: Double.infinity)
+
+  request.addValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
+  request.addValue("application/json", forHTTPHeaderField: "Content-Type")
+
+  request.httpMethod = "POST"
+  request.httpBody = jsonContent
+
+  let task = URLSession.shared.dataTask(with: request) { data, response, error in
+    guard let data = data else {
+      print(String(describing: error))
+      return
+    }
+    print(String(data: data, encoding: .utf8)!)
+    semaphore.signal()
+  }
+
+  task.resume()
+  semaphore.wait()
+}

scripts/code_coverage_report/generate_code_coverage_report/Sources/CoverageReportGenerator/main.swift

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import ArgumentParser
+import Foundation
+
+enum RequestType: EnumerableFlag {
+  case presubmit
+  case merge
+}
+
+struct CoverageReportGenerator: ParsableCommand {
+  @Flag(help: "Determine if the request to Metrics Service is for pull_requests or merge.")
+  var requestType: RequestType
+
+  @Argument(help: "A repo coverage data will be related to.")
+  var repo: String
+
+  @Option(
+    help: "presubmit: compare the diff to the base_commit; merge: store coverage data linking to this commit."
+  )
+  var commit: String
+
+  @Option(help: "Token to access an account of the Metrics Service")
+  var token: String
+
+  @Option(
+    help: "The directory of all xcresult bundles with code coverage data. ONLY files/dirs under this directory will be searched."
+  )
+  var xcresultDir: String
+
+  @Option(help: "Link to the log, leave \"\" if none.")
+  var logLink: String
+
+  @Option(
+    help: "This is for presubmit request. Number of a pull request that a coverage report will be posted on."
+  )
+  var pullRequestNum: Int?
+
+  @Option(help: "This is for presubmit request. Additional note for the report.")
+  var pullRequestNote: String?
+
+  @Option(
+    help: "This is for presubmit request. Coverage of commit will be compared to the coverage of this base_commit."
+  )
+  var baseCommit: String?
+
+  @Option(
+    help: "This is for merge request. Branch here will be linked to the coverage data, with the merged commit, in the database. "
+  )
+  var branch: String?
+
+  func run() throws {
+    if let coverageRequest = try combineCodeCoverageResultBundles(
+      from: URL(fileURLWithPath: xcresultDir),
+      log: logLink
+    ) {
+      sendMetricsServiceRequest(
+        repo: repo,
+        commits: commit,
+        jsonContent: coverageRequest.toData(),
+        token: token,
+        is_presubmit: requestType == RequestType.presubmit,
+        branch: branch,
+        pullRequest: pullRequestNum,
+        pullRequestNote: pullRequestNote,
+        baseCommit: baseCommit
+      )
+    } else {
+      print("coverageRequest is nil.")
+    }
+  }
+}
+
+CoverageReportGenerator.main()

scripts/code_coverage_report/get_updated_files.sh

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Updated files under the following paths will trigger code coverage workflows.
+# Updates in a pull request will generate a code coverage report in a pr.
 DATABASE_PATHS=("FirebaseDatabase.*" \
   ".github/workflows/database\\.yml" \
   "Example/Database/" \
@@ -28,6 +30,8 @@ echo "::set-output name=functions_run_job::false"
 # Get most rescent ancestor commit.
 common_commit=$(git merge-base remotes/origin/${pr_branch} remotes/origin/master)
 echo "The common commit is ${common_commit}."
+# Set base commit and this will be used to compare diffs of coverage to the current commit.
+echo "::set-output name=base_commit::${common_commit}"
 
 # List changed file from the base commit. This is generated by comparing the
 # head of the branch and the common commit from the master branch.

scripts/code_coverage_report/pod_test_code_coverage_report.sh

@@ -12,13 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
+
 SDK="$1"
 platform="$2"
 default_output_path="/Users/runner/${SDK}-${platform}.xcresult"
-output_path="${3:-default_output_path}"
+output_path="${3:-${default_output_path}}"
 if [ -d "/Users/runner/Library/Developer/Xcode/DerivedData" ]; then
 rm -r /Users/runner/Library/Developer/Xcode/DerivedData/*
 fi
+# Run unit tests of pods and put xcresult bundles into output_path, which
+# should be a targeted dir of actions/upload-artifact in workflows.
+# In code coverage workflow, files under output_path will be uploaded to
+# Github Actions.
 scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb "${SDK}".podspec --platforms="${platform}" --test-specs=unit
 find /Users/runner/Library/Developer/Xcode/DerivedData -type d -regex ".*/.*\.xcresult" -execdir cp -R '{}' "${output_path}" \;
-