Snippet scan summary (#1613)

Author: spatten

Changelog.md

@@ -1,5 +1,8 @@
 # FOSSA CLI Changelog
 
+## 3.13.1
+- Add a summary of the snippet scan when the `--x-snippet-scan` flag is used ([#1613](https://github.com/fossas/fossa-cli/pull/1613))
+
 ## 3.13.0
 - Change how debug logs are generated. They are now generated in a file called fossa.debug.zip, which can contain multiple files. For the common case of `fossa analyze --debug`, it will now contain the debug bundle (fossa.debug.json) and the telemetry json (fossa.telemetry.json). It will also contain Ficus logs if Ficus is run via --x-snippet-scan ([#1610](https://github.com/fossas/fossa-cli/pull/1610))
 

integration-test/Analysis/FicusSpec.hs

@@ -56,8 +56,8 @@ spec = do
       case result of
         Success _warnings analysisResult -> do
           case analysisResult of
-            Just (FicusSnippetScanResults analysisId) -> do
-              analysisId `shouldSatisfy` (> 0)
+            Just results -> do
+              ficusSnippetScanResultsAnalysisId results `shouldSatisfy` (> 0)
             Nothing -> do
               -- No snippet scan results returned - this is acceptable for integration testing
               True `shouldBe` True

src/App/Fossa/Ficus/Analyze.hs

@@ -21,6 +21,7 @@ import App.Fossa.Ficus.Types (
   FicusMessageData (..),
   FicusMessages (..),
   FicusPerStrategyFlag (..),
+  FicusScanStats (..),
   FicusSnippetScanResults (..),
  )
 import App.Types (ProjectRevision (..))
@@ -29,8 +30,7 @@ import Control.Carrier.Diagnostics (Diagnostics)
 import Control.Concurrent.Async (async, wait)
 import Control.Effect.Lift (Has, Lift, sendIO)
 import Control.Monad (when)
-import Data.Aeson (Object, decode, decodeStrictText, (.:))
-import Data.Aeson.Types (parseMaybe)
+import Data.Aeson (decode, decodeStrictText)
 import Data.ByteString.Lazy qualified as BL
 import Data.Conduit ((.|))
 import Data.Conduit qualified as Conduit
@@ -64,6 +64,7 @@ import System.Process.Typed (
   waitExitCode,
   withProcessWait,
  )
+import Text.Printf (printf)
 import Text.URI (render)
 import Text.URI.Builder (PathComponent (PathComponent), TrailingSlash (TrailingSlash), setPath)
 import Types (GlobFilter (..), LicenseScanPathFilters (..))
@@ -116,7 +117,7 @@ analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays m
   logDebugWithTime "runFicus completed, processing results..."
   case ficusResults of
     Just results ->
-      logInfo $ "Ficus analysis completed successfully with analysis ID: " <> pretty (ficusSnippetScanResultsAnalysisId results)
+      logInfo $ pretty (formatFicusScanSummary results)
     Nothing -> logInfo "Ficus analysis completed but no fingerprint findings were found"
   pure ficusResults
   where
@@ -131,13 +132,37 @@ analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays m
         , ficusConfigSnippetScanRetentionDays = snippetScanRetentionDays
         }
 
-findingToAnalysisId :: FicusFinding -> Maybe Int
-findingToAnalysisId (FicusFinding (FicusMessageData strategy payload))
+findingToSnippetScanResult :: FicusFinding -> Maybe FicusSnippetScanResults
+findingToSnippetScanResult (FicusFinding (FicusMessageData strategy payload))
   | Text.toLower strategy == "fingerprint" =
-      case decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload) :: Maybe Object of
-        Just obj -> parseMaybe (.: "analysis_id") obj
-        Nothing -> Nothing
-findingToAnalysisId _ = Nothing
+      decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload)
+findingToSnippetScanResult _ = Nothing
+
+formatFicusScanSummary :: FicusSnippetScanResults -> Text
+formatFicusScanSummary results =
+  let stats = ficusSnippetScanResultsStats results
+      aid = ficusSnippetScanResultsAnalysisId results
+   in Text.unlines
+        [ "Ficus snippet scan completed successfully!"
+        , ""
+        , "============================================================"
+        , "Snippet scan summary:"
+        , "  Analysis ID: " <> toText (show aid)
+        , "  Bucket ID: " <> toText (show $ ficusSnippetScanResultsBucketId results)
+        , "  Files skipped: " <> toText (show $ ficusStatsSkippedFiles stats)
+        , "  Total Files processed: " <> toText (show $ ficusStatsProcessedFiles stats)
+        , "  Unique Files processed: " <> toText (show $ ficusStatsUniqueProcessedFiles stats)
+        , "  Unique Files with matches found: " <> toText (show $ ficusStatsUniqueMatchedFiles stats)
+        , "  Unique Files with no matches found: " <> toText (show $ ficusStatsUniqueUnmatchedFiles stats)
+        , "  Unique Files already in our knowledge base: " <> toText (show $ ficusStatsUniqueExistingFiles stats)
+        , "  Unique Files new to our knowledge base: " <> toText (show $ ficusStatsUniqueNewFiles stats)
+        , "  Processing time: " <> formatProcessingTime (ficusStatsProcessingTimeSeconds stats) <> "s"
+        , "============================================================"
+        ]
+  where
+    -- Format the processing time as a string with 3 decimal places
+    formatProcessingTime :: Double -> Text
+    formatProcessingTime seconds = toText (printf "%.3f" seconds :: String)
 
 runFicus ::
   ( Has Diagnostics sig m
@@ -236,10 +261,10 @@ runFicus maybeDebugDir ficusConfig = do
                     pure acc
                   FicusMessageFinding finding -> do
                     hPutStrLn stderr $ "[" ++ timestamp ++ "] FINDING " <> toString (displayFicusFinding finding)
-                    let analysisFinding = FicusSnippetScanResults <$> findingToAnalysisId finding
+                    let analysisFinding = findingToSnippetScanResult finding
                     when (isJust acc && isJust analysisFinding) $
                       hPutStrLn stderr $
-                        "[" ++ timestamp ++ "] ERROR " <> "Found multiple analysis ids."
+                        "[" ++ timestamp ++ "] ERROR " <> "Found multiple ficus analysis responses."
                     pure $ acc <|> analysisFinding
             )
             Nothing

src/App/Fossa/Ficus/Types.hs

@@ -13,12 +13,12 @@ module App.Fossa.Ficus.Types (
   FicusHashFlag (..),
   FicusSnippetScanFlag,
   FicusSnippetScanResults (..),
+  FicusScanStats (..),
   FicusPerStrategyFlag (..),
 ) where
 
 import App.Types (ProjectRevision)
-import Data.Aeson (FromJSON, Value (Object), withText)
-import Data.Aeson qualified as A
+import Data.Aeson (FromJSON (parseJSON), Value (Object), withObject, withText)
 import Data.Aeson.Types (Parser, (.:))
 import Data.Text (Text)
 import Fossa.API.Types
@@ -27,7 +27,43 @@ import Path (Abs, Dir, Path)
 import Text.URI
 import Types (GlobFilter)
 
-newtype FicusSnippetScanResults = FicusSnippetScanResults {ficusSnippetScanResultsAnalysisId :: Int} deriving (Eq, Ord, Show, Generic)
+data FicusSnippetScanResults = FicusSnippetScanResults
+  { ficusSnippetScanResultsAnalysisId :: Int
+  , ficusSnippetScanResultsBucketId :: Int
+  , ficusSnippetScanResultsStats :: FicusScanStats
+  }
+  deriving (Eq, Ord, Show, Generic)
+
+instance FromJSON FicusSnippetScanResults where
+  parseJSON = withObject "FicusSnippetScanResults" $ \obj ->
+    FicusSnippetScanResults
+      <$> obj .: "analysis_id"
+      <*> obj .: "bucket_id"
+      <*> obj .: "stats"
+
+data FicusScanStats = FicusScanStats
+  { ficusStatsSkippedFiles :: Int
+  , ficusStatsProcessedFiles :: Int
+  , ficusStatsUniqueProcessedFiles :: Int
+  , ficusStatsUniqueNewFiles :: Int
+  , ficusStatsUniqueExistingFiles :: Int
+  , ficusStatsUniqueMatchedFiles :: Int
+  , ficusStatsUniqueUnmatchedFiles :: Int
+  , ficusStatsProcessingTimeSeconds :: Double
+  }
+  deriving (Eq, Ord, Show, Generic)
+
+instance FromJSON FicusScanStats where
+  parseJSON = withObject "FicusScanStats" $ \obj ->
+    FicusScanStats
+      <$> obj .: "skipped_files"
+      <*> obj .: "processed_files"
+      <*> obj .: "unique_processed_files"
+      <*> obj .: "unique_new_files"
+      <*> obj .: "unique_existing_files"
+      <*> obj .: "unique_matched_files"
+      <*> obj .: "unique_unmatched_files"
+      <*> obj .: "processing_time_seconds"
 
 data FicusMessages = FicusMessages
   { ficusMessageDebugs :: [FicusDebug]