diff --git a/Changelog.md b/Changelog.md index e3f5eb8c5..faf91f841 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # FOSSA CLI Changelog +## 3.13.1 +- Add a summary of the snippet scan when the `--x-snippet-scan` flag is used ([#1613](https://github.com/fossas/fossa-cli/pull/1613)) + ## 3.13.0 - Change how debug logs are generated. They are now generated in a file called fossa.debug.zip, which can contain multiple files. For the common case of `fossa analyze --debug`, it will now contain the debug bundle (fossa.debug.json) and the telemetry json (fossa.telemetry.json). It will also contain Ficus logs if Ficus is run via --x-snippet-scan ([#1610](https://github.com/fossas/fossa-cli/pull/1610)) diff --git a/integration-test/Analysis/FicusSpec.hs b/integration-test/Analysis/FicusSpec.hs index 5121d3a80..554b9fcba 100644 --- a/integration-test/Analysis/FicusSpec.hs +++ b/integration-test/Analysis/FicusSpec.hs @@ -56,8 +56,8 @@ spec = do case result of Success _warnings analysisResult -> do case analysisResult of - Just (FicusSnippetScanResults analysisId) -> do - analysisId `shouldSatisfy` (> 0) + Just results -> do + ficusSnippetScanResultsAnalysisId results `shouldSatisfy` (> 0) Nothing -> do -- No snippet scan results returned - this is acceptable for integration testing True `shouldBe` True diff --git a/src/App/Fossa/Ficus/Analyze.hs b/src/App/Fossa/Ficus/Analyze.hs index 77b2f38b2..a9f26f21c 100644 --- a/src/App/Fossa/Ficus/Analyze.hs +++ b/src/App/Fossa/Ficus/Analyze.hs @@ -21,6 +21,7 @@ import App.Fossa.Ficus.Types ( FicusMessageData (..), FicusMessages (..), FicusPerStrategyFlag (..), + FicusScanStats (..), FicusSnippetScanResults (..), ) import App.Types (ProjectRevision (..)) @@ -29,8 +30,7 @@ import Control.Carrier.Diagnostics (Diagnostics) import Control.Concurrent.Async (async, wait) import Control.Effect.Lift (Has, Lift, sendIO) import Control.Monad (when) -import Data.Aeson (Object, decode, decodeStrictText, (.:)) -import Data.Aeson.Types (parseMaybe) +import Data.Aeson (decode, decodeStrictText) import Data.ByteString.Lazy qualified as BL import Data.Conduit ((.|)) import Data.Conduit qualified as Conduit @@ -64,6 +64,7 @@ import System.Process.Typed ( waitExitCode, withProcessWait, ) +import Text.Printf (printf) import Text.URI (render) import Text.URI.Builder (PathComponent (PathComponent), TrailingSlash (TrailingSlash), setPath) import Types (GlobFilter (..), LicenseScanPathFilters (..)) @@ -116,7 +117,7 @@ analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays m logDebugWithTime "runFicus completed, processing results..." case ficusResults of Just results -> - logInfo $ "Ficus analysis completed successfully with analysis ID: " <> pretty (ficusSnippetScanResultsAnalysisId results) + logInfo $ pretty (formatFicusScanSummary results) Nothing -> logInfo "Ficus analysis completed but no fingerprint findings were found" pure ficusResults where @@ -131,13 +132,37 @@ analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays m , ficusConfigSnippetScanRetentionDays = snippetScanRetentionDays } -findingToAnalysisId :: FicusFinding -> Maybe Int -findingToAnalysisId (FicusFinding (FicusMessageData strategy payload)) +findingToSnippetScanResult :: FicusFinding -> Maybe FicusSnippetScanResults +findingToSnippetScanResult (FicusFinding (FicusMessageData strategy payload)) | Text.toLower strategy == "fingerprint" = - case decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload) :: Maybe Object of - Just obj -> parseMaybe (.: "analysis_id") obj - Nothing -> Nothing -findingToAnalysisId _ = Nothing + decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload) +findingToSnippetScanResult _ = Nothing + +formatFicusScanSummary :: FicusSnippetScanResults -> Text +formatFicusScanSummary results = + let stats = ficusSnippetScanResultsStats results + aid = ficusSnippetScanResultsAnalysisId results + in Text.unlines + [ "Ficus snippet scan completed successfully!" + , "" + , "============================================================" + , "Snippet scan summary:" + , " Analysis ID: " <> toText (show aid) + , " Bucket ID: " <> toText (show $ ficusSnippetScanResultsBucketId results) + , " Files skipped: " <> toText (show $ ficusStatsSkippedFiles stats) + , " Total Files processed: " <> toText (show $ ficusStatsProcessedFiles stats) + , " Unique Files processed: " <> toText (show $ ficusStatsUniqueProcessedFiles stats) + , " Unique Files with matches found: " <> toText (show $ ficusStatsUniqueMatchedFiles stats) + , " Unique Files with no matches found: " <> toText (show $ ficusStatsUniqueUnmatchedFiles stats) + , " Unique Files already in our knowledge base: " <> toText (show $ ficusStatsUniqueExistingFiles stats) + , " Unique Files new to our knowledge base: " <> toText (show $ ficusStatsUniqueNewFiles stats) + , " Processing time: " <> formatProcessingTime (ficusStatsProcessingTimeSeconds stats) <> "s" + , "============================================================" + ] + where + -- Format the processing time as a string with 3 decimal places + formatProcessingTime :: Double -> Text + formatProcessingTime seconds = toText (printf "%.3f" seconds :: String) runFicus :: ( Has Diagnostics sig m @@ -236,10 +261,10 @@ runFicus maybeDebugDir ficusConfig = do pure acc FicusMessageFinding finding -> do hPutStrLn stderr $ "[" ++ timestamp ++ "] FINDING " <> toString (displayFicusFinding finding) - let analysisFinding = FicusSnippetScanResults <$> findingToAnalysisId finding + let analysisFinding = findingToSnippetScanResult finding when (isJust acc && isJust analysisFinding) $ hPutStrLn stderr $ - "[" ++ timestamp ++ "] ERROR " <> "Found multiple analysis ids." + "[" ++ timestamp ++ "] ERROR " <> "Found multiple ficus analysis responses." pure $ acc <|> analysisFinding ) Nothing diff --git a/src/App/Fossa/Ficus/Types.hs b/src/App/Fossa/Ficus/Types.hs index 2250b665d..1db758169 100644 --- a/src/App/Fossa/Ficus/Types.hs +++ b/src/App/Fossa/Ficus/Types.hs @@ -13,12 +13,12 @@ module App.Fossa.Ficus.Types ( FicusHashFlag (..), FicusSnippetScanFlag, FicusSnippetScanResults (..), + FicusScanStats (..), FicusPerStrategyFlag (..), ) where import App.Types (ProjectRevision) -import Data.Aeson (FromJSON, Value (Object), withText) -import Data.Aeson qualified as A +import Data.Aeson (FromJSON (parseJSON), Value (Object), withObject, withText) import Data.Aeson.Types (Parser, (.:)) import Data.Text (Text) import Fossa.API.Types @@ -27,7 +27,43 @@ import Path (Abs, Dir, Path) import Text.URI import Types (GlobFilter) -newtype FicusSnippetScanResults = FicusSnippetScanResults {ficusSnippetScanResultsAnalysisId :: Int} deriving (Eq, Ord, Show, Generic) +data FicusSnippetScanResults = FicusSnippetScanResults + { ficusSnippetScanResultsAnalysisId :: Int + , ficusSnippetScanResultsBucketId :: Int + , ficusSnippetScanResultsStats :: FicusScanStats + } + deriving (Eq, Ord, Show, Generic) + +instance FromJSON FicusSnippetScanResults where + parseJSON = withObject "FicusSnippetScanResults" $ \obj -> + FicusSnippetScanResults + <$> obj .: "analysis_id" + <*> obj .: "bucket_id" + <*> obj .: "stats" + +data FicusScanStats = FicusScanStats + { ficusStatsSkippedFiles :: Int + , ficusStatsProcessedFiles :: Int + , ficusStatsUniqueProcessedFiles :: Int + , ficusStatsUniqueNewFiles :: Int + , ficusStatsUniqueExistingFiles :: Int + , ficusStatsUniqueMatchedFiles :: Int + , ficusStatsUniqueUnmatchedFiles :: Int + , ficusStatsProcessingTimeSeconds :: Double + } + deriving (Eq, Ord, Show, Generic) + +instance FromJSON FicusScanStats where + parseJSON = withObject "FicusScanStats" $ \obj -> + FicusScanStats + <$> obj .: "skipped_files" + <*> obj .: "processed_files" + <*> obj .: "unique_processed_files" + <*> obj .: "unique_new_files" + <*> obj .: "unique_existing_files" + <*> obj .: "unique_matched_files" + <*> obj .: "unique_unmatched_files" + <*> obj .: "processing_time_seconds" data FicusMessages = FicusMessages { ficusMessageDebugs :: [FicusDebug]