Update to accept specific strategies

Author: nficca

integration-test/Analysis/FicusSpec.hs

@@ -4,7 +4,7 @@
 module Analysis.FicusSpec (spec) where
 
 import App.Fossa.Ficus.Analyze (analyzeWithFicus)
-import App.Fossa.Ficus.Types (FicusAnalysisResults (..), FicusSnippetScanResults (..))
+import App.Fossa.Ficus.Types (FicusAnalysisResults (..), FicusSnippetScanResults (..), FicusStrategy (FicusStrategySnippetScan))
 import App.Types (ProjectRevision (..))
 import Control.Carrier.Diagnostics (runDiagnostics)
 import Control.Carrier.Stack (runStack)
@@ -51,7 +51,7 @@ spec = do
       testDataExists <- PIO.doesDirExist testDataDir
       testDataExists `shouldBe` True
 
-      result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing (Just 10) Nothing
+      result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision [FicusStrategySnippetScan] Nothing (Just 10) Nothing
 
       case result of
         Success _warnings analysisResult -> do

src/App/Fossa/Analyze.hs

@@ -51,7 +51,7 @@ import App.Fossa.Config.Analyze (
 import App.Fossa.Config.Analyze qualified as Config
 import App.Fossa.Config.Common (DestinationMeta (..), destinationApiOpts, destinationMetadata)
 import App.Fossa.Ficus.Analyze (analyzeWithFicus)
-import App.Fossa.Ficus.Types (FicusAnalysisResults (vendoredDependencyScanResults), FicusVendoredDependencyScanResults (FicusVendoredDependencyScanResults))
+import App.Fossa.Ficus.Types (FicusAnalysisResults (vendoredDependencyScanResults), FicusStrategy (FicusStrategySnippetScan, FicusStrategyVendetta), FicusVendoredDependencyScanResults (FicusVendoredDependencyScanResults))
 import App.Fossa.FirstPartyScan (runFirstPartyScan)
 import App.Fossa.Lernie.Analyze (analyzeWithLernie)
 import App.Fossa.Lernie.Types (LernieResults (..))
@@ -303,6 +303,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       allowedTactics = Config.allowedTacticTypes cfg
       withoutDefaultFilters = Config.withoutDefaultFilters cfg
       enableSnippetScan = Config.xSnippetScan cfg
+      enableVendetta = Config.xVendetta cfg
 
   manualSrcUnits <-
     Diag.errorBoundaryIO . diagToDebug $
@@ -341,24 +342,30 @@ analyze cfg = Diag.context "fossa-analyze" $ do
         if (fromFlag BinaryDiscovery $ Config.binaryDiscoveryEnabled $ Config.vsiOptions cfg)
           then analyzeDiscoverBinaries basedir filters
           else pure Nothing
+  let ficusStrategies = case [enableSnippetScan, enableVendetta] of
+        [True, True] -> [FicusStrategySnippetScan, FicusStrategyVendetta]
+        [True, False] -> [FicusStrategySnippetScan]
+        [False, True] -> [FicusStrategyVendetta]
+        [False, False] -> []
   maybeFicusResults <-
     Diag.errorBoundaryIO . diagToDebug $
-      if not enableSnippetScan
+      if null ficusStrategies
         then do
-          logInfo "Skipping ficus snippet scanning (--x-snippet-scan not set)"
+          logInfo "Skipping ficus scanning (--x-snippet-scan and/or x-vendetta not set)"
           pure Nothing
         else
           if filterIsVSIOnly filters
             then do
-              logInfo "Running in VSI only mode, skipping snippet-scan"
+              logInfo "Running in VSI only mode, skipping ficus scanning"
               pure Nothing
             else
-              Diag.context "snippet-scanning"
+              Diag.context "ficus-scanning"
                 . runStickyLogger SevInfo
                 $ analyzeWithFicus
                   basedir
                   maybeApiOpts
                   revision
+                  ficusStrategies
                   (Config.licenseScanPathFilters vendoredDepsOptions)
                   (orgSnippetScanSourceCodeRetentionDays =<< orgInfo)
                   (Config.debugDir cfg)

src/App/Fossa/Config/Analyze.hs

@@ -240,6 +240,7 @@ data AnalyzeCliOpts = AnalyzeCliOpts
   , analyzeWithoutDefaultFilters :: Flag WithoutDefaultFilters
   , analyzeStrictMode :: Flag StrictMode
   , analyzeSnippetScan :: Bool
+  , analyzeVendetta :: Bool
   }
   deriving (Eq, Ord, Show)
 
@@ -280,6 +281,7 @@ data AnalyzeConfig = AnalyzeConfig
   , mode :: Mode
   , xSnippetScan :: Bool
   , debugDir :: Maybe FilePath
+  , xVendetta :: Bool
   }
   deriving (Eq, Ord, Show, Generic)
 
@@ -352,6 +354,7 @@ cliParser =
     <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl
     <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Enforces strict analysis to ensure the most accurate results by rejecting fallbacks.")
     <*> switch (applyFossaStyle <> long "x-snippet-scan" <> stringToHelpDoc "Experimental flag to enable snippet scanning to identify open source code snippets using fingerprinting.")
+    <*> switch (applyFossaStyle <> long "x-vendetta" <> stringToHelpDoc "Experimental flag to enable vendored dependency scanning to identify open source components using file hashing.")
   where
     fossaDepsFileHelp :: Maybe (Doc AnsiStyle)
     fossaDepsFileHelp =
@@ -568,6 +571,7 @@ mergeStandardOpts maybeDebugDir maybeConfig envvars cliOpts@AnalyzeCliOpts{..} =
     <*> pure mode
     <*> pure analyzeSnippetScan
     <*> pure maybeDebugDir
+    <*> pure analyzeVendetta
 
 collectMavenScopeFilters ::
   (Has Diagnostics sig m) =>

src/App/Fossa/Ficus/Analyze.hs

@@ -24,6 +24,7 @@ import App.Fossa.Ficus.Types (
   FicusPerStrategyFlag (..),
   FicusScanStats (..),
   FicusSnippetScanResults (..),
+  FicusStrategy (FicusStrategyHash, FicusStrategyNoop, FicusStrategySnippetScan, FicusStrategyVendetta),
   FicusVendoredDependency (..),
   FicusVendoredDependencyScanResults (..),
  )
@@ -96,12 +97,13 @@ analyzeWithFicus ::
   Path Abs Dir ->
   Maybe ApiOpts ->
   ProjectRevision ->
+  [FicusStrategy] ->
   Maybe LicenseScanPathFilters ->
   Maybe Int ->
   Maybe FilePath -> -- Debug directory (if enabled)
   m (Maybe FicusAnalysisResults)
-analyzeWithFicus rootDir apiOpts revision filters snippetScanRetentionDays maybeDebugDir = do
-  Just <$> analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays maybeDebugDir
+analyzeWithFicus rootDir apiOpts revision strategies filters snippetScanRetentionDays maybeDebugDir = do
+  Just <$> analyzeWithFicusMain rootDir apiOpts revision strategies filters snippetScanRetentionDays maybeDebugDir
 
 analyzeWithFicusMain ::
   ( Has Diagnostics sig m
@@ -111,11 +113,12 @@ analyzeWithFicusMain ::
   Path Abs Dir ->
   Maybe ApiOpts ->
   ProjectRevision ->
+  [FicusStrategy] ->
   Maybe LicenseScanPathFilters ->
   Maybe Int ->
   Maybe FilePath -> -- Debug directory (if enabled)
   m FicusAnalysisResults
-analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays maybeDebugDir = do
+analyzeWithFicusMain rootDir apiOpts revision strategies filters snippetScanRetentionDays maybeDebugDir = do
   logDebugWithTime "Preparing Ficus analysis configuration..."
   ficusResults <- runFicus maybeDebugDir ficusConfig
   logDebugWithTime "runFicus completed, processing results..."
@@ -134,6 +137,7 @@ analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays m
         , ficusConfigRevision = revision
         , ficusConfigFlags = [All $ FicusAllFlag SkipHiddenFiles, All $ FicusAllFlag Gitignore]
         , ficusConfigSnippetScanRetentionDays = snippetScanRetentionDays
+        , ficusConfigOnlyStrategies = strategies
         }
 
 findingToSnippetScanResult :: FicusFinding -> Maybe FicusSnippetScanResults
@@ -171,7 +175,7 @@ formatFicusScanSummary results =
 
 findingToVendoredDependency :: FicusFinding -> Maybe FicusVendoredDependency
 findingToVendoredDependency (FicusFinding (FicusMessageData strategy payload))
-  | Text.toLower strategy == "vendored" =
+  | Text.toLower strategy == "vendetta" =
       decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload)
 findingToVendoredDependency _ = Nothing
 
@@ -400,11 +404,17 @@ ficusCommand ficusConfig bin = do
   pure cmd
   where
     snippetScanRetentionDays = ficusConfigSnippetScanRetentionDays ficusConfig
-    configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ maybe [] (\days -> ["--snippet-scan-retention-days", toText days]) snippetScanRetentionDays ++ [targetDir]
+    configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ configStrategies ++ maybe [] (\days -> ["--snippet-scan-retention-days", toText days]) snippetScanRetentionDays ++ [targetDir]
     targetDir = toText $ toFilePath $ ficusConfigRootDir ficusConfig
     secret = maybe "" (toText . unApiKey) $ ficusConfigSecret ficusConfig
     locator = renderLocator $ Locator "custom" (projectName $ ficusConfigRevision ficusConfig) (Just $ projectRevision $ ficusConfigRevision ficusConfig)
     configExcludes = concatMap (\path -> ["--exclude", unGlobFilter path]) $ ficusConfigExclude ficusConfig
+    configStrategies = concatMap (\strategy -> ["--only", strategyToArg strategy]) $ ficusConfigOnlyStrategies ficusConfig
+    strategyToArg = \case
+      FicusStrategySnippetScan -> "snippet-scan"
+      FicusStrategyNoop -> "noop"
+      FicusStrategyHash -> "hash"
+      FicusStrategyVendetta -> "vendetta"
 
     maskApiKeyInCommand :: Text -> Text
     maskApiKeyInCommand cmdText =

src/App/Fossa/Ficus/Types.hs

@@ -7,13 +7,15 @@ module App.Fossa.Ficus.Types (
   FicusDebug (..),
   FicusError (..),
   FicusAnalysisFlag (..),
+  FicusStrategy (..),
   FicusAllFlag (..),
   FicusWalkFlag (..),
   FicusNoopFlag (..),
   FicusHashFlag (..),
   FicusSnippetScanFlag,
   FicusSnippetScanResults (..),
   FicusScanStats (..),
+  FicusVendettaFlag,
   FicusPerStrategyFlag (..),
   FicusAnalysisResults (..),
   FicusVendoredDependency (..),
@@ -204,9 +206,17 @@ data FicusConfig = FicusConfig
   , ficusConfigRevision :: ProjectRevision -- TODO: get this from `projectRevision AnalyzeConfig`
   , ficusConfigFlags :: [FicusPerStrategyFlag]
   , ficusConfigSnippetScanRetentionDays :: Maybe Int
+  , ficusConfigOnlyStrategies :: [FicusStrategy]
   }
   deriving (Show, Eq, Generic)
 
+data FicusStrategy
+  = FicusStrategySnippetScan
+  | FicusStrategyNoop
+  | FicusStrategyHash
+  | FicusStrategyVendetta
+  deriving (Show, Eq, Generic)
+
 -- A flag for ficus paired with a proper strategy or pseudo-strategy.
 -- @Walk@ and @All@ are pseudo-strategies which accept similar flags,
 -- but expand into a subset of strategies in ficus.
@@ -216,6 +226,7 @@ data FicusPerStrategyFlag
   | SnippetScan FicusSnippetScanFlag
   | Noop FicusNoopFlag
   | Hash FicusHashFlag
+  | Vendetta FicusVendettaFlag
   deriving (Show, Eq, Generic)
 
 data FicusAnalysisFlag
@@ -233,6 +244,11 @@ newtype FicusNoopFlag = FicusNoopFlag FicusAnalysisFlag deriving (Show, Eq)
 newtype FicusHashFlag = FicusHashFlag FicusAnalysisFlag deriving (Show, Eq)
 
 data FicusSnippetScanFlag
-  = CommonFlag FicusAnalysisFlag
-  | BatchLen Int
+  = SnippetScanCommonFlag FicusAnalysisFlag
+  | SnippetScanBatchLen Int
+  deriving (Show, Eq)
+
+data FicusVendettaFlag
+  = VendettaCommonFlag FicusAnalysisFlag
+  | VendettaBatchLen Int
   deriving (Show, Eq)

test/Test/Fixtures.hs

@@ -678,6 +678,7 @@ standardAnalyzeConfig =
     , ANZ.mode = NonStrict
     , ANZ.xSnippetScan = False
     , ANZ.debugDir = Nothing
+    , ANZ.xVendetta = False
     }
 
 sampleJarParsedContent :: Text