diff --git a/docs/references/files/fossa-deps.md b/docs/references/files/fossa-deps.md index 95727da335..bc69695f9a 100644 --- a/docs/references/files/fossa-deps.md +++ b/docs/references/files/fossa-deps.md @@ -91,6 +91,25 @@ vendored-dependencies: For more details, please refer to the [feature](../../features/vendored-dependencies.md) walk through. +### `fork-aliases:` + +Denotes mapping of fork dependencies to their base dependencies. When a dependency matches a fork alias (by fetcher and project, ignoring version), it is replaced with the base locator, preserving the original version. This is useful when you have forked a dependency and want it to be treated as the original dependency in FOSSA. + +- `my-fork`: The locator for your fork of the dependency. Format: `+` or `+$` (e.g., `cargo+my-serde` or `cargo+my-serde$1.0.0`). (Required) +- `base`: The locator for the base/original dependency that your fork should be aliased to. Format: `+` or `+$` (e.g., `cargo+serde` or `cargo+serde$1.0.0`). (Required) +- `labels`: An optional list of labels to be added to the fork alias. + +```yaml +fork-aliases: +- my-fork: cargo+my-serde + base: cargo+serde + labels: + - label: internal + scope: org +``` + +> Note: Fork aliases match dependencies by fetcher and project name, ignoring version. When a match is found, the dependency is replaced with the base locator while preserving the original version from the fork. + ## Labels Each kind of dependency referenced above can have a `labels` field, which is a list of labels to be added to the dependency. @@ -140,6 +159,15 @@ vendored-dependencies: scope: project - label: internal-dependency scope: revision + +fork-aliases: +- my-fork: cargo+my-serde + base: cargo+serde + labels: + - label: internal + scope: org + - label: fork-approved + scope: revision ``` ## Errors in the `fossa-deps` file diff --git a/docs/references/files/fossa-deps.schema.json b/docs/references/files/fossa-deps.schema.json index d85f7ba178..5fec783db3 100644 --- a/docs/references/files/fossa-deps.schema.json +++ b/docs/references/files/fossa-deps.schema.json @@ -330,6 +330,32 @@ "version" ], "additionalProperties": false + }, + "fork-alias": { + "properties": { + "my-fork": { + "type": "string", + "description": "The locator for your fork of the dependency. Format: + or +$ (e.g., 'cargo+my-serde' or 'cargo+my-serde$1.0.0').", + "minLength": 1 + }, + "base": { + "type": "string", + "description": "The locator for the base/original dependency that your fork should be aliased to. Format: + or +$ (e.g., 'cargo+serde' or 'cargo+serde$1.0.0').", + "minLength": 1 + }, + "labels": { + "type": "array", + "description": "Optional labels to be applied to the fork alias.", + "items": { + "$ref": "#/$defs/label" + } + } + }, + "required": [ + "my-fork", + "base" + ], + "additionalProperties": false } }, "type": "object", @@ -364,6 +390,13 @@ "items": { "$ref": "#/$defs/remote-dependency" } + }, + "fork-aliases": { + "type": "array", + "description": "Fork aliases to map your fork dependencies to their base dependencies. When a dependency matches a fork alias (by fetcher and project, ignoring version), it is replaced with the base locator, preserving the original version.", + "items": { + "$ref": "#/$defs/fork-alias" + } } }, "required": [] diff --git a/spectrometer.cabal b/spectrometer.cabal index 8c8d9739b0..2ab2609a4d 100644 --- a/spectrometer.cabal +++ b/spectrometer.cabal @@ -722,6 +722,7 @@ test-suite unit-tests Scala.SbtDependencyTreeParsingSpec Scala.SbtDependencyTreeSpec Sqlite.SqliteSpec + Srclib.TypesSpec Swift.PackageResolvedSpec Swift.PackageSwiftSpec Swift.Xcode.PbxprojParserSpec diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 16ad47dd32..ea0ab2fcd6 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -32,7 +32,7 @@ import App.Fossa.Analyze.Types ( DiscoveredProjectIdentifier (..), DiscoveredProjectScan (..), ) -import App.Fossa.Analyze.Upload (ScanUnits (SourceUnitOnly), mergeSourceAndLicenseUnits, uploadSuccessfulAnalysis) +import App.Fossa.Analyze.Upload (ScanUnits (..), mergeSourceAndLicenseUnits, uploadSuccessfulAnalysis) import App.Fossa.BinaryDeps (analyzeBinaryDeps) import App.Fossa.Config.Analyze ( AnalysisTacticTypes (..), @@ -54,7 +54,7 @@ import App.Fossa.Ficus.Analyze (analyzeWithFicus) import App.Fossa.FirstPartyScan (runFirstPartyScan) import App.Fossa.Lernie.Analyze (analyzeWithLernie) import App.Fossa.Lernie.Types (LernieResults (..)) -import App.Fossa.ManualDeps (analyzeFossaDepsFile) +import App.Fossa.ManualDeps (ForkAlias (..), ManualDepsResult (..), analyzeFossaDepsFile) import App.Fossa.PathDependency (enrichPathDependencies, enrichPathDependencies', withPathDependencyNudge) import App.Fossa.PreflightChecks (PreflightCommandChecks (AnalyzeChecks), preflightChecks) import App.Fossa.Reachability.Upload (analyzeForReachability, onlyFoundUnits) @@ -103,10 +103,12 @@ import Data.Flag (Flag, fromFlag) import Data.Foldable (traverse_) import Data.Functor (($>)) import Data.List.NonEmpty qualified as NE +import Data.Map qualified as Map import Data.Maybe (fromMaybe, isJust, mapMaybe) import Data.String.Conversion (decodeUtf8, toText) import Data.Text.Extra (showT) import Data.Traversable (for) +import DepTypes (Dependency (..)) import Diag.Diagnostic as DI import Diag.Result (Result (Success), resultToMaybe) import Discovery.Archive qualified as Archive @@ -124,6 +126,8 @@ import Effect.Logger ( import Effect.ReadFS (ReadFS) import Errata (Errata (..)) import Fossa.API.Types (Organization (Organization, orgSnippetScanSourceCodeRetentionDays, orgSupportsReachability)) +import Graphing (Graphing) +import Graphing qualified import Path (Abs, Dir, Path, toFilePath) import Path.IO (makeRelative) import Prettyprinter ( @@ -136,8 +140,16 @@ import Prettyprinter.Render.Terminal ( Color (Cyan, Green, Yellow), color, ) +import Srclib.Converter (fetcherToDepType, toLocator) import Srclib.Converter qualified as Srclib -import Srclib.Types (LicenseSourceUnit (..), Locator, SourceUnit, sourceUnitToFullSourceUnit) +import Srclib.Types ( + LicenseSourceUnit (..), + Locator (..), + SourceUnit (..), + sourceUnitToFullSourceUnit, + toProjectLocator, + translateSourceUnitLocators, + ) import System.FilePath (()) import Types (DiscoveredProject (..), FoundTargets) @@ -303,13 +315,15 @@ analyze cfg = Diag.context "fossa-analyze" $ do withoutDefaultFilters = Config.withoutDefaultFilters cfg enableSnippetScan = Config.xSnippetScan cfg - manualSrcUnits <- + manualDepsResult <- Diag.errorBoundaryIO . diagToDebug $ if filterIsVSIOnly filters then do logInfo "Running in VSI only mode, skipping manual source units" - pure Nothing + pure $ ManualDepsResult Nothing [] else Diag.context "fossa-deps" . runStickyLogger SevInfo $ analyzeFossaDepsFile basedir customFossaDepsFile maybeApiOpts vendoredDepsOptions + let forkAliases = maybe [] manualDepsResultForkAliases (resultToMaybe manualDepsResult) + manualSrcUnits = fmap manualDepsResultSourceUnit manualDepsResult orgInfo <- for @@ -462,10 +476,18 @@ analyze cfg = Diag.context "fossa-analyze" $ do (Nothing, Just lernie) -> Just lernie (Just firstParty, Nothing) -> Just firstParty let keywordSearchResultsFound = (maybe False (not . null . lernieResultsKeywordSearches) lernieResults) - let outputResult = buildResult includeAll additionalSourceUnits filteredProjects' licenseSourceUnits + let forkAliasMap = mkForkAliasMap forkAliases + + -- Convert projects to source units and translate fork aliases in them + let scannedSourceUnits = map (Srclib.projectToSourceUnit (fromFlag IncludeAll includeAll)) filteredProjects' + let translatedAdditionalSourceUnits = map (translateSourceUnitLocators forkAliasMap) additionalSourceUnits + let translatedScannedSourceUnits = map (translateSourceUnitLocators forkAliasMap) scannedSourceUnits + let allTranslatedSourceUnits = translatedAdditionalSourceUnits ++ translatedScannedSourceUnits + + let outputResult = buildResult allTranslatedSourceUnits filteredProjects' licenseSourceUnits forkAliasMap scanUnits <- - case (keywordSearchResultsFound, checkForEmptyUpload includeAll projectScans filteredProjects' additionalSourceUnits licenseSourceUnits) of + case (keywordSearchResultsFound, checkForEmptyUpload projectScans filteredProjects' allTranslatedSourceUnits licenseSourceUnits) of (False, NoneDiscovered) -> Diag.warn ErrNoProjectsDiscovered $> emptyScanUnits (True, NoneDiscovered) -> Diag.warn ErrOnlyKeywordSearchResultsFound $> emptyScanUnits (False, FilteredAll) -> Diag.warn ErrFilteredAllProjects $> emptyScanUnits @@ -602,28 +624,49 @@ instance Diag.ToDiagnostic AnalyzeError where ] Errata (Just "Only keyword search results found") [] (Just body) -buildResult :: Flag IncludeAll -> [SourceUnit] -> [ProjectResult] -> Maybe LicenseSourceUnit -> Aeson.Value -buildResult includeAll srcUnits projects licenseSourceUnits = +buildResult :: [SourceUnit] -> [ProjectResult] -> Maybe LicenseSourceUnit -> Map.Map Locator Locator -> Aeson.Value +buildResult srcUnits projects licenseSourceUnits forkAliasMap = Aeson.object - [ "projects" .= map buildProject projects + [ "projects" .= map (buildProject forkAliasMap) projects , "sourceUnits" .= mergedUnits ] where mergedUnits = case licenseSourceUnits of - Nothing -> map sourceUnitToFullSourceUnit finalSourceUnits + Nothing -> map sourceUnitToFullSourceUnit srcUnits Just licenseUnits -> do - NE.toList $ mergeSourceAndLicenseUnits finalSourceUnits licenseUnits - finalSourceUnits = srcUnits ++ scannedUnits - scannedUnits = map (Srclib.projectToSourceUnit (fromFlag IncludeAll includeAll)) projects + NE.toList $ mergeSourceAndLicenseUnits srcUnits licenseUnits + +-- | Create a fork alias map from a list of fork aliases. +mkForkAliasMap :: [ForkAlias] -> Map.Map Locator Locator +mkForkAliasMap = Map.fromList . map (\ForkAlias{..} -> (toProjectLocator forkAliasMyFork, forkAliasBase)) -buildProject :: ProjectResult -> Aeson.Value -buildProject project = +buildProject :: Map.Map Locator Locator -> ProjectResult -> Aeson.Value +buildProject forkAliasMap project = Aeson.object [ "path" .= projectResultPath project , "type" .= projectResultType project - , "graph" .= graphingToGraph (projectResultGraph project) + , "graph" .= graphingToGraph (translateDependencyGraph forkAliasMap (projectResultGraph project)) ] +-- | Translate dependencies in a graph using fork aliases. +-- When a dependency matches a fork alias (by fetcher and project, ignoring version), +-- it is replaced with the base locator, preserving the original version. +translateDependencyGraph :: Map.Map Locator Locator -> Graphing Dependency -> Graphing Dependency +translateDependencyGraph forkAliasMap = Graphing.gmap (translateDependency forkAliasMap) + +-- | Translate a single dependency using fork aliases. +translateDependency :: Map.Map Locator Locator -> Dependency -> Dependency +translateDependency forkAliasMap dep = + case Map.lookup (toProjectLocator (toLocator dep)) forkAliasMap of + Nothing -> dep + Just baseLocator -> + let baseDepType = fromMaybe (dependencyType dep) (fetcherToDepType (locatorFetcher baseLocator)) + baseName = locatorProject baseLocator + in dep + { dependencyType = baseDepType + , dependencyName = baseName + } + updateProgress :: Has StickyLogger sig m => Progress -> m () updateProgress Progress{..} = logSticky' diff --git a/src/App/Fossa/Analyze/Filter.hs b/src/App/Fossa/Analyze/Filter.hs index e14cc0bcfb..0dcc579cfb 100644 --- a/src/App/Fossa/Analyze/Filter.hs +++ b/src/App/Fossa/Analyze/Filter.hs @@ -6,9 +6,6 @@ module App.Fossa.Analyze.Filter ( import App.Fossa.Analyze.Project (ProjectResult) import App.Fossa.Analyze.Types (DiscoveredProjectScan) import App.Fossa.Analyze.Upload (ScanUnits (..)) -import App.Fossa.Config.Analyze (IncludeAll (..)) -import Data.Flag (Flag, fromFlag) -import Srclib.Converter qualified as Srclib import Srclib.Types (LicenseSourceUnit (licenseSourceUnitLicenseUnits), LicenseUnit (licenseUnitName), SourceUnit) data CountedResult @@ -20,21 +17,21 @@ data CountedResult -- that the smaller list is the latter, and return that list. Starting with user-defined deps, -- we also include a check for an additional source unit from fossa-deps.yml -- and a check for any licenses found during the firstPartyScan -checkForEmptyUpload :: Flag IncludeAll -> [DiscoveredProjectScan] -> [ProjectResult] -> [SourceUnit] -> Maybe LicenseSourceUnit -> CountedResult -checkForEmptyUpload includeAll discovered filtered additionalUnits firstPartyScanResults = do - if null additionalUnits +checkForEmptyUpload :: [DiscoveredProjectScan] -> [ProjectResult] -> [SourceUnit] -> Maybe LicenseSourceUnit -> CountedResult +checkForEmptyUpload discovered filtered sourceUnits firstPartyScanResults = do + if null sourceUnits then case (discoveredLen, filteredLen, licensesMaybeFound) of (0, _, Nothing) -> NoneDiscovered (_, 0, Nothing) -> FilteredAll (0, 0, Just licenseSourceUnit) -> CountedScanUnits $ LicenseSourceUnitOnly licenseSourceUnit (0, _, Just licenseSourceUnit) -> CountedScanUnits $ LicenseSourceUnitOnly licenseSourceUnit (_, 0, Just licenseSourceUnit) -> CountedScanUnits $ LicenseSourceUnitOnly licenseSourceUnit - (_, _, Just licenseSourceUnit) -> CountedScanUnits $ SourceAndLicenseUnits discoveredUnits licenseSourceUnit - (_, _, Nothing) -> CountedScanUnits . SourceUnitOnly $ discoveredUnits - else -- If we have a additional source units, then there's always something to upload. + (_, _, Just licenseSourceUnit) -> CountedScanUnits $ SourceAndLicenseUnits sourceUnits licenseSourceUnit + (_, _, Nothing) -> CountedScanUnits . SourceUnitOnly $ sourceUnits + else -- If we have source units, then there's always something to upload. case licensesMaybeFound of - Nothing -> CountedScanUnits . SourceUnitOnly $ additionalUnits ++ discoveredUnits - Just licenseSourceUnit -> CountedScanUnits $ SourceAndLicenseUnits (additionalUnits ++ discoveredUnits) licenseSourceUnit + Nothing -> CountedScanUnits . SourceUnitOnly $ sourceUnits + Just licenseSourceUnit -> CountedScanUnits $ SourceAndLicenseUnits sourceUnits licenseSourceUnit where discoveredLen = length discovered filteredLen = length filtered @@ -45,7 +42,5 @@ checkForEmptyUpload includeAll discovered filtered additionalUnits firstPartySca then Just scanResults else Nothing - -- The smaller list is the post-filter list, since filtering cannot add projects - discoveredUnits = map (Srclib.projectToSourceUnit (fromFlag IncludeAll includeAll)) filtered isActualLicense :: LicenseUnit -> Bool isActualLicense licenseUnit = licenseUnitName licenseUnit /= "No_license_found" diff --git a/src/App/Fossa/Init/fossa-deps.yml b/src/App/Fossa/Init/fossa-deps.yml index 5a5da14caf..ebf5a171d2 100644 --- a/src/App/Fossa/Init/fossa-deps.yml +++ b/src/App/Fossa/Init/fossa-deps.yml @@ -116,3 +116,21 @@ # metadata: # Metadata of the dependency (Optional) # description: Django # Description of the dependency, this will be shown in FOSSA UI, and generated reports. (Optional) # homepage: https://www.djangoproject.com # Homepage of the dependency, this will be shown in FOSSA UI, and generated reports. (Optional) +# +# +# +# # fork-aliases +# # -- +# # Denotes mapping of fork dependencies to their base dependencies. When a dependency +# # matches a fork alias (by fetcher and project, ignoring version), it is replaced with +# # the base locator, preserving the original version. This is useful when you have forked +# # a dependency and want it to be treated as the original dependency in FOSSA. +# # +# # Learn more: https://github.com/fossas/fossa-cli/blob/master/docs/references/files/fossa-deps.md#fork-aliases +# # +# fork-aliases: +# - my-fork: cargo+my-serde # Locator for your fork. Format: + or +$ (Required) +# base: cargo+serde # Locator for the base/original dependency. Format: + or +$ (Required) +# labels: # Labels to be applied to the fork alias (Optional) +# - label: internal +# scope: org diff --git a/src/App/Fossa/ManualDeps.hs b/src/App/Fossa/ManualDeps.hs index 00a06fcbaf..294371099f 100644 --- a/src/App/Fossa/ManualDeps.hs +++ b/src/App/Fossa/ManualDeps.hs @@ -9,9 +9,11 @@ module App.Fossa.ManualDeps ( RemoteDependency (..), DependencyMetadata (..), VendoredDependency (..), + ForkAlias (..), ManualDependencies (..), LocatorDependency (..), FoundDepsFile (..), + ManualDepsResult (..), analyzeFossaDepsFile, findAndReadFossaDepsFile, findFossaDepsFile, @@ -82,6 +84,12 @@ data FoundDepsFile = ManualYaml (Path Abs File) | ManualJSON (Path Abs File) +data ManualDepsResult = ManualDepsResult + { manualDepsResultSourceUnit :: Maybe SourceUnit + , manualDepsResultForkAliases :: [ForkAlias] + } + deriving (Eq, Show) + analyzeFossaDepsFile :: ( Has Diagnostics sig m , Has ReadFS sig m @@ -95,21 +103,22 @@ analyzeFossaDepsFile :: Maybe FilePath -> Maybe ApiOpts -> VendoredDependencyOptions -> - m (Maybe SourceUnit) + m ManualDepsResult analyzeFossaDepsFile root maybeCustomFossaDepsPath maybeApiOpts vendoredDepsOptions = do maybeDepsFile <- case maybeCustomFossaDepsPath of Nothing -> findFossaDepsFile root Just filePath -> retrieveCustomFossaDepsFile filePath case maybeDepsFile of - Nothing -> pure Nothing + Nothing -> pure $ ManualDepsResult Nothing [] Just depsFile -> do manualDeps <- context "Reading fossa-deps file" $ readFoundDeps depsFile + let aliases = forkAliases manualDeps if hasNoDeps manualDeps - then pure Nothing - else - context "Converting fossa-deps to partial API payload" $ - Just <$> toSourceUnit root depsFile manualDeps maybeApiOpts vendoredDepsOptions + then pure $ ManualDepsResult Nothing aliases + else context "Converting fossa-deps to partial API payload" $ do + sourceUnit <- toSourceUnit root depsFile manualDeps maybeApiOpts vendoredDepsOptions + pure $ ManualDepsResult (Just sourceUnit) aliases retrieveCustomFossaDepsFile :: ( Has Diagnostics sig m @@ -234,6 +243,7 @@ collectInteriorLabels org ManualDependencies{..} = <> mapMaybe customDepToLabel customDependencies <> mapMaybe (remoteDepToLabel org) remoteDependencies <> mapMaybe locatorDepToLabel locatorDependencies + <> mapMaybe forkAliasToLabel forkAliases where liftEmpty :: (a, [b]) -> Maybe (a, [b]) liftEmpty (_, []) = Nothing @@ -267,6 +277,9 @@ collectInteriorLabels org ManualDependencies{..} = locatorDepToLabel (LocatorDependencyPlain _) = Nothing locatorDepToLabel (LocatorDependencyStructured locator labels) = liftEmpty (renderLocator locator, labels) + forkAliasToLabel :: ForkAlias -> Maybe (Text, [ProvidedPackageLabel]) + forkAliasToLabel ForkAlias{..} = liftEmpty (renderLocator forkAliasBase, forkAliasLabels) + -- | Run either archive upload or native license scan. scanAndUpload :: ( Has (Lift IO) sig m @@ -401,9 +414,24 @@ data ManualDependencies = ManualDependencies , vendoredDependencies :: [VendoredDependency] , remoteDependencies :: [RemoteDependency] , locatorDependencies :: [LocatorDependency] + , forkAliases :: [ForkAlias] } deriving (Eq, Ord, Show) +data ForkAlias = ForkAlias + { forkAliasMyFork :: Locator + , forkAliasBase :: Locator + , forkAliasLabels :: [ProvidedPackageLabel] + } + deriving (Eq, Ord, Show) + +instance FromJSON ForkAlias where + parseJSON = withObject "ForkAlias" $ \obj -> + ForkAlias + <$> obj .: "my-fork" + <*> obj .: "base" + <*> obj .:? "labels" .!= [] + data LocatorDependency = LocatorDependencyPlain Locator | LocatorDependencyStructured Locator [ProvidedPackageLabel] @@ -475,11 +503,12 @@ instance FromJSON ManualDependencies where <*> (obj .:? "vendored-dependencies" .!= []) <*> (obj .:? "remote-dependencies" .!= []) <*> (obj .:? "locator-dependencies" .!= []) + <*> (obj .:? "fork-aliases" .!= []) where isMissingOr1 :: Maybe Int -> Parser () isMissingOr1 (Just x) | x /= 1 = fail $ "Invalid fossa-deps version: " <> show x isMissingOr1 _ = pure () - parseJSON (Null) = pure $ ManualDependencies mempty mempty mempty mempty mempty + parseJSON (Null) = pure $ ManualDependencies mempty mempty mempty mempty mempty mempty parseJSON other = fail $ "Expected object or Null for ManualDependencies, but got: " <> show other depTypeParser :: Text -> Parser DepType diff --git a/src/Srclib/Types.hs b/src/Srclib/Types.hs index 308996c234..0aada2a9d2 100644 --- a/src/Srclib/Types.hs +++ b/src/Srclib/Types.hs @@ -35,6 +35,8 @@ module Srclib.Types ( sourceUnitToFullSourceUnit, licenseUnitToFullSourceUnit, textToOriginPath, + toProjectLocator, + translateSourceUnitLocators, ) where import Data.Aeson @@ -653,3 +655,39 @@ instance ToJSON Locator where instance FromJSON Locator where parseJSON = withText "Locator" (pure . parseLocator) + +-- | Convert a locator to its project locator by removing its revision. +-- This is used for matching locators ignoring version. +toProjectLocator :: Locator -> Locator +toProjectLocator loc = loc{locatorRevision = Nothing} + +-- | Translate all locators in a SourceUnit using the provided translation map. +-- The map keys are target locators (normalized, without version), and values are the replacement locators. +-- When a locator matches a key (by fetcher and project, ignoring version), +-- it is replaced with the value from the map, preserving the original version. +-- The translation is applied to all locators in: +-- - buildImports +-- - sourceDepLocator in each dependency +-- - sourceDepImports in each dependency +translateSourceUnitLocators :: Map Locator Locator -> SourceUnit -> SourceUnit +translateSourceUnitLocators translationMap unit = + unit{sourceUnitBuild = translateBuild <$> sourceUnitBuild unit} + where + translateBuild :: SourceUnitBuild -> SourceUnitBuild + translateBuild build = + build + { buildImports = map translateLocator (buildImports build) + , buildDependencies = map translateDependency (buildDependencies build) + } + translateDependency :: SourceUnitDependency -> SourceUnitDependency + translateDependency dep = + dep + { sourceDepLocator = translateLocator (sourceDepLocator dep) + , sourceDepImports = map translateLocator (sourceDepImports dep) + } + translateLocator :: Locator -> Locator + translateLocator loc = + case Map.lookup (toProjectLocator loc) translationMap of + Nothing -> loc + Just replacement -> + replacement{locatorRevision = locatorRevision loc} diff --git a/test/App/Fossa/ManualDepsSpec.hs b/test/App/Fossa/ManualDepsSpec.hs index 11d2261f5c..c7a9a767ba 100644 --- a/test/App/Fossa/ManualDepsSpec.hs +++ b/test/App/Fossa/ManualDepsSpec.hs @@ -9,6 +9,7 @@ import App.Fossa.Config.Analyze (VendoredDependencyOptions (..)) import App.Fossa.ManualDeps ( CustomDependency (CustomDependency), DependencyMetadata (DependencyMetadata), + ForkAlias (ForkAlias), LinuxReferenceDependency (..), LocatorDependency (..), ManagedReferenceDependency (..), @@ -42,7 +43,7 @@ getTestDataFile :: String -> SpecM a BS.ByteString getTestDataFile name = runIO . BS.readFile $ "test/App/Fossa/testdata/" <> name theWorks :: ManualDependencies -theWorks = ManualDependencies references customs vendors remotes locators +theWorks = ManualDependencies references customs vendors remotes locators forkAliases where references = [ Managed (ManagedReferenceDependency "one" GemType Nothing []) @@ -65,9 +66,11 @@ theWorks = ManualDependencies references customs vendors remotes locators [ LocatorDependencyPlain (Locator "fetcher-1" "one" Nothing) , LocatorDependencyPlain (Locator "fetcher-2" "two" (Just "1.0.0")) ] + forkAliases = + [ForkAlias (Locator "cargo" "my-serde" Nothing) (Locator "cargo" "serde" Nothing) []] theWorksLabeled :: ManualDependencies -theWorksLabeled = ManualDependencies references customs vendors remotes locators +theWorksLabeled = ManualDependencies references customs vendors remotes locators forkAliases where references = [ Managed (ManagedReferenceDependency "one" GemType Nothing [ProvidedPackageLabel "gem-label" ProvidedPackageLabelScopeRevision]) @@ -92,6 +95,8 @@ theWorksLabeled = ManualDependencies references customs vendors remotes locators [ LocatorDependencyStructured (Locator "fetcher-1" "one" Nothing) [ProvidedPackageLabel "locator-dependency-label" ProvidedPackageLabelScopeOrg] , LocatorDependencyStructured (Locator "fetcher-2" "two" (Just "1.0.0")) [ProvidedPackageLabel "locator-dependency-label" ProvidedPackageLabelScopeOrg] ] + forkAliases = + [ForkAlias (Locator "cargo" "my-serde" Nothing) (Locator "cargo" "serde" Nothing) []] theWorksLabels :: Maybe OrgId -> Map Text [ProvidedPackageLabel] theWorksLabels org = @@ -195,6 +200,7 @@ spec = do customDepSpec vendorDepSpec locatorDepSpec + forkAliasSpec describe "getScanCfg" $ do it' "should fail if you try to force a license scan but the FOSSA server does not support it" $ do @@ -344,6 +350,14 @@ locatorDepSpec = do (encodeUtf8 locatorDepWithEmptyDep) "parsing Locator failed, expected String, but encountered Null" +forkAliasSpec :: Spec +forkAliasSpec = do + describe "fork alias" $ do + it "should parse fork alias" $ + case Yaml.decodeEither' (encodeUtf8 forkAliasDep) of + Left err -> expectationFailure $ displayException err + Right yamlDeps -> yamlDeps `shouldBe` forkAliasManualDep + linuxReferenceDep :: Text linuxReferenceDep = [r| @@ -433,6 +447,17 @@ linuxRefManualDep os epoch = mempty mempty mempty + mempty + +forkAliasManualDep :: ManualDependencies +forkAliasManualDep = + ManualDependencies + mempty + mempty + mempty + mempty + mempty + [ForkAlias (Locator "cargo" "my-serde" Nothing) (Locator "cargo" "serde" Nothing) []] customDepWithEmptyVersion :: Text customDepWithEmptyVersion = @@ -538,3 +563,11 @@ locatorDepWithEmptyDep = locator-dependencies: - |] + +forkAliasDep :: Text +forkAliasDep = + [r| +fork-aliases: +- my-fork: cargo+my-serde + base: cargo+serde +|] diff --git a/test/App/Fossa/testdata/the-works-labeled.yml b/test/App/Fossa/testdata/the-works-labeled.yml index e92c896337..c0da050199 100644 --- a/test/App/Fossa/testdata/the-works-labeled.yml +++ b/test/App/Fossa/testdata/the-works-labeled.yml @@ -97,3 +97,7 @@ locator-dependencies: labels: - label: locator-dependency-label scope: org + +fork-aliases: + - my-fork: cargo+my-serde + base: cargo+serde diff --git a/test/App/Fossa/testdata/the-works.json b/test/App/Fossa/testdata/the-works.json index 68e198550b..d0d3fb4ace 100755 --- a/test/App/Fossa/testdata/the-works.json +++ b/test/App/Fossa/testdata/the-works.json @@ -65,5 +65,11 @@ "locator-dependencies": [ "fetcher-1+one", "fetcher-2+two$1.0.0" + ], + "fork-aliases": [ + { + "my-fork": "cargo+my-serde", + "base": "cargo+serde" + } ] } diff --git a/test/App/Fossa/testdata/the-works.yml b/test/App/Fossa/testdata/the-works.yml index 0edd105174..7cdcc67d14 100644 --- a/test/App/Fossa/testdata/the-works.yml +++ b/test/App/Fossa/testdata/the-works.yml @@ -43,3 +43,7 @@ remote-dependencies: locator-dependencies: - "fetcher-1+one" - "fetcher-2+two$1.0.0" + +fork-aliases: + - my-fork: cargo+my-serde + base: cargo+serde diff --git a/test/Srclib/TypesSpec.hs b/test/Srclib/TypesSpec.hs new file mode 100644 index 0000000000..5854c45057 --- /dev/null +++ b/test/Srclib/TypesSpec.hs @@ -0,0 +1,216 @@ +module Srclib.TypesSpec (spec) where + +import Data.Map qualified as Map +import Srclib.Types ( + Locator (..), + SourceUnit (..), + SourceUnitBuild (..), + SourceUnitDependency (..), + toProjectLocator, + translateSourceUnitLocators, + ) +import Test.Hspec (Spec, describe, expectationFailure, it, shouldBe) +import Types (GraphBreadth (Complete)) + +spec :: Spec +spec = do + describe "translateSourceUnitLocators" $ do + it "should translate locators in buildImports" $ do + let myForkLocator = Locator "go" "github.com/myorg/gin" (Just "v1.9.1") + baseLocator = Locator "go" "github.com/gin-gonic/gin" Nothing + translationMap = Map.singleton (toProjectLocator myForkLocator) baseLocator + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [myForkLocator] + , buildDependencies = [] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedImports = buildImports <$> sourceUnitBuild translated + + translatedImports `shouldBe` Just [Locator "go" "github.com/gin-gonic/gin" (Just "v1.9.1")] + + it "should translate locators in sourceDepLocator" $ do + let myForkLocator = Locator "go" "github.com/myorg/testify" (Just "v1.8.4") + baseLocator = Locator "go" "github.com/stretchr/testify" Nothing + translationMap = Map.singleton (toProjectLocator myForkLocator) baseLocator + dep = SourceUnitDependency myForkLocator [] + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [] + , buildDependencies = [dep] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedDeps = buildDependencies <$> sourceUnitBuild translated + + case translatedDeps of + Just [translatedDep] -> + sourceDepLocator translatedDep `shouldBe` Locator "go" "github.com/stretchr/testify" (Just "v1.8.4") + _ -> expectationFailure "Expected exactly one dependency" + + it "should translate locators in sourceDepImports" $ do + let myForkLocator = Locator "go" "github.com/myorg/gin" (Just "v1.9.1") + baseLocator = Locator "go" "github.com/gin-gonic/gin" Nothing + translationMap = Map.singleton (toProjectLocator myForkLocator) baseLocator + dep = SourceUnitDependency (Locator "go" "other" Nothing) [myForkLocator] + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [] + , buildDependencies = [dep] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedDeps = buildDependencies <$> sourceUnitBuild translated + + case translatedDeps of + Just [translatedDep] -> + sourceDepImports translatedDep `shouldBe` [Locator "go" "github.com/gin-gonic/gin" (Just "v1.9.1")] + _ -> expectationFailure "Expected exactly one dependency" + + it "should preserve revision when translating" $ do + let myForkLocator = Locator "go" "github.com/myorg/gin" (Just "v1.9.1") + baseLocator = Locator "go" "github.com/gin-gonic/gin" Nothing + translationMap = Map.singleton (toProjectLocator myForkLocator) baseLocator + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [myForkLocator] + , buildDependencies = [] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedImports = buildImports <$> sourceUnitBuild translated + + -- The revision from the original locator should be preserved + translatedImports `shouldBe` Just [Locator "go" "github.com/gin-gonic/gin" (Just "v1.9.1")] + + it "should not translate locators that don't match the map" $ do + let myForkLocator = Locator "go" "github.com/myorg/gin" (Just "v1.9.1") + baseLocator = Locator "go" "github.com/gin-gonic/gin" Nothing + otherLocator = Locator "go" "github.com/other/pkg" (Just "v1.0.0") + translationMap = Map.singleton (toProjectLocator myForkLocator) baseLocator + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [myForkLocator, otherLocator] + , buildDependencies = [] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedImports = buildImports <$> sourceUnitBuild translated + + -- myForkLocator should be translated to baseLocator, otherLocator should remain unchanged + translatedImports `shouldBe` Just [Locator "go" "github.com/gin-gonic/gin" (Just "v1.9.1"), otherLocator] + + it "should match locators ignoring version" $ do + let myForkLocatorV1 = Locator "go" "github.com/myorg/gin" (Just "v1.9.1") + myForkLocatorV2 = Locator "go" "github.com/myorg/gin" (Just "v2.0.0") + baseLocator = Locator "go" "github.com/gin-gonic/gin" Nothing + translationMap = Map.singleton (toProjectLocator myForkLocatorV1) baseLocator + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + ( Just + SourceUnitBuild + { buildArtifact = "default" + , buildSucceeded = True + , buildImports = [myForkLocatorV2] + , buildDependencies = [] + } + ) + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + let translatedImports = buildImports <$> sourceUnitBuild translated + + -- Should match myForkLocatorV2 even though it has a different version, because we match by fetcher+project only + translatedImports `shouldBe` Just [Locator "go" "github.com/gin-gonic/gin" (Just "v2.0.0")] + + it "should handle SourceUnit without build" $ do + let translationMap = Map.empty + sourceUnit = + SourceUnit + "test" + "go" + "go.mod" + Nothing + Complete + [] + [] + Nothing + Nothing + + let translated = translateSourceUnitLocators translationMap sourceUnit + + -- Should remain unchanged + translated `shouldBe` sourceUnit