Merge branch 'merge-notice' into 'main'

Adding feature to allow merging notice files

Closes #1

See merge request !3

features/fossa-with-cache/fossaWithCacheScript.feature

@@ -20,6 +20,23 @@ Feature: Main Cached FOSSA Script
     And the script does not build a FossaAPI
     And the script exits with status 1
 
+  Scenario: Script exits early if merge-with file is not specified:
+    When running main fossa-with-cache script with "--merge-with"
+    Then the script does not build a Config
+    And the script does not build a Store
+    And the script does not build a FossaCLI
+    And the script does not build a FossaAPI
+    And the script exits with status 2
+
+  Scenario: Script exits early if merge-with file does not exist:
+    When running main fossa-with-cache script with "--merge-with NOTICE"
+    Then logs include "Could not find merge file: NOTICE"
+    And the script does not build a Config
+    And the script does not build a Store
+    And the script does not build a FossaCLI
+    And the script does not build a FossaAPI
+    And the script exits with status 1
+
   Scenario: Main Flow
     When running main fossa-with-cache script
 
@@ -40,12 +57,22 @@ Feature: Main Cached FOSSA Script
 
     And the script builds a FossaConfig from "attribution-config.json"
     And the script builds a FossaNotice from "generated-raw-NOTICE"
+    And the script does not merge FossaNotice objects
     And the script outputs the formatted script to "generated-clean-NOTICE"
     And logs include "Cleaning / Formatting the NOTICE ..."
 
     And the status is set to "generated"
     And the script exits with status 0
 
+  Scenario: Merging Flow
+    Given file "NOTICE" exists with content "Merge Content"
+    When running main fossa-with-cache script with "--merge-with NOTICE"
+    Then logs include "Will merge generated NOTICE file with: NOTICE"
+    And the script outputs "Merge Content" to "merged-with-NOTICE"
+    And the script builds a FossaNotice with content "Merge Content"
+    And the script merges FossaNotice objects
+    And the script exits with status 0
+
   Scenario: Caching Disabled
     Given caching is not allowed
     When running main fossa-with-cache script

features/fossa-with-cache/noticeMerging.feature

+@notice
+Feature: NOTICE Differences
+
+  Merging FOSSA NOTICEs
+
+  Scenario: Attributions from both are combined
+    Given the NOTICE file contents are:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - basis
+      - shared
+      """
+
+    When the NOTICE is merged with:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - shared
+      - merged
+      """
+
+    Then the formatted output is:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - basis
+      - merged
+      - shared
+      """
+
+  Scenario: License Types from both are combined
+    Given the NOTICE file contents are:
+      """
+      ========================================================================
+      License Type A
+      ========================================================================
+      - basis
+
+      ========================================================================
+      License Type B
+      ========================================================================
+      - shared
+      """
+
+    When the NOTICE is merged with:
+      """
+      ========================================================================
+      License Type C
+      ========================================================================
+      - merged
+
+      ========================================================================
+      License Type B
+      ========================================================================
+      - shared
+      """
+
+    Then the formatted output is:
+      """
+      ========================================================================
+      License Type A
+      ========================================================================
+      - basis
+
+      ========================================================================
+      License Type B
+      ========================================================================
+      - shared
+
+      ========================================================================
+      License Type C
+      ========================================================================
+      - merged
+      """
+
+  Scenario: URLs from both are combined
+    Given the NOTICE file contents are:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - attribution (from https://alpha.com, https://shared.com)
+      """
+
+    When the NOTICE is merged with:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - attribution (from https://omega.com, https://shared.com)
+      """
+
+    Then the formatted output is:
+      """
+      ========================================================================
+      License Type
+      ========================================================================
+      - attribution (from https://alpha.com, https://omega.com, https://shared.com)
+      """

features/steps/fossaWithCacheSteps.py

 import logging
 import time
 import tempfile
+import sys
 
 import pint
 
@@ -16,6 +17,8 @@ from cacheUtils.fossaCLI import FossaCLI
 from cacheUtils.fossaAPI import FossaAPI
 from cacheUtils.fossaNotice import FossaNotice, FossaConfig
 
+from mockFilesystem import mockFilesystem
+
 class SysExitCalled(BaseException):
 	pass
 
@@ -47,9 +50,15 @@ def createExceptions(context, component, method, msg):
 	})
 	context.componentExceptions = exceptions
 
+@given('file "{file}" exists with content "{content}"')
+def addFile(context, file, content):
+	if not hasattr(context, 'files'):
+		context.files = {}
+
+	context.files[file] = content
+
 # --------------------------------------------------------------------------------
 
-@when("running main fossa-with-cache script")
 @patch('sys.exit')
 @patch('time.sleep')
 @patch('tempfile.TemporaryDirectory', auto_spec=True)
@@ -100,11 +109,29 @@ def runScript(context, FossaConfig, FossaNotice, FossaAPI, FossaCLI, Store, Conf
 
 	try:
 		scriptContents = open("src/fossa-with-cache.py").read()
-		exec(scriptContents)
+
+		if hasattr(context, 'files'):
+			with patch('builtins.open') as openPatch:
+				mockFilesystem(context.files, open=openPatch);
+				exec(scriptContents)
+
+		else:
+			exec(scriptContents)
 
 	except SysExitCalled:
 		pass
 
+@when("running main fossa-with-cache script")
+def runScriptWithNoArgs(context):
+	sys.argv = ['fossa-with-cache']
+	runScript(context)
+
+@when('running main fossa-with-cache script with "{args}"')
+def runScriptWithArgs(context, args):
+	sys.argv = ['fossa-with-cache'] + args.split(' ')
+	print('steps argv = %r' % sys.argv)
+	runScript(context)
+
 # --------------------------------------------------------------------------------
 # Basic Exit Status
 
@@ -164,14 +191,28 @@ def checkFossaConfig(context, file):
 		context.store.storeRoot.readJSON.return_value)
 
 @then('the script builds a FossaNotice from "{file}"')
-def checkFossaNotice(context, file):
+def checkFossaNoticeFromFile(context, file):
 	context.store.instanceRoot.readText.assert_called_with(file)
 
-	context.FossaNotice.assert_called_once()
-	context.FossaNotice.assert_called_with(
+	context.FossaNotice.assert_any_call(
 		context.store.instanceRoot.readText.return_value,
 	    context.fossaConfig)
 
+@then('the script builds a FossaNotice with content "{content}"')
+def checkFossaNoticeFromContent(context, content):
+	context.FossaNotice.assert_any_call(
+		content,
+	    context.fossaConfig)
+
+@then('the script merges FossaNotice objects')
+def checkFossaNoticeMerge(context):
+	context.FossaNotice.return_value.merge.assert_called_with(
+		context.FossaNotice.return_value)
+
+@then('the script does not merge FossaNotice objects')
+def checkFossaNoticeNoMerge(context):
+	context.FossaNotice.return_value.merge.assert_not_called()
+
 @then("the script does not build a {cls}")
 def checkAbsence(context, cls):
 	getattr(context, cls).assert_not_called()
@@ -221,7 +262,11 @@ def checkNoCacheSearch(context):
 
 @then('the script outputs the formatted script to "{file}"')
 def checkOutput(context, file):
-	context.store.instanceRoot.outputText.assert_called_with(file, str(context.fossaNotice))
+	context.store.instanceRoot.outputText.assert_any_call(file, str(context.fossaNotice))
+
+@then('the script outputs "{content}" to "{file}"')
+def checkOutput(context, file, content):
+	context.store.instanceRoot.outputText.assert_any_call(file, content)
 
 @then('the status is set to "{status}"')
 def checkStatus(context, status):

features/steps/mockFilesystem.py

 from unittest.mock import mock_open
 
 def mockFilesystem(files, **mockObjects):
-	def _openImpl(name, mode):
+	def _openImpl(name, mode, **kwargs):
 		if mode == 'r':
 			if name in files:
 				return mock_open(read_data=files[name]).return_value
 			else:
 				raise OSError("File does not exist")
 
-		elif mode == 'w':
-			files[name] = ""
+		elif mode == 'w' or mode == 'a':
+			if name not in files or mode == 'w':
+				files[name] = ""
 
 			def _writeImpl(str):
 				files[name] += str
@@ -19,7 +20,7 @@ def mockFilesystem(files, **mockObjects):
 			return mockFile
 
 		else:
-			raise AssertionError("Invalid mode (only 'r' and 'w' supported)")
+			raise AssertionError("Invalid mode (only 'r', 'w', 'a' supported)")
 
 	# ----------------------------------------
 

features/steps/noticeSteps.py

@@ -42,7 +42,13 @@ def buildNotice(context):
 	context.basis = FossaNotice(context.text, context.configObj)
 
 # ----------------------------------------
-#
+
+@when('the NOTICE is merged with')
+def mergeNotice(context):
+	context.mergeNotice = FossaNotice(context.text, context.configObj)
+	context.notice.merge(context.mergeNotice)
+
+# ----------------------------------------
 
 @then('the formatted output is')
 def checkOutput(context):

src/.gitignore

+*.egg-info

src/cacheUtils/fossaNotice.py

@@ -72,6 +72,7 @@ class FossaSection:
 		self.title = title
 		self.header = []
 		self.attributions = {}
+		self.config = config
 		self.footer = []
 
 		while len(lines) > 0 and lines[-1] == '':
@@ -83,15 +84,13 @@ class FossaSection:
 			m = attributionRegex.match(line)
 
 			if m:
-				name = config.preferredName(m.groups()[0].strip())
-				if name not in self.attributions:
-					self.attributions[name] = config.knownURLs(name)
-
+				name = m.groups()[0].strip()
 				if m.groups()[2]:
-					url = m.groups()[2].strip()
-					for part in url.split(', '):
-						if len(part.strip()) > 0:
-							self.attributions[name].add(part.strip())
+					urls = m.groups()[2].strip().split(', ')
+					self._addAttribution(name, urls)
+
+				else:
+					self._addAttribution(name)
 
 			elif len(self.attributions) > 0:
 				self.footer.append(line)
@@ -137,6 +136,21 @@ class FossaSection:
 
 		return None
 
+	def merge(self, mergeSection):
+		for mergeName in mergeSection.attributions.keys():
+			name = self.config.preferredName(mergeName)
+			urls = mergeSection.attributions[mergeName]
+			self._addAttribution(name, urls)
+
+	def _addAttribution(self, name, urls = []):
+		preferredName = self.config.preferredName(name)
+		if preferredName not in self.attributions:
+			self.attributions[preferredName] = self.config.knownURLs(preferredName)
+
+		for url in urls:
+			if len(url.strip()) > 0:
+				self.attributions[preferredName].add(url.strip())
+
 	def __str__(self):
 		lines = bannerLines(self.title)
 		lines.extend(self.header)
@@ -194,6 +208,15 @@ class FossaNotice:
 
 		return None
 
+	def merge(self, mergeNotice):
+		for title in mergeNotice.sections.keys():
+			dummySection = FossaSection(title, [], self.config)
+			selfSection = self.sections.get(title, dummySection)
+			mergeSection = mergeNotice.sections.get(title, dummySection)
+
+			selfSection.merge(mergeSection)
+			self.sections[title] = selfSection
+
 	def _loadNotice(self, contents):
 		self.sections = {};
 

src/fossa-with-cache.py

 #!/usr/bin/env python3
 
+import argparse
 import sys
 import subprocess
 import time
 import tempfile
 import logging
+import os
 
 import pint
 
@@ -33,6 +35,24 @@ ch.setFormatter(formatter)
 log.addHandler(fh)
 log.addHandler(ch)
 
+# ----------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument('--merge-with', nargs=1, help='a NOTICE file to merge the generated content with')
+
+args = parser.parse_args()
+mergeContent = None
+
+if args.merge_with is not None:
+	try:
+		with open(args.merge_with[0], 'r') as f:
+			mergeContent = f.read()
+
+		log.info("Will merge generated NOTICE file with: %s", args.merge_with[0])
+
+	except:
+		log.error('Could not find merge file: %s', args.merge_with[0])
+		sys.exit(1)
+
 # ----------------------------------------
 # Some global objects to help the script run
 
@@ -89,6 +109,13 @@ try:
 	config = FossaConfig(store.storeRoot.readJSON('attribution-config.json'))
 
 	notice = FossaNotice(noticeFileContents, config)
+
+	if mergeContent is not None:
+		store.instanceRoot.outputText('merged-with-NOTICE', mergeContent)
+
+		mergeNotice = FossaNotice(mergeContent, config)
+		notice.merge(mergeNotice)
+
 	store.instanceRoot.outputText('generated-clean-NOTICE', str(notice))
 
 	# ----------------------------------------