From cb5fa5993fbec9142df4912ff4d3ceac7220f08b Mon Sep 17 00:00:00 2001 From: edoardo Date: Mon, 7 Jun 2021 14:36:13 +0200 Subject: [PATCH] import Debian CPE list --- .../utils/mappers/PurlMapper.java | 62 ++++++++++++++++--- .../utils/parsers/ParserManager.java | 3 +- .../mappers/PurlMapperTest.java | 41 +++++++++++- 3 files changed, 95 insertions(+), 11 deletions(-) diff --git a/src/main/java/eu/fasten/vulnerabilityproducer/utils/mappers/PurlMapper.java b/src/main/java/eu/fasten/vulnerabilityproducer/utils/mappers/PurlMapper.java index 141cba1..a47a8ef 100644 --- a/src/main/java/eu/fasten/vulnerabilityproducer/utils/mappers/PurlMapper.java +++ b/src/main/java/eu/fasten/vulnerabilityproducer/utils/mappers/PurlMapper.java @@ -22,6 +22,7 @@ import com.google.gson.reflect.TypeToken; import eu.fasten.vulnerabilityproducer.utils.PatchFinder; import eu.fasten.vulnerabilityproducer.utils.Vulnerability; +import eu.fasten.vulnerabilityproducer.utils.connections.*; import org.apache.commons.io.FileUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -38,24 +39,68 @@ public class PurlMapper { VersionRanger versionRanger; PatchFinder patchFinder; DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd"); - public HashMap mavenMap; // repo_url -> purl - public HashMap pypiMap; // repo_url -> purl - public HashMap cpeMap; // cpe_base -> repo_url - public HashMap cpe2purl; // cpe_base -> purl + public HashMap mavenMap; // repo_url -> purl + public HashMap pypiMap; // repo_url -> purl + public HashMap cpeMap; // cpe_base -> repo_url + public HashMap debianCpeMap; // cpe_base -> purl + public HashMap cpe2purl; // cpe_base -> purl public String strategy; private final Logger logger = LoggerFactory.getLogger(PurlMapper.class.getName()); public PurlMapper(VersionRanger versionRanger, PatchFinder patchFinder, String pathToMaps, - String inferStrategy) { + String inferStrategy, + JavaHttpClient client) { this.versionRanger = versionRanger; this.patchFinder = patchFinder; this.strategy = inferStrategy; this.mavenMap = loadReposMapFromMemory(pathToMaps + "repo_map_maven.json"); this.pypiMap = loadReposMapFromMemory(pathToMaps + "repo_map_pypi.json"); this.cpeMap = loadReposMapFromMemory(pathToMaps + "cpe_map_repos.json"); - this.cpe2purl = createCpe2PurlMap(cpeMap, mavenMap, pypiMap); + this.debianCpeMap = loadDebianCPEMap(client); + this.cpe2purl = createCpe2PurlMap(cpeMap, mavenMap, pypiMap, debianCpeMap); + } + + /** + * Loads the CPE list from Debian Security Advisories. + * Also handles aliases of those CPEs. + * @return cpe -> purl map. + */ + public HashMap loadDebianCPEMap(JavaHttpClient client) { + var map = new HashMap(); + var listUrl = "https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CPE/list"; + var aliasesUrl = "https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CPE/aliases"; + + var cpeList = client.sendGet(listUrl); + Arrays.stream(cpeList.split("\n")).forEach(line -> { + var info = line.split(";"); + var pkg = info[0]; + var cpe = info[1].replace("/", "2.3:"); + var purl = "pkg:deb/debian/" + pkg; + map.put(cpe, purl); + }); + + var aliasesList = client.sendGet(aliasesUrl); + var aliases = aliasesList.split("\n"); + for (int i = 0; i < aliases.length; i++) { + var line = aliases[i]; + if (line.startsWith("cpe")) { + var cpeGroup = new ArrayList(); + while (line.startsWith("cpe")) { + cpeGroup.add(line.replace("/", "2.3:")); + i += 1; + line = aliases[i]; + } + cpeGroup.forEach(cpeAlias -> { + if (map.containsKey(cpeAlias)) { + cpeGroup.forEach(cpe -> map.put(cpe, map.get(cpeAlias))); + } + }); + } + } + + return map; } private static HashMap loadReposMapFromMemory(String path) { @@ -80,8 +125,9 @@ private static HashMap loadReposMapFromMemory(String path) { */ private HashMap createCpe2PurlMap(HashMap cpeMap, HashMap mavenMap, - HashMap pypiMap) { - var cpe2purl = new HashMap(); + HashMap pypiMap, + HashMap debianCpeMap) { + var cpe2purl = debianCpeMap; // initialize to debian CPE list cpeMap.keySet().forEach(cpe -> { var repo = cpeMap.get(cpe); if (mavenMap.containsKey(repo)) cpe2purl.put(cpe, mavenMap.get(repo)); diff --git a/src/main/java/eu/fasten/vulnerabilityproducer/utils/parsers/ParserManager.java b/src/main/java/eu/fasten/vulnerabilityproducer/utils/parsers/ParserManager.java index 064d8d3..59b3ca8 100644 --- a/src/main/java/eu/fasten/vulnerabilityproducer/utils/parsers/ParserManager.java +++ b/src/main/java/eu/fasten/vulnerabilityproducer/utils/parsers/ParserManager.java @@ -79,7 +79,8 @@ public ParserManager(JavaHttpClient client, this.ossFuzzParser = new OSSFuzzParser(mnt); this.ghParser = new GHParser(client, ghToken, versionRanger, mnt + "/trackers/ghcursor.txt"); this.patchFinder = new PatchFinder(mongoDatabase, client, ghToken); - this.purlMapper = new PurlMapper(versionRanger, patchFinder, mnt + "/datasets/purl_maps/", inferStrategy); + this.purlMapper = new PurlMapper(versionRanger, patchFinder, + mnt + "/datasets/purl_maps/", inferStrategy, client); this.nvdParser = new NVDParser(new JSONParser(), client, this.purlMapper, mnt); } diff --git a/src/test/java/eu/fasten/vulnerabilityproducer/mappers/PurlMapperTest.java b/src/test/java/eu/fasten/vulnerabilityproducer/mappers/PurlMapperTest.java index 8ef3db0..36cac7f 100644 --- a/src/test/java/eu/fasten/vulnerabilityproducer/mappers/PurlMapperTest.java +++ b/src/test/java/eu/fasten/vulnerabilityproducer/mappers/PurlMapperTest.java @@ -19,9 +19,10 @@ package eu.fasten.vulnerabilityproducer.mappers; import eu.fasten.vulnerabilityproducer.utils.*; +import eu.fasten.vulnerabilityproducer.utils.connections.*; import eu.fasten.vulnerabilityproducer.utils.mappers.PurlMapper; import eu.fasten.vulnerabilityproducer.utils.mappers.VersionRanger; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.mockito.Mockito; import java.util.*; @@ -34,7 +35,34 @@ public class PurlMapperTest { VersionRanger vrMock = Mockito.mock(VersionRanger.class); PatchFinder pfMock = Mockito.mock(PatchFinder.class); - PurlMapper purlMapper = new PurlMapper(vrMock, pfMock, "./src/test/resources/purl_maps/", "both"); + JavaHttpClient client = Mockito.mock(JavaHttpClient.class); + PurlMapper purlMapper; + + @BeforeEach + public void init() { + var listUrl = "https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CPE/list"; + var aliasesUrl = "https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CPE/aliases"; + + var cpeList = "" + + "a2ps;cpe:/a:gnu:a2ps\n" + + "abc2ps;cpe:/a:abc2ps:abc2ps\n" + + "asterisk;cpe:/a:digium:asterisk\n"; + var aliasesList = "" + + "#\n" + + "# Comments\n" + + "#\n" + + "cpe:/a:asterisk:asterisk\n" + + "cpe:/a:asterisk:open_source\n" + + "cpe:/a:asterisk:p_b_x\n" + + "cpe:/a:digium:asterisk\n" + + "cpe:/a:asterisk:opensource\n" + + "\n" + + "# End comment\n"; + when(client.sendGet(listUrl)).thenReturn(cpeList); + when(client.sendGet(aliasesUrl)).thenReturn(aliasesList); + + purlMapper = new PurlMapper(vrMock, pfMock, "./src/test/resources/purl_maps/", "both", client); + } @Test public void getBasePurlTest() { @@ -52,6 +80,15 @@ public void creationCPE2PURLMap() { assertEquals("pkg:maven/org.google.guava/guava", purlMapper.cpe2purl.get("cpe:2.3:a:google:guava")); } + @Test + public void importDebianCPE() { + assertTrue(purlMapper.cpe2purl.containsKey("cpe:2.3:a:gnu:a2ps")); + assertEquals("pkg:deb/debian/a2ps", purlMapper.cpe2purl.get("cpe:2.3:a:gnu:a2ps")); + + assertTrue(purlMapper.cpe2purl.containsKey("cpe:2.3:a:asterisk:opensource")); + assertEquals("pkg:deb/debian/asterisk", purlMapper.cpe2purl.get("cpe:2.3:a:asterisk:opensource")); + } + // repo2purl @Test public void inferPurlPyPIMap() {