import org.apache.maven.artifact.versioning.ComparableVersion; import org.json.JSONArray; import org.json.JSONObject; import java.io.*; import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; /** * Script for generating a list of maven version comparison testdata based off * every version mentioned in the OSV Maven database, sorted using the native * Maven implementation. *

* To run this, you need to ensure copies of the following libraries are present * on the class path: * *

* The easiest way to do this is by putting the jars into a lib subfolder and then running: * * java -cp generators/lib/* generators/GenerateMavenVersions.java * */ public class GenerateMavenVersions { /** * An array of version comparisons that are known to be unsupported and so * should be commented out in the generated fixture. *

* Generally this is because the native implementation has a suspected bug * that causes the comparison to return incorrect results, and so supporting * such comparisons in the detector would in fact be wrong. */ private static final String[] UNSUPPORTED_COMPARISONS = { "0.0.0-2021-07-06T00-28-13-573087f7 < 0.0.0-2021-07-06T01-14-42-efe42242", "0.0.0-2021-12-06T00-08-57-89a33731 < 0.0.0-2021-12-06T01-21-56-e3888760", "0.0.0-2022-02-01T00-45-53-0300684a < 0.0.0-2022-02-01T05-45-16-7258ece0", "0.0.0-2022-02-28T00-18-39-7fe0d845 < 0.0.0-2022-02-28T04-15-47-83c97ebe", "0.0.0-2022-04-29T00-08-11-7086a3ec < 0.0.0-2022-04-29T01-20-09-b424f986", "0.0.0-2022-06-14T00-21-33-f21869a7 < 0.0.0-2022-06-14T02-56-29-1db980e0", "0.0.0-2022-08-16T00-14-19-aeae3dc3 < 0.0.0-2022-08-16T10-34-26-7a56f709", "0.0.0-2022-08-22T00-46-32-4652d3db < 0.0.0-2022-08-22T06-46-40-e7409ac5", "0.0.0-2022-10-31T00-42-12-322ba6b9 < 0.0.0-2022-10-31T01-23-06-c6652489", "0.0.0-2022-10-31T07-00-43-71eccd49 < 0.0.0-2022-10-31T07-05-43-97874976", "0.0.0-2022-12-01T00-02-29-fe8d6705 < 0.0.0-2022-12-01T01-56-22-5b442198", "0.0.0-2022-12-18T00-44-34-a222f475 < 0.0.0-2022-12-18T01-45-19-fec81751", "0.0.0-2023-03-20T00-52-15-4b4c0e7 < 0.0.0-2023-03-20T01-49-44-80e3135" }; public static boolean isUnsupportedComparison(String line) { return Arrays.stream(UNSUPPORTED_COMPARISONS).anyMatch(line::equals); } public static String uncomment(String line) { if(line.startsWith("#")) { return line.substring(1); } if(line.startsWith("//")) { return line.substring(2); } return line; } public static String downloadMavenDb() throws IOException { URL website = new URL("https://osv-vulnerabilities.storage.googleapis.com/Maven/all.zip"); String file = "./maven-db.zip"; ReadableByteChannel rbc = Channels.newChannel(website.openStream()); try(FileOutputStream fos = new FileOutputStream(file)) { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } return file; } public static Map> fetchPackageVersions() throws IOException { String dbPath = downloadMavenDb(); List osvs = loadOSVs(dbPath); Map> packages = new HashMap<>(); osvs.forEach(osv -> osv.getJSONArray("affected").forEach(aff -> { JSONObject affected = (JSONObject) aff; if(!affected.has("package") || affected.getJSONObject("package").getString("ecosystem").equals("Maven")) { return; } String pkgName = affected.getJSONObject("package").getString("name"); if(!affected.has("versions")) { return; } JSONArray versions = affected.getJSONArray("versions"); packages.putIfAbsent(pkgName, new ArrayList<>()); if(versions.isEmpty()) { return; } versions.forEach(version -> packages.get(pkgName).add((String) version)); })); packages.forEach((key, _ignore) -> packages.put( key, packages.get(key) .stream() .distinct() .sorted(Comparator.comparing(ComparableVersion::new)) .collect(Collectors.toList()) )); return packages; } public static List loadOSVs(String pathToDbZip) throws IOException { List osvs = new ArrayList<>(); try(ZipFile zipFile = new ZipFile(pathToDbZip)) { Enumeration entries = zipFile.entries(); while(entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); InputStream stream = zipFile.getInputStream(entry); BufferedReader streamReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); StringBuilder responseStrBuilder = new StringBuilder(); String inputStr; while((inputStr = streamReader.readLine()) != null) { responseStrBuilder.append(inputStr); } osvs.add(new JSONObject(responseStrBuilder.toString())); } } return osvs; } public static void writeToFile(String outfile, List lines) throws IOException { try(PrintWriter writer = new PrintWriter(outfile, StandardCharsets.UTF_8)) { lines.forEach(writer::println); } } public static boolean compareVers(String version1, String op, String version2) { ComparableVersion v1 = new ComparableVersion(version1); ComparableVersion v2 = new ComparableVersion(version2); int r = v1.compareTo(v2); if(op.equals("=")) { return r == 0; } if(op.equals("<")) { return r < 0; } if(op.equals(">")) { return r > 0; } throw new RuntimeException("unsupported comparison operator " + op); } public static boolean compareVersions(List lines, String select) { boolean didAnyFail = false; for(String line : lines) { line = line.trim(); if(line.isEmpty() || line.startsWith("#") || line.startsWith("//")) { String maybeUnsupported = uncomment(line).trim(); if(isUnsupportedComparison(maybeUnsupported)) { System.out.printf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybeUnsupported); } continue; } String[] parts = line.split(" "); String v1 = parts[0]; String op = parts[1]; String v2 = parts[2]; boolean r = compareVers(v1, op, v2); if(!r) { didAnyFail = true; } if(select.equals("failures") && r) { continue; } if(select.equals("successes") && !r) { continue; } String color = r ? "\033[92m" : "\033[91m"; String rs = r ? "T" : "F"; System.out.printf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line); } return didAnyFail; } public static boolean compareVersionsInFile(String filepath, String select) throws IOException { List lines = new ArrayList<>(); try(BufferedReader br = new BufferedReader(new FileReader(filepath))) { String line = br.readLine(); while(line != null) { lines.add(line); line = br.readLine(); } } return compareVersions(lines, select); } public static List generateVersionCompares(List versions) { return IntStream.range(1, versions.size()).mapToObj(i -> { String currentVersion = versions.get(i); String previousVersion = versions.get(i - 1); String op = compareVers(currentVersion, "=", previousVersion) ? "=" : "<"; String comparison = String.format("%s %s %s", previousVersion, op, currentVersion); if(isUnsupportedComparison(comparison)) { comparison = "# " + comparison; } return comparison; }).collect(Collectors.toList()); } public static List generatePackageCompares(Map> packages) { return packages .values() .stream() .map(GenerateMavenVersions::generateVersionCompares) .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); } public static String getSelectFilter() { // set this to either "failures" or "successes" to only have those comparison results // printed; setting it to anything else will have all comparison results printed String value = System.getenv("VERSION_GENERATOR_PRINT"); if(value == null) { return "failures"; } return value; } public static void main(String[] args) throws IOException { String outfile = "internal/semantic/testdata/maven-versions-generated.txt"; Map> packages = fetchPackageVersions(); writeToFile(outfile, generatePackageCompares(packages)); String show = getSelectFilter(); boolean didAnyFail = compareVersionsInFile(outfile, show); if(didAnyFail) { System.exit(1); } } }