From a19adf283f60b70268b2bd96165fcee779bb0a4e Mon Sep 17 00:00:00 2001 From: FalsePattern <30945458+FalsePattern@users.noreply.github.com> Date: Sat, 6 Aug 2022 14:18:18 +0200 Subject: [PATCH] parallel library downloads --- .../lib/dependencies/DependencyLoader.java | 114 ++++++++++++++++- .../lib/dependencies/Library.java | 119 ++++++++++++++++++ .../dependencies/DependencyLoaderImpl.java | 46 +++++-- .../lib/updates/UpdateChecker.java | 17 +-- 4 files changed, 278 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/falsepattern/lib/dependencies/Library.java diff --git a/src/main/java/com/falsepattern/lib/dependencies/DependencyLoader.java b/src/main/java/com/falsepattern/lib/dependencies/DependencyLoader.java index 23b7df3..517d2b3 100644 --- a/src/main/java/com/falsepattern/lib/dependencies/DependencyLoader.java +++ b/src/main/java/com/falsepattern/lib/dependencies/DependencyLoader.java @@ -20,11 +20,14 @@ */ package com.falsepattern.lib.dependencies; +import com.falsepattern.lib.DeprecationDetails; import com.falsepattern.lib.StableAPI; +import com.falsepattern.lib.internal.FalsePatternLib; import com.falsepattern.lib.internal.impl.dependencies.DependencyLoaderImpl; -import lombok.Builder; import lombok.NonNull; +import java.util.concurrent.CompletableFuture; + @StableAPI(since = "0.6.0") public class DependencyLoader { @@ -34,10 +37,117 @@ public class DependencyLoader { DependencyLoaderImpl.addMavenRepo(url); } - @Builder + @StableAPI.Expose(since = "0.10.0") + public static CompletableFuture loadLibrariesAsync(Library... libraries) { + return DependencyLoaderImpl.loadLibrariesAsync(libraries); + } + + @StableAPI.Expose(since = "0.10.0") + public static void loadLibraries(Library... libraries) { + DependencyLoaderImpl.loadLibraries(libraries); + } + + @Deprecated + @DeprecationDetails(deprecatedSince = "0.10.0") @StableAPI.Expose public static void loadLibrary(@NonNull String loadingModId, @NonNull String groupId, @NonNull String artifactId, @NonNull Version minVersion, Version maxVersion, @NonNull Version preferredVersion, String regularSuffix, String devSuffix) { + FalsePatternLib.getLog().warn(DependencyLoader.class.getName() + ".loadLibrary is deprecated and will be removed in FalsePatternLib 0.11! Use loadLibraries instead!"); DependencyLoaderImpl.loadLibrary(loadingModId, groupId, artifactId, minVersion, maxVersion, preferredVersion, regularSuffix, devSuffix); } + + @DeprecationDetails(deprecatedSince = "0.10.0") + @StableAPI.Expose + @Deprecated + public static VoidBuilder builder() { + return new VoidBuilder(); + } + + @StableAPI(since = "0.6.0") + @DeprecationDetails(deprecatedSince = "0.10.0") + public static class VoidBuilder { + private String loadingModId; + private String groupId; + private String artifactId; + private Version minVersion; + private Version maxVersion; + private Version preferredVersion; + private String regularSuffix; + private String devSuffix; + + @Deprecated + VoidBuilder() { + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder loadingModId(@NonNull String loadingModId) { + this.loadingModId = loadingModId; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder groupId(@NonNull String groupId) { + this.groupId = groupId; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder artifactId(@NonNull String artifactId) { + this.artifactId = artifactId; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder minVersion(@NonNull Version minVersion) { + this.minVersion = minVersion; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder maxVersion(Version maxVersion) { + this.maxVersion = maxVersion; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder preferredVersion(@NonNull Version preferredVersion) { + this.preferredVersion = preferredVersion; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder regularSuffix(String regularSuffix) { + this.regularSuffix = regularSuffix; + return this; + } + + @StableAPI.Expose + @Deprecated + public VoidBuilder devSuffix(String devSuffix) { + this.devSuffix = devSuffix; + return this; + } + + @StableAPI.Expose + @Deprecated + public void build() { + DependencyLoader.loadLibrary(loadingModId, groupId, artifactId, minVersion, maxVersion, preferredVersion, + regularSuffix, devSuffix); + } + + @Override + public String toString() { + return "DependencyLoader.VoidBuilder(loadingModId=" + this.loadingModId + ", groupId=" + this.groupId + + ", artifactId=" + this.artifactId + ", minVersion=" + this.minVersion + ", maxVersion=" + + this.maxVersion + ", preferredVersion=" + this.preferredVersion + ", regularSuffix=" + + this.regularSuffix + ", devSuffix=" + this.devSuffix + ")"; + } + } } diff --git a/src/main/java/com/falsepattern/lib/dependencies/Library.java b/src/main/java/com/falsepattern/lib/dependencies/Library.java new file mode 100644 index 0000000..b4495c2 --- /dev/null +++ b/src/main/java/com/falsepattern/lib/dependencies/Library.java @@ -0,0 +1,119 @@ +package com.falsepattern.lib.dependencies; + +import com.falsepattern.lib.StableAPI; +import lombok.NonNull; + +@StableAPI(since = "0.10.0") +public class Library { + @StableAPI.Expose + @NonNull public final String loadingModId; + @StableAPI.Expose + @NonNull public final String groupId; + @StableAPI.Expose + @NonNull public final String artifactId; + @StableAPI.Expose + @NonNull public final Version minVersion; + @StableAPI.Expose + public final Version maxVersion; + @StableAPI.Expose + @NonNull public final Version preferredVersion; + @StableAPI.Expose + public final String regularSuffix; + @StableAPI.Expose + public final String devSuffix; + + @StableAPI.Expose + public Library(@NonNull String loadingModId, @NonNull String groupId, @NonNull String artifactId, @NonNull Version minVersion, Version maxVersion, @NonNull Version preferredVersion, String regularSuffix, String devSuffix) { + this.loadingModId = loadingModId; + this.groupId = groupId; + this.artifactId = artifactId; + this.minVersion = minVersion; + this.maxVersion = maxVersion; + this.preferredVersion = preferredVersion; + this.regularSuffix = regularSuffix; + this.devSuffix = devSuffix; + } + + @StableAPI.Expose + public static LibraryBuilder builder() { + return new LibraryBuilder(); + } + + @StableAPI(since = "0.10.0") + public static class LibraryBuilder { + private String loadingModId; + private String groupId; + private String artifactId; + private Version minVersion; + private Version maxVersion; + private Version preferredVersion; + private String regularSuffix; + private String devSuffix; + + @StableAPI.Internal + LibraryBuilder() { + } + + @StableAPI.Expose + public LibraryBuilder loadingModId(@NonNull String loadingModId) { + this.loadingModId = loadingModId; + return this; + } + + @StableAPI.Expose + public LibraryBuilder groupId(@NonNull String groupId) { + this.groupId = groupId; + return this; + } + + @StableAPI.Expose + public LibraryBuilder artifactId(@NonNull String artifactId) { + this.artifactId = artifactId; + return this; + } + + @StableAPI.Expose + public LibraryBuilder minVersion(@NonNull Version minVersion) { + this.minVersion = minVersion; + return this; + } + + @StableAPI.Expose + public LibraryBuilder maxVersion(Version maxVersion) { + this.maxVersion = maxVersion; + return this; + } + + @StableAPI.Expose + public LibraryBuilder preferredVersion(@NonNull Version preferredVersion) { + this.preferredVersion = preferredVersion; + return this; + } + + @StableAPI.Expose + public LibraryBuilder regularSuffix(String regularSuffix) { + this.regularSuffix = regularSuffix; + return this; + } + + @StableAPI.Expose + public LibraryBuilder devSuffix(String devSuffix) { + this.devSuffix = devSuffix; + return this; + } + + @StableAPI.Expose + public Library build() { + return new Library(loadingModId, groupId, artifactId, minVersion, maxVersion, preferredVersion, + regularSuffix, devSuffix); + } + + @Override + public String toString() { + return "Library.LibraryBuilder(loadingModId=" + this.loadingModId + ", groupId=" + this.groupId + + ", artifactId=" + this.artifactId + ", minVersion=" + this.minVersion + ", maxVersion=" + + this.maxVersion + ", preferredVersion=" + this.preferredVersion + ", regularSuffix=" + + this.regularSuffix + ", devSuffix=" + this.devSuffix + ")"; + } + } +} diff --git a/src/main/java/com/falsepattern/lib/internal/impl/dependencies/DependencyLoaderImpl.java b/src/main/java/com/falsepattern/lib/internal/impl/dependencies/DependencyLoaderImpl.java index 9e5c7a7..bbfcf38 100644 --- a/src/main/java/com/falsepattern/lib/internal/impl/dependencies/DependencyLoaderImpl.java +++ b/src/main/java/com/falsepattern/lib/internal/impl/dependencies/DependencyLoaderImpl.java @@ -21,12 +21,14 @@ package com.falsepattern.lib.internal.impl.dependencies; import com.falsepattern.lib.dependencies.DependencyLoader; +import com.falsepattern.lib.dependencies.Library; import com.falsepattern.lib.dependencies.Version; import com.falsepattern.lib.internal.FalsePatternLib; import com.falsepattern.lib.internal.Internet; import com.falsepattern.lib.internal.Tags; import com.falsepattern.lib.internal.config.LibraryConfig; import com.falsepattern.lib.util.FileUtil; +import io.netty.util.internal.ConcurrentSet; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -45,27 +47,40 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; import java.security.MessageDigest; -import java.util.HashMap; -import java.util.HashSet; +import java.util.ArrayList; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; public class DependencyLoaderImpl { /** - * Dependency checksum formats in a decreasing order of quality. + * Library checksum formats in a decreasing order of quality. */ private static final String[] CHECKSUM_TYPES = new String[]{"sha512", "sha256", "sha1", "md5"}; - private static final Map loadedLibraries = new HashMap<>(); - private static final Map loadedLibraryMods = new HashMap<>(); - private static final Set mavenRepositories = new HashSet<>(); - private static final Logger log = LogManager.getLogger(Tags.MODNAME + " Dependency Downloader"); + private static final Map loadedLibraries = new ConcurrentHashMap<>(); + private static final Map loadedLibraryMods = new ConcurrentHashMap<>(); + private static final Set mavenRepositories = new ConcurrentSet<>(); + private static final Logger log = LogManager.getLogger(Tags.MODNAME + " Library Downloader"); + + private static final AtomicLong counter = new AtomicLong(0); + private static final ExecutorService executor = Executors.newCachedThreadPool(r -> { + val thread = new Thread(r); + thread.setDaemon(true); + thread.setName("Dependency Download Thread " + counter.incrementAndGet()); + return thread; + }); public static void addMavenRepo(String url) { mavenRepositories.add(url); } + @Deprecated public static void loadLibrary(@NonNull String loadingModId, @NonNull String groupId, @NonNull String artifactId, @NonNull Version minVersion, Version maxVersion, @NonNull Version preferredVersion, String regularSuffix, String devSuffix) { new DependencyLoadTask(loadingModId, groupId, artifactId, minVersion, maxVersion, preferredVersion, regularSuffix, devSuffix).load(); @@ -115,7 +130,7 @@ public class DependencyLoaderImpl { } } - private static void addToClasspath(File file) { + private static synchronized void addToClasspath(File file) { try { val cl = (LaunchClassLoader) DependencyLoader.class.getClassLoader(); cl.addURL(file.toURI().toURL()); @@ -133,6 +148,21 @@ public class DependencyLoaderImpl { Internet.transferAndClose(is, new BufferedOutputStream(Files.newOutputStream(target.toPath()))); } + public static void loadLibraries(Library... libraries) { + loadLibrariesAsync(libraries).join(); + } + + public static CompletableFuture loadLibrariesAsync(Library... libraries) { + val futures = new ArrayList>(); + for (val library: libraries) { + val task = new DependencyLoadTask(library.loadingModId, library.groupId, library.artifactId, + library.minVersion, library.maxVersion, library.preferredVersion, + library.regularSuffix, library.devSuffix); + futures.add(CompletableFuture.runAsync(task::load, executor)); + } + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + @RequiredArgsConstructor private static class DependencyLoadTask { @NonNull diff --git a/src/main/java/com/falsepattern/lib/updates/UpdateChecker.java b/src/main/java/com/falsepattern/lib/updates/UpdateChecker.java index f33f074..079d4a0 100644 --- a/src/main/java/com/falsepattern/lib/updates/UpdateChecker.java +++ b/src/main/java/com/falsepattern/lib/updates/UpdateChecker.java @@ -23,6 +23,7 @@ package com.falsepattern.lib.updates; import com.falsepattern.json.node.JsonNode; import com.falsepattern.lib.StableAPI; import com.falsepattern.lib.dependencies.DependencyLoader; +import com.falsepattern.lib.dependencies.Library; import com.falsepattern.lib.dependencies.SemanticVersion; import com.falsepattern.lib.internal.Internet; import com.falsepattern.lib.internal.Tags; @@ -88,14 +89,14 @@ public class UpdateChecker { if (!jsonLibraryLoaded.get()) { try { DependencyLoader.addMavenRepo("https://maven.falsepattern.com/"); - DependencyLoader.builder() - .loadingModId(Tags.MODID) - .groupId("com.falsepattern") - .artifactId("json") - .minVersion(new SemanticVersion(0, 4, 0)) - .maxVersion(new SemanticVersion(0, Integer.MAX_VALUE, Integer.MAX_VALUE)) - .preferredVersion(new SemanticVersion(0, 4, 1)) - .build(); + DependencyLoader.loadLibraries(Library.builder() + .loadingModId(Tags.MODID) + .groupId("com.falsepattern") + .artifactId("json") + .minVersion(new SemanticVersion(0, 4, 0)) + .maxVersion(new SemanticVersion(0, Integer.MAX_VALUE, Integer.MAX_VALUE)) + .preferredVersion(new SemanticVersion(0, 4, 1)) + .build()); } catch (Exception e) { throw new CompletionException( new UpdateCheckException("Failed to load json library for update checker!", e));