Compare commits

...
12 changed files with 195 additions and 116 deletions

No files matched your search

@@ -27,15 +27,16 @@ import org.jackhuang.hmcl.terracotta.provider.AbstractTerracottaProvider;
import org.jackhuang.hmcl.util.DigestUtils;
import org.jackhuang.hmcl.util.io.ChecksumMismatchException;
import org.jackhuang.hmcl.util.io.FileUtils;
import org.jackhuang.hmcl.util.io.UrlResponseInfo;
import org.jackhuang.hmcl.util.logging.Logger;
import org.jackhuang.hmcl.util.platform.OperatingSystem;
import org.jetbrains.annotations.Nullable;
import org.jackhuang.hmcl.util.tree.TarFileTree;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestInputStream;
@@ -66,7 +67,7 @@ public final class TerracottaBundle {
.thenComposeAsync(Schedulers.javafx(), pkg -> {
FileDownloadTask download = new FileDownloadTask(links, pkg, hash) {
@Override
protected Context getContext(HttpResponse<?> response, boolean checkETag, String bmclapiHash) throws IOException {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException {
FetchTask.Context delegate = super.getContext(response, checkETag, bmclapiHash);
return new Context() {
@Override
@@ -35,7 +35,6 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
@@ -245,7 +244,7 @@ public class Skin {
}
@Override
protected Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) throws IOException {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException {
return new Context() {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -266,7 +265,7 @@ public class Skin {
setResult(new ByteArrayInputStream(baos.toByteArray()));
if (checkETag) {
repository.cacheBytes(UrlResponseInfo.of(response), baos.toByteArray());
repository.cacheBytes(response, baos.toByteArray());
}
}
};
@@ -47,20 +47,21 @@ import static org.jackhuang.hmcl.util.logging.Logger.LOG;
public final class CurseForgeRemoteModRepository implements RemoteModRepository {
private static final String PREFIX = "https://api.curseforge.com";
private static final String apiKey = System.getProperty("hmcl.curseforge.apikey", JarUtils.getAttribute("hmcl.curseforge.apikey", ""));
private static final Semaphore SEMAPHORE = new Semaphore(16);
public static final String API_KEY = System.getProperty("hmcl.curseforge.apikey", JarUtils.getAttribute("hmcl.curseforge.apikey", ""));
private static final int WORD_PERFECT_MATCH_WEIGHT = 5;
private static <R extends HttpRequest> R withApiKey(R request) {
if (request.getUrl().startsWith(PREFIX) && !apiKey.isEmpty()) {
request.header("X-API-KEY", apiKey);
if (request.getUrl().startsWith(PREFIX) && !API_KEY.isEmpty()) {
request.header("X-API-KEY", API_KEY);
}
return request;
}
public static boolean isAvailable() {
return !apiKey.isEmpty();
return !API_KEY.isEmpty();
}
private final Type type;
@@ -25,7 +25,6 @@ import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
@@ -84,7 +83,7 @@ public final class CacheFileTask extends FetchTask<Path> {
}
@Override
protected Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) throws IOException {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException {
assert checkETag;
assert response != null;
@@ -125,7 +124,7 @@ public final class CacheFileTask extends FetchTask<Path> {
}
try {
setResult(repository.cacheRemoteFile(UrlResponseInfo.of(response), temp));
setResult(repository.cacheRemoteFile(response, temp));
} finally {
deleteTempFile();
}
@@ -29,9 +29,6 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URLConnection;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ExecutorService;
@@ -47,7 +44,7 @@ import static org.jackhuang.hmcl.util.logging.Logger.LOG;
public abstract class FetchTask<T> extends Task<T> {
protected static final int DEFAULT_RETRY = 3;
protected static final int DEFAULT_RETRY = 5;
protected final List<URI> uris;
protected int retry = DEFAULT_RETRY;
@@ -86,7 +83,7 @@ public abstract class FetchTask<T> extends Task<T> {
return getContext(null, false, null);
}
protected abstract Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) throws IOException;
protected abstract Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException;
@Override
public void execute() throws Exception {
@@ -136,8 +133,8 @@ public abstract class FetchTask<T> extends Task<T> {
private static final class HttpResumeContext {
private static final Pattern CONTENT_RANGE_PATTERN = Pattern.compile("bytes ([0-9]+)-([0-9]+)/([0-9]+)");
static @Nullable FetchTask.HttpResumeContext of(HttpResponse<?> response) throws IOException {
if (response.statusCode() != HttpURLConnection.HTTP_OK)
static @Nullable FetchTask.HttpResumeContext of(int statusCode, UrlResponseInfo response) throws IOException {
if (statusCode != HttpURLConnection.HTTP_OK)
return null;
boolean acceptRanges = response.headers().firstValue("accept-ranges").orElse("").equalsIgnoreCase("bytes");
@@ -183,8 +180,8 @@ public abstract class FetchTask<T> extends Task<T> {
return strongETag != null ? strongETag : Objects.requireNonNull(lastModified);
}
boolean canResume(HttpResponse<?> response) throws IOException {
if (response.statusCode() != HttpURLConnection.HTTP_PARTIAL)
boolean canResume(int statusCode, UrlResponseInfo response) throws IOException {
if (statusCode != HttpURLConnection.HTTP_PARTIAL)
return false;
var contentEncoding = ContentEncoding.fromHeaders(response.headers());
@@ -312,8 +309,10 @@ public abstract class FetchTask<T> extends Task<T> {
beforeDownload(uri);
updateProgress(0);
HttpResponse<InputStream> response;
HttpURLConnection connection = null;
UrlResponseInfo responseInfo;
String bmclapiHash;
int responseCode;
URI currentURI = uri;
@@ -329,51 +328,56 @@ public abstract class FetchTask<T> extends Task<T> {
}
do {
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(currentURI)
.timeout(NetworkUtils.TIMEOUT)
.header("User-Agent", NetworkUtils.USER_AGENT);
headers.forEach(requestBuilder::header);
response = Holder.HTTP_CLIENT.send(requestBuilder.build(), BODY_HANDLER);
connection = NetworkUtils.createHttpConnection(currentURI);
boolean keepConnection = false;
try {
headers.forEach(connection::setRequestProperty);
responseCode = connection.getResponseCode();
responseInfo = UrlResponseInfo.of(connection);
bmclapiHash = response.headers().firstValue("x-bmclapi-hash").orElse(null);
if (DigestUtils.isSha1Digest(bmclapiHash)) {
Optional<Path> cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash);
if (cache.isPresent()) {
closeResponseBody(response);
useCachedResult(cache.get());
LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri));
return;
}
}
int code = response.statusCode();
if (code >= 300 && code <= 308 && code != 306 && code != 304) {
if (redirects == null) {
redirects = new ArrayList<>();
} else if (redirects.size() >= 20) {
throw new IOException("Too much redirects");
bmclapiHash = responseInfo.headers().firstValue("x-bmclapi-hash").orElse(null);
if (DigestUtils.isSha1Digest(bmclapiHash)) {
Optional<Path> cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash);
if (cache.isPresent()) {
useCachedResult(cache.get());
LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri));
return;
}
}
String location = response.headers().firstValue("Location").orElse(null);
if (StringUtils.isBlank(location))
throw new IOException("Redirected to an empty location");
if (responseCode >= 300 && responseCode <= 308 && responseCode != 306 && responseCode != 304) {
if (redirects == null) {
redirects = new ArrayList<>();
} else if (redirects.size() >= 20) {
throw new IOException("Too much redirects");
}
URI target = currentURI.resolve(NetworkUtils.encodeLocation(location));
redirects.add(target);
String location = connection.getHeaderField("Location");
if (StringUtils.isBlank(location))
throw new IOException("Redirected to an empty location");
if (!NetworkUtils.isHttpUri(target))
throw new IOException("Redirected to not http URI: " + target);
URI target = currentURI.resolve(NetworkUtils.encodeLocation(location));
redirects.add(target);
closeResponseBody(response);
currentURI = target;
} else {
break;
if (!NetworkUtils.isHttpUri(target))
throw new IOException("Redirected to not http URI: " + target);
currentURI = target;
} else {
keepConnection = true;
break;
}
} finally {
if (!keepConnection && connection != null) {
closeHttpConnection(connection);
connection = null;
}
}
} while (true);
boolean closeResponseBody = true;
InputStream inputStream = null;
boolean responseBodyConsumed = false;
try {
int responseCode = response.statusCode();
if (resumeRequested && responseCode == 416) {
resumeContext = null;
discardContext(context);
@@ -385,7 +389,7 @@ public abstract class FetchTask<T> extends Task<T> {
if (useCachedResult && responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) {
// Handle cache
try {
Path cache = repository.getCachedRemoteFile(currentURI, false);
Path cache = repository.getCachedRemoteFile(responseInfo.uri(), false);
useCachedResult(cache);
LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri));
return;
@@ -406,14 +410,14 @@ public abstract class FetchTask<T> extends Task<T> {
throw new ResponseCodeException(uri, responseCode);
}
long contentLength = response.headers().firstValueAsLong("content-length").orElse(-1L);
var contentEncoding = ContentEncoding.fromHeaders(response.headers());
long contentLength = responseInfo.headers().firstValueAsLong("content-length").orElse(-1L);
var contentEncoding = ContentEncoding.fromHeaders(responseInfo.headers());
if (context == null) {
context = getContext(response, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(response);
context = getContext(responseInfo, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(responseCode, responseInfo);
} else if (resumeRequested) {
if (resumeContext.canResume(response)) {
if (resumeContext.canResume(responseCode, responseInfo)) {
// Resume download
LOG.info("Resuming " + resumeContext.uri + " from " + resumeContext.countUncompressed);
} else {
@@ -426,16 +430,18 @@ public abstract class FetchTask<T> extends Task<T> {
}
} else {
discardContext(context);
context = getContext(response, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(response);
context = getContext(responseInfo, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(responseCode, responseInfo);
}
try {
closeResponseBody = false;
inputStream = connection.getInputStream();
download(context,
resumeContext, response.body(),
resumeContext, inputStream,
contentLength,
contentEncoding);
inputStream = null;
responseBodyConsumed = true;
} catch (IOException | InterruptedException | RuntimeException | Error e) {
if (context.broken) {
IOUtils.closeQuietly(context, e);
@@ -456,9 +462,9 @@ public abstract class FetchTask<T> extends Task<T> {
context = null;
return;
} finally {
if (closeResponseBody) {
closeResponseBody(response);
}
IOUtils.closeQuietly(inputStream);
if (connection != null && !responseBodyConsumed)
closeHttpConnection(connection);
}
} catch (InterruptedException e) {
throw e;
@@ -492,10 +498,10 @@ public abstract class FetchTask<T> extends Task<T> {
throw toDownloadException(uri, null, exceptions);
}
private static void closeResponseBody(HttpResponse<InputStream> response) {
InputStream body = response.body();
if (body != null)
IOUtils.closeQuietly(body);
/// Disconnects an HTTP URL connection whose response body will not be consumed.
private static void closeHttpConnection(HttpURLConnection connection) {
IOUtils.closeQuietly(connection.getErrorStream());
connection.disconnect();
}
private static void discardContext(@Nullable Context context) {
@@ -649,12 +655,6 @@ public abstract class FetchTask<T> extends Task<T> {
CACHED
}
private static final HttpResponse.BodyHandler<InputStream> BODY_HANDLER = responseInfo -> {
if (responseInfo.statusCode() / 100 == 2)
return HttpResponse.BodySubscribers.ofInputStream();
else
return HttpResponse.BodySubscribers.replacing(null);
};
public static int DEFAULT_CONCURRENCY = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64);
private static int downloadExecutorConcurrency = DEFAULT_CONCURRENCY;
@@ -726,24 +726,4 @@ public abstract class FetchTask<T> extends Task<T> {
initialized = true;
}
/// Ensure that [#HTTP_CLIENT] is initialized after ProxyManager has been initialized.
private static final class Holder {
private static final HttpClient HTTP_CLIENT;
static {
if (!initialized) {
throw new AssertionError("FetchTask.Holder accessed before ProxyManager initialization.");
}
boolean useHttp2 = !"false".equalsIgnoreCase(System.getProperty("hmcl.http2"));
HTTP_CLIENT = HttpClient.newBuilder()
.connectTimeout(NetworkUtils.TIMEOUT)
.version(useHttp2 ? HttpClient.Version.HTTP_2 : HttpClient.Version.HTTP_1_1)
.build();
}
private Holder() {
}
}
}
@@ -28,7 +28,6 @@ import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.FileSystem;
@@ -179,7 +178,7 @@ public class FileDownloadTask extends FetchTask<Void> {
}
@Override
protected Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) throws IOException {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException {
Path temp = Files.createTempFile(null, null);
String algorithm;
@@ -271,7 +270,7 @@ public class FileDownloadTask extends FetchTask<Void> {
}
if (checkETag) {
repository.cacheRemoteFile(UrlResponseInfo.of(response), file);
repository.cacheRemoteFile(response, file);
}
} finally {
if (!moved) {
@@ -26,7 +26,6 @@ import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpResponse;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -63,7 +62,7 @@ public final class GetTask extends FetchTask<String> {
}
@Override
protected Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) {
long length = -1;
if (response != null)
length = response.headers().firstValueAsLong("content-length").orElse(-1L);
@@ -92,7 +91,7 @@ public final class GetTask extends FetchTask<String> {
setResult(result);
if (checkETag) {
repository.cacheText(UrlResponseInfo.of(response), result);
repository.cacheText(response, result);
}
}
};
@@ -41,7 +41,7 @@ public final class IOUtils {
private IOUtils() {
}
public static final int DEFAULT_BUFFER_SIZE = 8 * 1024;
public static final int DEFAULT_BUFFER_SIZE = 32 * 1024;
public static BufferedReader newBufferedReaderMaybeNativeEncoding(Path file) throws IOException {
if (NATIVE_CHARSET == UTF_8)
@@ -18,6 +18,7 @@
package org.jackhuang.hmcl.util.io;
import org.glavo.url.WebURL;
import org.jackhuang.hmcl.mod.curse.CurseForgeRemoteModRepository;
import org.jackhuang.hmcl.util.Pair;
import org.jackhuang.hmcl.util.StringUtils;
import org.jetbrains.annotations.NotNull;
@@ -47,7 +48,7 @@ public final class NetworkUtils {
public static final String PARAMETER_SEPARATOR = "&";
public static final String NAME_VALUE_SEPARATOR = "=";
public static final Duration TIMEOUT = Duration.ofSeconds(30);
public static final Duration TIMEOUT = Duration.ofSeconds(10);
public static final int TIMEOUT_MILLIS = (int) TIMEOUT.toMillis();
private NetworkUtils() {
@@ -163,6 +164,45 @@ public final class NetworkUtils {
}
}
private static final List<Pair<String, String>> API_KEYS;
static {
if (CurseForgeRemoteModRepository.API_KEY.isEmpty()) {
API_KEYS = List.of();
} else {
API_KEYS = List.of(
pair("api.curseforge.com", CurseForgeRemoteModRepository.API_KEY),
pair("forgecdn.net", CurseForgeRemoteModRepository.API_KEY)
);
}
}
private static boolean matchDomainSuffix(String domain, String suffix) {
return domain.endsWith(suffix)
&& (domain.length() == suffix.length() || domain.charAt(domain.length() - suffix.length() - 1) == '.');
}
public static void injectApiKey(WebURL url, URLConnection connection) {
if (!(connection instanceof HttpURLConnection))
return;
if (connection.getRequestProperty("x-api-key") != null) {
return;
}
String host = url.getHost();
if (host == null || host.isEmpty())
return;
for (Pair<String, String> pair : API_KEYS) {
String hostSuffix = pair.getKey();
if (matchDomainSuffix(host, hostSuffix)) {
connection.addRequestProperty("x-api-key", pair.getValue());
return;
}
}
}
public static URLConnection createConnection(WebURL url) throws IOException {
URLConnection connection;
try {
@@ -176,6 +216,7 @@ public final class NetworkUtils {
httpConnection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag());
httpConnection.setRequestProperty("User-Agent", USER_AGENT);
httpConnection.setInstanceFollowRedirects(false);
injectApiKey(url, connection);
}
return connection;
}
@@ -293,10 +334,10 @@ public final class NetworkUtils {
throw new IOException("Too much redirects");
}
HttpURLConnection redirected = (HttpURLConnection) new URL(conn.getURL(), encodeLocation(newURL))
.openConnection();
properties
.forEach((key, value) -> value.forEach(element -> redirected.addRequestProperty(key, element)));
WebURL redirectedUrl = WebURL.of(conn.getURL()).resolve(newURL);
HttpURLConnection redirected = (HttpURLConnection) redirectedUrl.toURL().openConnection();
properties.forEach((key, value) -> value.forEach(element -> redirected.addRequestProperty(key, element)));
injectApiKey(redirectedUrl, redirected);
redirected.setRequestMethod(method);
conn = redirected;
++redirect;
@@ -17,17 +17,60 @@
*/
package org.jackhuang.hmcl.util.io;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.net.http.HttpHeaders;
import java.net.http.HttpResponse;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/// Response URI and headers captured from a URL response.
///
/// @author Glavo
public record UrlResponseInfo(URI uri, HttpHeaders headers) {
/// Creates response metadata from a URL connection.
public static UrlResponseInfo of(URLConnection connection) throws IOException {
return new UrlResponseInfo(toURI(connection.getURL()), HttpHeaders.of(headers(connection), (name, value) -> true));
}
/// Creates response metadata from an HTTP client response.
public static UrlResponseInfo of(HttpResponse<?> httpResponse) {
return new UrlResponseInfo(httpResponse.uri(), httpResponse.headers());
}
/// Creates response metadata from an HTTP client response-info object.
public static UrlResponseInfo of(URI uri, HttpResponse.ResponseInfo info) {
return new UrlResponseInfo(uri, info.headers());
}
/// Converts a response URL into a URI.
private static URI toURI(URL url) throws IOException {
try {
return url.toURI();
} catch (URISyntaxException e) {
throw new IOException("Invalid response URL: " + url, e);
}
}
/// Copies named header fields from a URL connection.
private static Map<String, List<String>> headers(URLConnection connection) {
Map<String, List<String>> headerFields = connection.getHeaderFields();
if (headerFields == null || headerFields.isEmpty()) {
return Map.of();
}
LinkedHashMap<String, List<String>> headers = new LinkedHashMap<>();
for (Map.Entry<String, List<String>> entry : headerFields.entrySet()) {
String name = entry.getKey();
List<String> values = entry.getValue();
if (name != null && values != null) {
headers.put(name, List.copyOf(values));
}
}
return headers;
}
}
@@ -914,6 +914,23 @@
26.1
26.1.1-rc-1
26.1.1
26w14a
26.1.2-rc-1
26.1.2
26w14a
26.2-snapshot-1
26.2-snapshot-2
26.2-snapshot-3
26.2-snapshot-4
26.2-snapshot-5
26.2-snapshot-6
26.2-snapshot-7
26.2-snapshot-8
26.2-pre-1
26.2-pre-2
26.2-pre-3
26.2-pre-4
26.2-pre-5
26.2-pre-6
26.2-rc-1
26.2-rc-2
26.2
@@ -21,6 +21,7 @@ import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import org.jackhuang.hmcl.util.CacheRepository;
import org.jackhuang.hmcl.util.io.NetworkUtils;
import org.jackhuang.hmcl.util.io.UrlResponseInfo;
import org.jetbrains.annotations.NotNullByDefault;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.Test;
@@ -30,7 +31,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpResponse;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -45,7 +45,7 @@ import static org.junit.jupiter.api.Assertions.*;
/// Tests HTTP download retry and resume behavior.
@NotNullByDefault
public final class FetchTaskTest {
/// Marks HTTP client dependencies as initialized for isolated unit tests.
/// Marks HTTP connection dependencies as initialized for isolated unit tests.
@org.junit.jupiter.api.BeforeAll
public static void notifyFetchTaskInitialized() {
FetchTask.notifyInitialized();
@@ -257,7 +257,7 @@ public final class FetchTaskTest {
}
@Override
protected Context getContext(@Nullable HttpResponse<?> response, boolean checkETag, String bmclapiHash) {
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) {
Charset charset = NetworkUtils.getCharsetFromContentType(response == null
? null
: response.headers().firstValue("content-type").orElse(null));