feat: use libpsl (#2575)

Use libpsl to calculate public and private parts of URL hosts.
This commit is contained in:
Charles Kerr 2022-02-12 11:30:27 -06:00 committed by GitHub
parent 708fc1531e
commit e14c7f38e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 538 additions and 406 deletions

4
.gitmodules vendored
View File

@ -35,3 +35,7 @@
[submodule "third-party/libdeflate"]
path = third-party/libdeflate
url = https://github.com/transmission/libdeflate
[submodule "third-party/libpsl"]
path = third-party/libpsl
url = https://github.com/transmission/libpsl.git
branch = post-3.0.0-transmission

View File

@ -41,6 +41,7 @@ tr_auto_option(USE_SYSTEM_MINIUPNPC "Use system miniupnpc library" AUTO)
tr_auto_option(USE_SYSTEM_NATPMP "Use system natpmp library" AUTO)
tr_auto_option(USE_SYSTEM_UTP "Use system utp library" AUTO)
tr_auto_option(USE_SYSTEM_B64 "Use system b64 library" AUTO)
tr_auto_option(USE_SYSTEM_PSL "Use system psl library" AUTO)
tr_auto_option(USE_QT_VERSION "Use specific Qt version" AUTO 5 6)
tr_list_option(WITH_CRYPTO "Use specified crypto library" AUTO openssl cyassl polarssl ccrypto)
tr_auto_option(WITH_INOTIFY "Enable inotify support (on systems that support it)" AUTO)
@ -109,6 +110,7 @@ string(SUBSTRING "${TR_VCS_REVISION}" 0 10 TR_VCS_REVISION)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
set(PSL_MINIMUM 0.21.1)
set(CURL_MINIMUM 7.28.0)
set(CYASSL_MINIMUM 3.0)
set(DEFLATE_MINIMUM 1.9)
@ -354,6 +356,8 @@ endif()
tr_add_external_auto_library(DHT dht dht)
tr_add_external_auto_library(PSL libpsl psl)
if(ENABLE_UTP)
tr_add_external_auto_library(UTP libutp utp)

View File

@ -45,7 +45,7 @@ If you're new to building programs from source code, this is typically easier th
$ git clone https://github.com/transmission/transmission Transmission
$ cd Transmission
$ git submodule update --init
$ git submodule update --init --recursive
$ mkdir build
$ cd build
$ # Use -DCMAKE_BUILD_TYPE=RelWithDebInfo to build optimized binary.
@ -58,7 +58,7 @@ If you're new to building programs from source code, this is typically easier th
$ cd Transmission/build
$ make clean
$ git pull --rebase --prune
$ git submodule update
$ git submodule update --recursive
$ # Use -DCMAKE_BUILD_TYPE=RelWithDebInfo to build optimized binary.
$ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
$ make

View File

@ -13,6 +13,7 @@
0A89346B736DBCF81F3A4852 /* torrent-metainfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 0A89346B736DBCF81F3A4853 /* torrent-metainfo.h */; };
1BB44E07B1B52E28291B4E32 /* file-piece-map.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1BB44E07B1B52E28291B4E30 /* file-piece-map.cc */; };
1BB44E07B1B52E28291B4E33 /* file-piece-map.h in Headers */ = {isa = PBXBuildFile; fileRef = 1BB44E07B1B52E28291B4E31 /* file-piece-map.h */; };
2856E0656A49F2665D69E760 /* benc.h in Headers */ = {isa = PBXBuildFile; fileRef = 2856E0656A49F2665D69E761 /* benc.h */; };
35F373030C2DA89000DAA8F2 /* FilePriorityCell.mm in Sources */ = {isa = PBXBuildFile; fileRef = 35F373010C2DA88F00DAA8F2 /* FilePriorityCell.mm */; };
3C7A11970D0B2EE300B5701F /* getgateway.c in Sources */ = {isa = PBXBuildFile; fileRef = 3C7A11910D0B2EE300B5701F /* getgateway.c */; };
3C7A11980D0B2EE300B5701F /* getgateway.h in Headers */ = {isa = PBXBuildFile; fileRef = 3C7A11920D0B2EE300B5701F /* getgateway.h */; };
@ -399,8 +400,11 @@
C3CEBBFA2794A0D200683BE0 /* compiler_gcc.h in Headers */ = {isa = PBXBuildFile; fileRef = C3CEBBF72794A0D200683BE0 /* compiler_gcc.h */; };
C3CEBBFB2794A0D200683BE0 /* compiler_msc.h in Headers */ = {isa = PBXBuildFile; fileRef = C3CEBBF82794A0D200683BE0 /* compiler_msc.h */; };
C3CEBBFC2794A12200683BE0 /* libdeflate.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C3CEBBA927949CA000683BE0 /* libdeflate.a */; };
C3D9062627B7E3E200EF2386 /* lookup_string_in_fixed_set.c in Sources */ = {isa = PBXBuildFile; fileRef = C3D9061727B7E1DE00EF2386 /* lookup_string_in_fixed_set.c */; };
C3D9062727B7E3E800EF2386 /* psl.c in Sources */ = {isa = PBXBuildFile; fileRef = C3D9061827B7E1DE00EF2386 /* psl.c */; };
C3D9062A27B7EAC600EF2386 /* libpsl.h in Headers */ = {isa = PBXBuildFile; fileRef = C3D9061B27B7E31100EF2386 /* libpsl.h */; };
C3D9062F27B7F7E200EF2386 /* libpsl.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C3D9062127B7E3C900EF2386 /* libpsl.a */; };
CAB35C64252F6F5E00552A55 /* mime-types.h in Headers */ = {isa = PBXBuildFile; fileRef = CAB35C62252F6F5E00552A55 /* mime-types.h */; };
2856E0656A49F2665D69E760 /* benc.h in Headers */ = {isa = PBXBuildFile; fileRef = 2856E0656A49F2665D69E761 /* benc.h */; };
E138A9780C04D88F00C5426C /* ProgressGradients.mm in Sources */ = {isa = PBXBuildFile; fileRef = E138A9760C04D88F00C5426C /* ProgressGradients.mm */; };
E23B55A5FC3B557F7746D510 /* interned-string.h in Headers */ = {isa = PBXBuildFile; fileRef = E23B55A5FC3B557F7746D511 /* interned-string.h */; settings = {ATTRIBUTES = (Project, ); }; };
E71A5565279C2DD600EBFA1E /* tr-assert.mm in Sources */ = {isa = PBXBuildFile; fileRef = E71A5564279C2DD600EBFA1E /* tr-assert.mm */; };
@ -512,6 +516,13 @@
remoteGlobalIDString = C3CEBB9F27949CA000683BE0;
remoteInfo = deflate;
};
C3D9062D27B7F7CE00EF2386 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 29B97313FDCFA39411CA2CEA /* Project object */;
proxyType = 1;
remoteGlobalIDString = C3D9062027B7E3C900EF2386;
remoteInfo = psl;
};
/* End PBXContainerItemProxy section */
/* Begin PBXCopyFilesBuildPhase section */
@ -547,6 +558,7 @@
13E42FB307B3F0F600E4EEF1 /* CoreData.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreData.framework; path = System/Library/Frameworks/CoreData.framework; sourceTree = SDKROOT; };
1BB44E07B1B52E28291B4E30 /* file-piece-map.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = "file-piece-map.cc"; sourceTree = "<group>"; };
1BB44E07B1B52E28291B4E31 /* file-piece-map.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "file-piece-map.h"; sourceTree = "<group>"; };
2856E0656A49F2665D69E761 /* benc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = benc.h; sourceTree = "<group>"; };
29B97316FDCFA39411CA2CEA /* main.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = main.mm; sourceTree = "<group>"; };
29B97324FDCFA39411CA2CEA /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = System/Library/Frameworks/AppKit.framework; sourceTree = SDKROOT; };
29B97325FDCFA39411CA2CEA /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
@ -1111,8 +1123,11 @@
C3CEBBF62794A0D200683BE0 /* common_defs.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = common_defs.h; path = common/common_defs.h; sourceTree = "<group>"; };
C3CEBBF72794A0D200683BE0 /* compiler_gcc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = compiler_gcc.h; path = common/compiler_gcc.h; sourceTree = "<group>"; };
C3CEBBF82794A0D200683BE0 /* compiler_msc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = compiler_msc.h; path = common/compiler_msc.h; sourceTree = "<group>"; };
C3D9061727B7E1DE00EF2386 /* lookup_string_in_fixed_set.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lookup_string_in_fixed_set.c; path = src/lookup_string_in_fixed_set.c; sourceTree = "<group>"; };
C3D9061827B7E1DE00EF2386 /* psl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = psl.c; path = src/psl.c; sourceTree = "<group>"; };
C3D9061B27B7E31100EF2386 /* libpsl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = libpsl.h; path = include/libpsl.h; sourceTree = "<group>"; };
C3D9062127B7E3C900EF2386 /* libpsl.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libpsl.a; sourceTree = BUILT_PRODUCTS_DIR; };
CAB35C62252F6F5E00552A55 /* mime-types.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mime-types.h"; sourceTree = "<group>"; };
2856E0656A49F2665D69E761 /* benc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "benc.h"; sourceTree = "<group>"; };
E138A9750C04D88F00C5426C /* ProgressGradients.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ProgressGradients.h; sourceTree = "<group>"; };
E138A9760C04D88F00C5426C /* ProgressGradients.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ProgressGradients.mm; sourceTree = "<group>"; };
E23B55A5FC3B557F7746D511 /* interned-string.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "interned-string.h"; sourceTree = SOURCE_ROOT; };
@ -1209,6 +1224,7 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
C3D9062F27B7F7E200EF2386 /* libpsl.a in Frameworks */,
C3CEBBFC2794A12200683BE0 /* libdeflate.a in Frameworks */,
C1A7517526ED048C0038B90A /* libarc4.a in Frameworks */,
C1639A741A55F4E000E42033 /* libb64.a in Frameworks */,
@ -1265,6 +1281,13 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
C3D9061F27B7E3C900EF2386 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
@ -1376,6 +1399,7 @@
C1639A6F1A55F4D600E42033 /* libb64.a */,
C1A7516426ED03140038B90A /* libarc4.a */,
C3CEBBA927949CA000683BE0 /* libdeflate.a */,
C3D9062127B7E3C900EF2386 /* libpsl.a */,
);
name = Products;
sourceTree = "<group>";
@ -1397,6 +1421,7 @@
BE75C3570C72A0D600DBEFE0 /* libevent */,
BE1183410CE15DF00002D0F3 /* libminiupnp */,
3C7A11880D0B2E6700B5701F /* libnatpmp */,
C3D9061627B7E12F00EF2386 /* libpsl */,
C1639A751A55F52800E42033 /* b64 */,
4DDBB71509E16B3F00284745 /* Libraries */,
A2F35BBA15C5A0A100EBF632 /* Frameworks */,
@ -1931,6 +1956,17 @@
name = common;
sourceTree = "<group>";
};
C3D9061627B7E12F00EF2386 /* libpsl */ = {
isa = PBXGroup;
children = (
C3D9061B27B7E31100EF2386 /* libpsl.h */,
C3D9061727B7E1DE00EF2386 /* lookup_string_in_fixed_set.c */,
C3D9061827B7E1DE00EF2386 /* psl.c */,
);
name = libpsl;
path = "third-party/libpsl";
sourceTree = "<group>";
};
E1B6FBF80C0D719B0015FE4D /* Info Window */ = {
isa = PBXGroup;
children = (
@ -2186,6 +2222,14 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
C3D9061D27B7E3C900EF2386 /* Headers */ = {
isa = PBXHeadersBuildPhase;
buildActionMask = 2147483647;
files = (
C3D9062A27B7EAC600EF2386 /* libpsl.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXHeadersBuildPhase section */
/* Begin PBXNativeTarget section */
@ -2226,6 +2270,7 @@
A22CFCC70FC24F990009BD3E /* PBXTargetDependency */,
A2E384E4130DFB51001F501B /* PBXTargetDependency */,
C165AB8D1A55FAA900D37711 /* PBXTargetDependency */,
C3D9062E27B7F7CE00EF2386 /* PBXTargetDependency */,
);
name = libtransmission;
productName = transmission;
@ -2447,6 +2492,24 @@
productReference = C3CEBBA927949CA000683BE0 /* libdeflate.a */;
productType = "com.apple.product-type.library.static";
};
C3D9062027B7E3C900EF2386 /* psl */ = {
isa = PBXNativeTarget;
buildConfigurationList = C3D9062227B7E3C900EF2386 /* Build configuration list for PBXNativeTarget "psl" */;
buildPhases = (
C3D9062C27B7F4FE00EF2386 /* ShellScript */,
C3D9061D27B7E3C900EF2386 /* Headers */,
C3D9061E27B7E3C900EF2386 /* Sources */,
C3D9061F27B7E3C900EF2386 /* Frameworks */,
);
buildRules = (
);
dependencies = (
);
name = psl;
productName = libpsl;
productReference = C3D9062127B7E3C900EF2386 /* libpsl.a */;
productType = "com.apple.product-type.library.static";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
@ -2466,6 +2529,9 @@
C1639A6E1A55F4D600E42033 = {
CreatedOnToolsVersion = 6.1.1;
};
C3D9062027B7E3C900EF2386 = {
CreatedOnToolsVersion = 13.0;
};
};
};
buildConfigurationList = 4DF0C59A089918A300DD8943 /* Build configuration list for PBXProject "Transmission" */;
@ -2503,6 +2569,7 @@
C1639A6E1A55F4D600E42033 /* b64 */,
C1A7516326ED03140038B90A /* arc4 */,
C3CEBB9F27949CA000683BE0 /* deflate */,
C3D9062027B7E3C900EF2386 /* psl */,
);
};
/* End PBXProject section */
@ -2571,7 +2638,7 @@
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "sh update-version-h.sh";
shellScript = "sh update-version-h.sh\n";
};
A2305097100C0293003FDB0C /* ShellScript */ = {
isa = PBXShellScriptBuildPhase;
@ -2644,6 +2711,25 @@
shellPath = /bin/sh;
shellScript = "cd third-party/dht && rm -f dht && ln -s . dht\n";
};
C3D9062C27B7F4FE00EF2386 /* ShellScript */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
files = (
);
inputFileListPaths = (
);
inputPaths = (
"third-party/libpsl/include/libpsl.h.in",
);
outputFileListPaths = (
);
outputPaths = (
"third-party/libpsl/include/libpsl.h",
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "sed 's|@LIBPSL_[A-Z_]*@|0|' < third-party/libpsl/include/libpsl.h.in > third-party/libpsl/include/libpsl.h\n";
};
/* End PBXShellScriptBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
@ -2944,6 +3030,15 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
C3D9061E27B7E3C900EF2386 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
C3D9062627B7E3E200EF2386 /* lookup_string_in_fixed_set.c in Sources */,
C3D9062727B7E3E800EF2386 /* psl.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXTargetDependency section */
@ -3017,6 +3112,11 @@
target = C3CEBB9F27949CA000683BE0 /* deflate */;
targetProxy = C33E46A12794B3CC0090F2AA /* PBXContainerItemProxy */;
};
C3D9062E27B7F7CE00EF2386 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = C3D9062027B7E3C900EF2386 /* psl */;
targetProxy = C3D9062D27B7F7CE00EF2386 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin PBXVariantGroup section */
@ -3277,6 +3377,7 @@
"third-party/libutp",
"third-party/utfcpp/source",
"third-party/libdeflate",
"third-party/libpsl/include",
);
OTHER_CFLAGS = (
"$(inherited)",
@ -3469,6 +3570,7 @@
"third-party/libutp",
"third-party/utfcpp/source",
"third-party/libdeflate",
"third-party/libpsl/include",
);
OTHER_CFLAGS = (
"$(inherited)",
@ -3718,6 +3820,7 @@
"third-party/libutp",
"third-party/utfcpp/source",
"third-party/libdeflate",
"third-party/libpsl/include",
);
OTHER_CFLAGS = (
"$(inherited)",
@ -4012,6 +4115,33 @@
};
name = Release;
};
C3D9062327B7E3C900EF2386 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_PREPROCESSOR_DEFINITIONS_NOT_USED_IN_PRECOMPS = "PACKAGE_VERSION=\"\\\"0\\\"\"";
HEADER_SEARCH_PATHS = "third-party/libpsl/include";
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Debug;
};
C3D9062427B7E3C900EF2386 /* Release - Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_PREPROCESSOR_DEFINITIONS_NOT_USED_IN_PRECOMPS = "PACKAGE_VERSION=\"\\\"0\\\"\"";
HEADER_SEARCH_PATHS = "third-party/libpsl/include";
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = "Release - Debug";
};
C3D9062527B7E3C900EF2386 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_PREPROCESSOR_DEFINITIONS_NOT_USED_IN_PRECOMPS = "PACKAGE_VERSION=\"\\\"0\\\"\"";
HEADER_SEARCH_PATHS = "third-party/libpsl/include";
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
@ -4165,6 +4295,16 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Debug;
};
C3D9062227B7E3C900EF2386 /* Build configuration list for PBXNativeTarget "psl" */ = {
isa = XCConfigurationList;
buildConfigurations = (
C3D9062327B7E3C900EF2386 /* Debug */,
C3D9062427B7E3C900EF2386 /* Release - Debug */,
C3D9062527B7E3C900EF2386 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Debug;
};
/* End XCConfigurationList section */
};
rootObject = 29B97313FDCFA39411CA2CEA /* Project object */;

View File

@ -38,6 +38,7 @@ for:
git submodule update --init --recursive
choco install python3 --pre
choco install ActivePerl
choco install nasm
choco install jom

34
cmake/FindPSL.cmake Normal file
View File

@ -0,0 +1,34 @@
if(PSL_PREFER_STATIC_LIB)
set(PSL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
if(WIN32)
set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib ${CMAKE_FIND_LIBRARY_SUFFIXES})
else()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
endif()
endif()
if(UNIX)
find_package(PkgConfig QUIET)
pkg_check_modules(_PSL QUIET libpsl)
endif()
find_path(PSL_INCLUDE_DIR NAMES libpsl.h HINTS ${_PSL_INCLUDEDIR})
find_library(PSL_LIBRARY NAMES libpsl HINTS ${_PSL_LIBDIR})
set(PSL_INCLUDE_DIRS ${PSL_INCLUDE_DIR})
set(PSL_LIBRARIES ${PSL_LIBRARY})
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(PSL
REQUIRED_VARS
PSL_LIBRARY
PSL_INCLUDE_DIR
)
mark_as_advanced(PSL_INCLUDE_DIR PSL_LIBRARY)
if(PSL_PREFER_STATIC_LIB)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${PSL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
unset(PSL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES)
endif()

View File

@ -352,41 +352,43 @@ The 'source' column here corresponds to the data structure there.
| Key | Value Type | transmission.h source
|:--|:--|:--
| `announce` | string | tr_tracker_info
| `id` | number | tr_tracker_info
| `scrape` | string | tr_tracker_info
| `tier` | number | tr_tracker_info
| `announce` | string | tr_tracker_view
| `id` | number | tr_tracker_view
| `scrape` | string | tr_tracker_view
| `sitename` | string | tr_tracker_view
| `tier` | number | tr_tracker_view
`trackerStats`: array of objects, each containing:
| Key | Value Type | transmission.h source
|:--|:--|:--
| `announce` | string | tr_tracker_stat
| `announceState` | number | tr_tracker_stat
| `downloadCount` | number | tr_tracker_stat
| `hasAnnounced` | boolean | tr_tracker_stat
| `hasScraped` | boolean | tr_tracker_stat
| `host` | string | tr_tracker_stat
| `id` | number | tr_tracker_stat
| `isBackup` | boolean | tr_tracker_stat
| `lastAnnouncePeerCount` | number | tr_tracker_stat
| `lastAnnounceResult` | string | tr_tracker_stat
| `lastAnnounceStartTime` | number | tr_tracker_stat
| `lastAnnounceSucceeded` | boolean | tr_tracker_stat
| `lastAnnounceTime` | number | tr_tracker_stat
| `lastAnnounceTimedOut` | boolean | tr_tracker_stat
| `lastScrapeResult` | string | tr_tracker_stat
| `lastScrapeStartTime` | number | tr_tracker_stat
| `lastScrapeSucceeded` | boolean | tr_tracker_stat
| `lastScrapeTime` | number | tr_tracker_stat
| `lastScrapeTimedOut` | boolean | tr_tracker_stat
| `leecherCount` | number | tr_tracker_stat
| `nextAnnounceTime` | number | tr_tracker_stat
| `nextScrapeTime` | number | tr_tracker_stat
| `scrape` | string | tr_tracker_stat
| `scrapeState` | number | tr_tracker_stat
| `seederCount` | number | tr_tracker_stat
| `tier` | number | tr_tracker_stat
| `announceState` | number | tr_tracker_view
| `announce` | string | tr_tracker_view
| `downloadCount` | number | tr_tracker_view
| `hasAnnounced` | boolean | tr_tracker_view
| `hasScraped` | boolean | tr_tracker_view
| `host` | string | tr_tracker_view
| `id` | number | tr_tracker_view
| `isBackup` | boolean | tr_tracker_view
| `lastAnnouncePeerCount` | number | tr_tracker_view
| `lastAnnounceResult` | string | tr_tracker_view
| `lastAnnounceStartTime` | number | tr_tracker_view
| `lastAnnounceSucceeded` | boolean | tr_tracker_view
| `lastAnnounceTime` | number | tr_tracker_view
| `lastAnnounceTimedOut` | boolean | tr_tracker_view
| `lastScrapeResult` | string | tr_tracker_view
| `lastScrapeStartTime` | number | tr_tracker_view
| `lastScrapeSucceeded` | boolean | tr_tracker_view
| `lastScrapeTime` | number | tr_tracker_view
| `lastScrapeTimedOut` | boolean | tr_tracker_view
| `leecherCount` | number | tr_tracker_view
| `nextAnnounceTime` | number | tr_tracker_view
| `nextScrapeTime` | number | tr_tracker_view
| `scrapeState` | number | tr_tracker_view
| `scrape` | string | tr_tracker_view
| `seederCount` | number | tr_tracker_view
| `sitename` | string | tr_tracker_view
| `tier` | number | tr_tracker_view
`wanted`: An array of `tr_torrentFileCount()` booleans true if the corresponding file is to be downloaded. (Source: `tr_file_view`)
@ -936,6 +938,8 @@ Transmission 4.0.0 (`rpc-version-semver` 5.3.0, `rpc-version`: 17)
| `session-get` | new arg `script-torrent-added-filename`
| `torrent-add` | new arg `labels`
| `torrent-get` | new arg `file-count`
| `torrent-get` | new arg `tracker.sitename`
| `torrent-get` | new arg `trackerStats.sitename`
| `torrent-get` | new arg `primary-mime-type`

View File

@ -11,6 +11,7 @@
#include <libtransmission/transmission.h>
#include <libtransmission/web.h> /* tr_webRun() */
#include <libtransmission/web-utils.h>
#include "FaviconCache.h"
#include "Utils.h" /* gtr_get_host_from_url() */
@ -144,5 +145,6 @@ void gtr_get_favicon_from_url(
Glib::ustring const& url,
std::function<void(Glib::RefPtr<Gdk::Pixbuf> const&)> const& pixbuf_ready_func)
{
gtr_get_favicon(session, gtr_get_host_from_url(url), pixbuf_ready_func);
auto const host = std::string{ tr_urlParse(url.c_str())->host };
gtr_get_favicon(session, host, pixbuf_ready_func);
}

View File

@ -19,7 +19,7 @@
#include "FilterBar.h"
#include "HigWorkarea.h" /* GUI_PAD */
#include "Session.h" /* MC_TORRENT */
#include "Utils.h" /* gtr_get_host_from_url() */
#include "Utils.h"
namespace
{
@ -62,7 +62,7 @@ private:
Glib::RefPtr<Gtk::TreeModelFilter> filter_model_;
int active_activity_type_ = 0;
int active_tracker_type_ = 0;
Glib::ustring active_tracker_host_;
Glib::ustring active_tracker_sitename_;
sigc::connection activity_model_row_changed_tag_;
sigc::connection activity_model_row_inserted_tag_;
@ -96,17 +96,17 @@ class TrackerFilterModelColumns : public Gtk::TreeModelColumnRecord
public:
TrackerFilterModelColumns()
{
add(name);
add(displayname);
add(count);
add(type);
add(host);
add(sitename);
add(pixbuf);
}
Gtk::TreeModelColumn<Glib::ustring> name; /* human-readable name; ie, Legaltorrents */
Gtk::TreeModelColumn<Glib::ustring> displayname; /* human-readable name; ie, Legaltorrents */
Gtk::TreeModelColumn<int> count; /* how many matches there are */
Gtk::TreeModelColumn<int> type;
Gtk::TreeModelColumn<std::string> host; /* pattern-matching text; ie, legaltorrents.com */
Gtk::TreeModelColumn<Glib::ustring> sitename; // pattern-matching text; see tr_parsed_url.sitename
Gtk::TreeModelColumn<Glib::RefPtr<Gdk::Pixbuf>> pixbuf;
};
@ -115,20 +115,7 @@ TrackerFilterModelColumns const tracker_filter_cols;
/* human-readable name; ie, Legaltorrents */
Glib::ustring get_name_from_host(std::string const& host)
{
std::string name;
if (tr_addressIsIP(host.c_str()))
{
name = host;
}
else if (auto const dot = host.rfind('.'); dot != std::string::npos)
{
name = host.substr(0, dot);
}
else
{
name = host;
}
std::string name = host;
if (!name.empty())
{
@ -164,48 +151,56 @@ bool tracker_filter_model_update(Glib::RefPtr<Gtk::TreeStore> const& tracker_mod
{
tracker_model->steal_data(DIRTY_KEY);
struct site_info
{
int count = 0;
std::string host;
std::string sitename;
bool operator<(site_info const& that) const
{
return sitename < that.sitename;
}
};
/* Walk through all the torrents, tallying how many matches there are
* for the various categories. Also make a sorted list of all tracker
* hosts s.t. we can merge it with the existing list */
int num_torrents = 0;
std::vector<std::string const*> hosts;
std::set<std::string> strings;
std::unordered_map<std::string const*, int> hosts_hash;
auto n_torrents = int{ 0 };
auto site_infos = std::unordered_map<std::string /*site*/, site_info>{};
auto* tmodel = static_cast<Gtk::TreeModel*>(tracker_model->get_data(TORRENT_MODEL_KEY));
for (auto const& row : tmodel->children())
{
auto const* tor = static_cast<tr_torrent const*>(row.get_value(torrent_cols.torrent));
std::set<std::string const*> keys;
auto torrent_sites_and_hosts = std::map<std::string, std::string>{};
for (size_t i = 0, n = tr_torrentTrackerCount(tor); i < n; ++i)
{
auto const* const key = &*strings.insert(gtr_get_host_from_url(tr_torrentTracker(tor, i).announce)).first;
if (auto const count = hosts_hash.find(key); count == hosts_hash.end())
{
hosts_hash.emplace(key, 0);
hosts.push_back(key);
}
keys.insert(key);
auto const view = tr_torrentTracker(tor, i);
torrent_sites_and_hosts.try_emplace(view.sitename, view.host);
}
for (auto const* const key : keys)
for (auto const& [sitename, host] : torrent_sites_and_hosts)
{
++hosts_hash.at(key);
auto& info = site_infos[sitename];
info.sitename = sitename;
info.host = host;
++info.count;
}
++num_torrents;
++n_torrents;
}
std::sort(hosts.begin(), hosts.end(), [](auto const* lhs, auto const& rhs) { return *lhs < *rhs; });
auto const n_sites = std::size(site_infos);
auto sites_v = std::vector<site_info>(n_sites);
std::transform(std::begin(site_infos), std::end(site_infos), std::begin(sites_v), [](auto const& it) { return it.second; });
std::sort(std::begin(sites_v), std::end(sites_v));
// update the "all" count
auto iter = tracker_model->children().begin();
if (iter)
{
tracker_model_update_count(iter, num_torrents);
tracker_model_update_count(iter, n_torrents);
}
// offset past the "All" and the separator
@ -216,9 +211,9 @@ bool tracker_filter_model_update(Glib::RefPtr<Gtk::TreeStore> const& tracker_mod
for (;;)
{
// are we done yet?
bool const new_hosts_done = i >= hosts.size();
bool const old_hosts_done = !iter;
if (new_hosts_done && old_hosts_done)
bool const new_sites_done = i >= n_sites;
bool const old_sites_done = !iter;
if (new_sites_done && old_sites_done)
{
break;
}
@ -226,18 +221,18 @@ bool tracker_filter_model_update(Glib::RefPtr<Gtk::TreeStore> const& tracker_mod
// decide what to do
bool remove_row = false;
bool insert_row = false;
if (new_hosts_done)
if (new_sites_done)
{
remove_row = true;
}
else if (old_hosts_done)
else if (old_sites_done)
{
insert_row = true;
}
else
{
auto const host = iter->get_value(tracker_filter_cols.host);
int const cmp = host.compare(*hosts.at(i));
auto const sitename = iter->get_value(tracker_filter_cols.sitename);
int const cmp = sitename.compare(sites_v.at(i).sitename);
if (cmp < 0)
{
@ -257,16 +252,16 @@ bool tracker_filter_model_update(Glib::RefPtr<Gtk::TreeStore> const& tracker_mod
else if (insert_row)
{
auto* session = static_cast<tr_session*>(tracker_model->get_data(SESSION_KEY));
auto const* host = hosts.at(i);
auto const& site = sites_v.at(i);
auto const add = tracker_model->insert(iter);
add->set_value(tracker_filter_cols.host, *host);
add->set_value(tracker_filter_cols.name, get_name_from_host(*host));
add->set_value(tracker_filter_cols.count, hosts_hash.at(host));
add->set_value(tracker_filter_cols.sitename, Glib::ustring{ site.sitename });
add->set_value(tracker_filter_cols.displayname, get_name_from_host(site.sitename));
add->set_value(tracker_filter_cols.count, site.count);
add->set_value(tracker_filter_cols.type, static_cast<int>(TRACKER_FILTER_TYPE_HOST));
auto path = tracker_model->get_path(add);
gtr_get_favicon(
session,
*host,
site.host,
[ref = Gtk::TreeRowReference(tracker_model, path)](auto const& pixbuf) mutable
{ favicon_ready_cb(pixbuf, ref); });
// ++iter;
@ -274,9 +269,7 @@ bool tracker_filter_model_update(Glib::RefPtr<Gtk::TreeStore> const& tracker_mod
}
else // update row
{
auto const* const host = hosts.at(i);
auto const count = hosts_hash.at(host);
tracker_model_update_count(iter, count);
tracker_model_update_count(iter, sites_v.at(i).count);
++iter;
++i;
}
@ -290,7 +283,7 @@ Glib::RefPtr<Gtk::TreeStore> tracker_filter_model_new(Glib::RefPtr<Gtk::TreeMode
auto const store = Gtk::TreeStore::create(tracker_filter_cols);
auto iter = store->append();
iter->set_value(tracker_filter_cols.name, Glib::ustring(_("All")));
iter->set_value(tracker_filter_cols.displayname, Glib::ustring(_("All")));
iter->set_value(tracker_filter_cols.type, static_cast<int>(TRACKER_FILTER_TYPE_ALL));
iter = store->append();
@ -360,7 +353,7 @@ Gtk::ComboBox* FilterBar::Impl::tracker_combo_box_new(Glib::RefPtr<Gtk::TreeMode
{
auto* r = Gtk::make_managed<Gtk::CellRendererText>();
c->pack_start(*r, false);
c->add_attribute(r->property_text(), tracker_filter_cols.name);
c->add_attribute(r->property_text(), tracker_filter_cols.displayname);
}
{
@ -391,7 +384,7 @@ bool test_tracker(tr_torrent const* tor, int active_tracker_type, Glib::ustring
for (size_t i = 0, n = tr_torrentTrackerCount(tor); i < n; ++i)
{
if (gtr_get_host_from_url(tr_torrentTracker(tor, i).announce) == host)
if (tr_torrentTracker(tor, i).sitename == host)
{
return true;
}
@ -653,7 +646,7 @@ bool FilterBar::Impl::is_row_visible(Gtk::TreeModel::const_iterator const& iter)
{
auto* tor = static_cast<tr_torrent*>(iter->get_value(torrent_cols.torrent));
return tor != nullptr && test_tracker(tor, active_tracker_type_, active_tracker_host_) &&
return tor != nullptr && test_tracker(tor, active_tracker_type_, active_tracker_sitename_) &&
test_torrent_activity(tor, active_activity_type_) && testText(tor, filter_text_);
}
@ -673,12 +666,12 @@ void FilterBar::Impl::selection_changed_cb()
if (auto const iter = tracker_->get_active(); iter)
{
active_tracker_type_ = iter->get_value(tracker_filter_cols.type);
active_tracker_host_ = iter->get_value(tracker_filter_cols.host);
active_tracker_sitename_ = iter->get_value(tracker_filter_cols.sitename);
}
else
{
active_tracker_type_ = TRACKER_FILTER_TYPE_ALL;
active_tracker_host_.clear();
active_tracker_sitename_.clear();
}
/* refilter */

View File

@ -123,37 +123,6 @@ Glib::ustring tr_strltime(time_t seconds)
}
}
/* pattern-matching text; ie, legaltorrents.com */
Glib::ustring gtr_get_host_from_url(Glib::ustring const& url)
{
Glib::ustring host;
if (auto const pch = url.find("://"); pch != Glib::ustring::npos)
{
auto const hostend = url.find_first_of(":/", pch + 3);
host = url.substr(pch + 3, hostend == Glib::ustring::npos ? hostend : (hostend - pch - 3));
}
if (tr_addressIsIP(host.c_str()))
{
return url;
}
else
{
auto const first_dot = host.find('.');
auto const last_dot = host.rfind('.');
if (first_dot != Glib::ustring::npos && last_dot != Glib::ustring::npos && first_dot != last_dot)
{
return host.substr(first_dot + 1);
}
else
{
return host;
}
}
}
namespace
{

View File

@ -61,9 +61,6 @@ Glib::ustring tr_strltime(time_t secs);
****
***/
/* http://www.legaltorrents.com/some/announce/url --> legaltorrents.com */
Glib::ustring gtr_get_host_from_url(Glib::ustring const& url);
bool gtr_is_magnet_link(Glib::ustring const& str);
bool gtr_is_hex_hashcode(std::string const& str);

View File

@ -257,6 +257,7 @@ include_directories(
${CRYPTO_INCLUDE_DIRS}
${CURL_INCLUDE_DIRS}
${EVENT2_INCLUDE_DIRS}
${PSL_INCLUDE_DIRS}
${NATPMP_INCLUDE_DIRS}
${MINIUPNPC_INCLUDE_DIRS}
${DHT_INCLUDE_DIRS}
@ -280,6 +281,7 @@ foreach(UT ${EVENT2_UPSTREAM_TARGET}
${DHT_UPSTREAM_TARGET}
${DEFLATE_UPSTREAM_TARGET}
${UTP_UPSTREAM_TARGET}
${PSL_UPSTREAM_TARGET}
${B64_UPSTREAM_TARGET})
add_dependencies(${TR_NAME} ${UT})
endforeach()
@ -290,6 +292,7 @@ target_link_libraries(${TR_NAME}
${CRYPTO_LIBRARIES}
${CURL_LIBRARIES}
${EVENT2_LIBRARIES}
${PSL_LIBRARIES}
${NATPMP_LIBRARIES}
${MINIUPNPC_LIBRARIES}
${DHT_LIBRARIES}

View File

@ -236,6 +236,7 @@ struct tr_tracker
explicit tr_tracker(tr_announcer* announcer, tr_announce_list::tracker_info const& info)
: host{ info.host }
, announce_url{ info.announce_str }
, sitename{ info.announce.sitename }
, scrape_info{ std::empty(info.scrape_str) ? nullptr : tr_announcerGetScrapeInfo(announcer, info.scrape_str) }
, id{ info.id }
{
@ -270,6 +271,7 @@ struct tr_tracker
tr_interned_string const host;
tr_interned_string const announce_url;
std::string_view const sitename;
tr_scrape_info* const scrape_info;
std::string tracker_id;
@ -1575,6 +1577,10 @@ static tr_tracker_view trackerView(tr_torrent const& tor, int tier_index, tr_tie
view.host = tracker.host.c_str();
view.announce = tracker.announce_url.c_str();
view.scrape = tracker.scrape_info == nullptr ? "" : tracker.scrape_info->scrape_url.c_str();
*std::copy_n(
std::begin(tracker.sitename),
std::min(std::size(tracker.sitename), sizeof(view.sitename) - 1),
view.sitename) = '\0';
view.id = tracker.id;
view.tier = tier_index;

View File

@ -18,7 +18,7 @@ using namespace std::literals;
namespace
{
auto constexpr my_static = std::array<std::string_view, 383>{ ""sv,
auto constexpr my_static = std::array<std::string_view, 384>{ ""sv,
"activeTorrentCount"sv,
"activity-date"sv,
"activityDate"sv,
@ -327,6 +327,7 @@ auto constexpr my_static = std::array<std::string_view, 383>{ ""sv,
"show-statusbar"sv,
"show-toolbar"sv,
"show-tracker-scrapes"sv,
"sitename"sv,
"size-bytes"sv,
"size-units"sv,
"sizeWhenDone"sv,

View File

@ -330,6 +330,7 @@ enum
TR_KEY_show_statusbar,
TR_KEY_show_toolbar,
TR_KEY_show_tracker_scrapes,
TR_KEY_sitename,
TR_KEY_size_bytes,
TR_KEY_size_units,
TR_KEY_sizeWhenDone,

View File

@ -410,23 +410,25 @@ static void addTrackers(tr_torrent const* tor, tr_variant* trackers)
{
for (auto const& tracker : tor->announceList())
{
tr_variant* d = tr_variantListAddDict(trackers, 4);
auto* const d = tr_variantListAddDict(trackers, 5);
tr_variantDictAddQuark(d, TR_KEY_announce, tracker.announce_str.quark());
tr_variantDictAddInt(d, TR_KEY_id, tracker.id);
tr_variantDictAddQuark(d, TR_KEY_scrape, tracker.scrape_str.quark());
tr_variantDictAddStrView(d, TR_KEY_sitename, tracker.announce.sitename);
tr_variantDictAddInt(d, TR_KEY_tier, tracker.tier);
}
}
static void addTrackerStats(tr_tracker_view const& tracker, tr_variant* list)
{
auto* const d = tr_variantListAddDict(list, 26);
auto* const d = tr_variantListAddDict(list, 27);
tr_variantDictAddStr(d, TR_KEY_announce, tracker.announce);
tr_variantDictAddInt(d, TR_KEY_announceState, tracker.announceState);
tr_variantDictAddInt(d, TR_KEY_downloadCount, tracker.downloadCount);
tr_variantDictAddBool(d, TR_KEY_hasAnnounced, tracker.hasAnnounced);
tr_variantDictAddBool(d, TR_KEY_hasScraped, tracker.hasScraped);
tr_variantDictAddStr(d, TR_KEY_host, tracker.host);
tr_variantDictAddStr(d, TR_KEY_sitename, tracker.sitename);
tr_variantDictAddInt(d, TR_KEY_id, tracker.id);
tr_variantDictAddBool(d, TR_KEY_isBackup, tracker.isBackup);
tr_variantDictAddInt(d, TR_KEY_lastAnnouncePeerCount, tracker.lastAnnouncePeerCount);

View File

@ -1369,7 +1369,13 @@ struct tr_tracker_view
{
char const* announce; // full announce URL
char const* scrape; // full scrape URL
char const* host; // human-readable tracker name. (`${host}:${port}`)
char const* host; // uniquely-identifying tracker name (`${host}:${port}`)
// The tracker site's name. Uses the first label before the public suffix
// (https://publicsuffix.org/) in the announce URL's host.
// e.g. "https://www.example.co.uk/announce/"'s sitename is "example"
// RFC 1034 says labels must be less than 64 chars
char sitename[64];
char lastAnnounceResult[128]; // if hasAnnounced, the human-readable result of latest announce
char lastScrapeResult[128]; // if hasScraped, the human-readable result of the latest scrape

View File

@ -15,6 +15,9 @@
#include <event2/buffer.h>
#define PSL_STATIC
#include <libpsl.h>
#include "transmission.h"
#include "net.h"
@ -292,6 +295,48 @@ bool tr_isValidTrackerScheme(std::string_view scheme)
return std::find(std::begin(Schemes), std::end(Schemes), scheme) != std::end(Schemes);
}
// www.example.com -> example
// www.example.co.uk -> example
// 127.0.0.1 -> 127.0.0.1
std::string_view getSiteName(std::string_view host)
{
// is it empty?
if (std::empty(host))
{
return host;
}
// is it an IP?
auto addr = tr_address{};
auto const szhost = std::string(host);
if (tr_address_from_string(&addr, szhost.c_str()))
{
return host;
}
// is it a registered name?
char* lower = nullptr;
if (PSL_SUCCESS == psl_str_to_utf8lower(szhost.c_str(), nullptr, nullptr, &lower))
{
// www.example.com -> example.com
char const* const top = psl_registrable_domain(psl_builtin(), lower);
if (top != nullptr)
{
host.remove_prefix(top - lower);
}
psl_free_string(lower);
}
// example.com -> example
auto const dot_pos = host.find('.');
if (dot_pos != std::string_view::npos)
{
host = host.substr(0, dot_pos);
}
return host;
}
} // namespace
std::optional<tr_url_parsed_t> tr_urlParse(std::string_view url)
@ -336,6 +381,7 @@ std::optional<tr_url_parsed_t> tr_urlParse(std::string_view url)
auto remain = parsed.authority;
parsed.host = tr_strvSep(&remain, ':');
parsed.sitename = getSiteName(parsed.host);
parsed.portstr = !std::empty(remain) ? remain : getPortForScheme(parsed.scheme);
parsed.port = parsePort(parsed.portstr);
}

View File

@ -24,15 +24,18 @@ bool tr_urlIsValid(std::string_view url);
struct tr_url_parsed_t
{
std::string_view scheme;
std::string_view authority;
std::string_view host;
std::string_view path;
std::string_view portstr;
std::string_view query;
std::string_view fragment;
std::string_view full;
int port = -1;
// http://example.com:80/over/there?name=ferret#nose
std::string_view scheme; // "http"
std::string_view authority; // "example.com:80"
std::string_view host; // "example.com"
std::string_view sitename; // "example"
std::string_view portstr; // "80"
std::string_view path; // /"over/there"
std::string_view query; // "name=ferret"
std::string_view fragment; // "nose"
std::string_view full; // "http://example.com:80/over/there?name=ferret#nose"
int port = -1; // 80
};
std::optional<tr_url_parsed_t> tr_urlParse(std::string_view url);

View File

@ -11,49 +11,16 @@
#include <QNetworkRequest>
#include <QStandardPaths>
#include <libtransmission/transmission.h>
#include <libtransmission/web-utils.h> // tr_urlParse()
#include "FaviconCache.h"
/***
****
***/
#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
Q_NETWORK_EXPORT bool qIsEffectiveTLD(QStringView domain);
#endif
namespace
{
QString getTopLevelDomain(QUrl const& url)
{
#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
auto const host = url.host();
auto const dot = QChar(QLatin1Char('.'));
for (auto dot_pos = host.indexOf(dot); dot_pos != -1; dot_pos = host.indexOf(dot, dot_pos + 1))
{
if (qIsEffectiveTLD(QStringView(&host.data()[dot_pos + 1], host.size() - dot_pos - 1)))
{
return host.mid(dot_pos);
}
}
return {};
#else
return url.topLevelDomain();
#endif
}
} // namespace
/***
****
***/
FaviconCache::FaviconCache()
: nam_(new QNetworkAccessManager(this))
{
@ -84,12 +51,12 @@ QString getScrapedFile()
return QDir(base).absoluteFilePath(QStringLiteral("favicons-scraped.txt"));
}
void markUrlAsScraped(QString const& url_str)
void markSiteAsScraped(QString const& sitename)
{
auto skip_file = QFile(getScrapedFile());
if (skip_file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
{
skip_file.write(url_str.toUtf8());
skip_file.write(sitename.toUtf8());
skip_file.write("\n");
}
}
@ -112,22 +79,20 @@ void FaviconCache::ensureCacheDirHasBeenScanned()
{
while (!skip_file.atEnd())
{
auto const url = QString::fromUtf8(skip_file.readLine()).trimmed();
auto const key = getKey(QUrl{ url });
keys_.insert({ url, key });
pixmaps_.try_emplace(key);
auto const sitename = QString::fromUtf8(skip_file.readLine()).trimmed();
pixmaps_.try_emplace(sitename);
}
}
// load the cached favicons
auto cache_dir = QDir(getCacheDir());
cache_dir.mkpath(cache_dir.absolutePath());
for (auto const& file : cache_dir.entryList(QDir::Files | QDir::Readable))
for (auto const& sitename : cache_dir.entryList(QDir::Files | QDir::Readable))
{
QPixmap pixmap(cache_dir.absoluteFilePath(file));
QPixmap pixmap(cache_dir.absoluteFilePath(sitename));
if (!pixmap.isNull())
{
pixmaps_[file] = scale(pixmap);
pixmaps_[sitename] = scale(pixmap);
}
}
}
@ -136,9 +101,9 @@ void FaviconCache::ensureCacheDirHasBeenScanned()
****
***/
QString FaviconCache::getDisplayName(Key const& key)
QString FaviconCache::getDisplayName(QString const& sitename)
{
auto name = key;
auto name = sitename;
if (!name.isEmpty())
{
name.front() = name.front().toTitleCase();
@ -146,111 +111,82 @@ QString FaviconCache::getDisplayName(Key const& key)
return name;
}
FaviconCache::Key FaviconCache::getKey(QUrl const& url)
{
auto host = url.host();
// remove tld
auto const suffix = getTopLevelDomain(url);
host.truncate(host.size() - suffix.size());
// remove subdomain
auto const pos = host.indexOf(QLatin1Char('.'));
return pos < 0 ? host : host.remove(0, pos + 1);
}
FaviconCache::Key FaviconCache::getKey(QString const& displayName)
{
return displayName.toLower();
}
QSize FaviconCache::getIconSize()
{
return { 16, 16 };
}
QPixmap FaviconCache::find(Key const& key)
QPixmap FaviconCache::find(QString const& sitename)
{
ensureCacheDirHasBeenScanned();
return pixmaps_[key];
return pixmaps_[sitename];
}
FaviconCache::Key FaviconCache::add(QString const& url_str)
void FaviconCache::add(QString const& sitename, QString const& url_str)
{
ensureCacheDirHasBeenScanned();
// find or add this url's key
if (auto k_it = keys_.find(url_str); k_it != keys_.end())
{
return k_it->second;
}
auto const url = QUrl{ url_str };
auto const key = getKey(url);
keys_.insert({ url_str, key });
// Try to download a favicon if we don't have one.
// Add a placeholder to prevent repeat downloads.
if (pixmaps_.try_emplace(key).second)
auto const already_had_it = !pixmaps_.try_emplace(sitename).second;
if (already_had_it)
{
markUrlAsScraped(url_str);
auto const scrape = [this](auto const host)
{
auto const schemes = std::array<QString, 2>{
QStringLiteral("http"),
QStringLiteral("https"),
};
auto const suffixes = std::array<QString, 5>{
QStringLiteral("gif"), //
QStringLiteral("ico"), //
QStringLiteral("jpg"), //
QStringLiteral("png"), //
QStringLiteral("svg"), //
};
for (auto const& scheme : schemes)
{
for (auto const& suffix : suffixes)
{
auto const path = QStringLiteral("%1://%2/favicon.%3").arg(scheme).arg(host).arg(suffix);
nam_->get(QNetworkRequest(path));
}
}
};
// tracker.domain.com
auto host = url.host();
scrape(host);
auto const delim = QStringLiteral(".");
auto const has_subdomain = host.count(delim) > 1;
if (has_subdomain)
{
auto const original_subdomain = host.left(host.indexOf(delim));
host.remove(0, original_subdomain.size() + 1);
// domain.com
scrape(host);
auto const www = QStringLiteral("www");
if (original_subdomain != www)
{
// www.domain.com
scrape(QStringLiteral("%1.%2").arg(www).arg(host));
}
}
return;
}
return key;
markSiteAsScraped(sitename);
auto const scrape = [this, sitename](auto const host)
{
auto const schemes = std::array<QString, 2>{
QStringLiteral("http"),
QStringLiteral("https"),
};
auto const suffixes = std::array<QString, 5>{
QStringLiteral("gif"), //
QStringLiteral("ico"), //
QStringLiteral("jpg"), //
QStringLiteral("png"), //
QStringLiteral("svg"), //
};
for (auto const& scheme : schemes)
{
for (auto const& suffix : suffixes)
{
auto const path = QStringLiteral("%1://%2/favicon.%3").arg(scheme).arg(host).arg(suffix);
auto request = QNetworkRequest(path);
request.setAttribute(QNetworkRequest::UserMax, sitename);
nam_->get(request);
}
}
};
// scrape tracker.domain.com
auto const host = QUrl(url_str).host();
scrape(host);
if (auto const idx = host.indexOf(sitename); idx != -1)
{
// scrape domain.com
auto const root = host.mid(idx);
if (root != host)
{
scrape(root);
}
// scrape www.domain.com
if (auto const www = QStringLiteral("www.") + root; www != host)
{
scrape(www);
}
}
}
void FaviconCache::onRequestFinished(QNetworkReply* reply)
{
auto const key = getKey(reply->url());
QPixmap pixmap;
QByteArray const content = reply->readAll();
auto const content = reply->readAll();
auto pixmap = QPixmap{};
if (reply->error() == QNetworkReply::NoError)
{
@ -259,19 +195,21 @@ void FaviconCache::onRequestFinished(QNetworkReply* reply)
if (!pixmap.isNull())
{
auto sitename = reply->request().attribute(QNetworkRequest::UserMax).toString();
// save it in memory...
pixmaps_[key] = scale(pixmap);
pixmaps_[sitename] = scale(pixmap);
// save it on disk...
QDir cache_dir(getCacheDir());
cache_dir.mkpath(cache_dir.absolutePath());
QFile file(cache_dir.absoluteFilePath(key));
QFile file(cache_dir.absoluteFilePath(sitename));
file.open(QIODevice::WriteOnly);
file.write(content);
file.close();
// notify listeners
emit pixmapReady(key);
emit pixmapReady(sitename);
}
reply->deleteLater();

View File

@ -6,14 +6,11 @@
#pragma once
#include <unordered_map>
#include <vector>
#include <QObject>
#include <QPixmap>
#include <QString>
#include <libtransmission/tr-macros.h>
#include "Utils.h" // std::hash<QString>
class QNetworkAccessManager;
@ -23,36 +20,28 @@ class QUrl;
class FaviconCache : public QObject
{
Q_OBJECT
TR_DISABLE_COPY_MOVE(FaviconCache)
public:
FaviconCache();
using Key = QString;
using Keys = std::vector<Key>;
// This will emit a signal when (if) the icon becomes ready.
void add(QString const& sitename, QString const& url);
// returns a cached pixmap, or a nullptr pixmap if there's no match in the cache
QPixmap find(Key const& key);
QPixmap find(QString const& sitename);
static Key getKey(QString const& display_name);
// This will emit a signal when (if) the icon becomes ready.
Key add(QString const& url);
static QString getDisplayName(Key const& key);
static QString getDisplayName(QString const& sitename);
static QSize getIconSize();
signals:
void pixmapReady(Key const& key);
void pixmapReady(QString const& sitename);
private slots:
void onRequestFinished(QNetworkReply* reply);
private:
static Key getKey(QUrl const& url);
void ensureCacheDirHasBeenScanned();
QNetworkAccessManager* nam_ = {};
std::unordered_map<Key, QPixmap> pixmaps_;
std::unordered_map<QString, Key> keys_;
std::unordered_map<QString /*sitename*/, QPixmap> pixmaps_;
};

View File

@ -114,27 +114,26 @@ void FilterBar::refreshTrackers()
ROW_FIRST_TRACKER
};
auto torrents_per_tracker = std::unordered_map<FaviconCache::Key, int>{};
auto torrents_per_sitename = std::unordered_map<QString, int>{};
for (auto const& tor : torrents_.torrents())
{
for (auto const& key : tor->trackerKeys())
for (auto const& sitename : tor->sitenames())
{
++torrents_per_tracker[key];
++torrents_per_sitename[sitename];
}
}
// update the "All" row
auto const num_trackers = torrents_per_tracker.size();
auto const num_trackers = torrents_per_sitename.size();
auto* item = tracker_model_->item(ROW_TOTALS);
item->setData(int(num_trackers), FilterBarComboBox::CountRole);
item->setData(getCountString(num_trackers), FilterBarComboBox::CountStringRole);
auto update_tracker_item = [](QStandardItem* i, auto const& it)
{
auto const& key = it->first;
auto const& display_name = FaviconCache::getDisplayName(key);
auto const& count = it->second;
auto const icon = trApp->faviconCache().find(key);
auto const& [sitename, count] = *it;
auto const display_name = FaviconCache::getDisplayName(sitename);
auto const icon = trApp->faviconCache().find(sitename);
i->setData(display_name, Qt::DisplayRole);
i->setData(display_name, TRACKER_ROLE);
@ -145,10 +144,10 @@ void FilterBar::refreshTrackers()
return i;
};
auto new_trackers = std::map<FaviconCache::Key, int>(torrents_per_tracker.begin(), torrents_per_tracker.end());
auto old_it = tracker_counts_.cbegin();
auto new_trackers = std::map<QString, int>(torrents_per_sitename.begin(), torrents_per_sitename.end());
auto old_it = sitename_counts_.cbegin();
auto new_it = new_trackers.cbegin();
auto const old_end = tracker_counts_.cend();
auto const old_end = sitename_counts_.cend();
auto const new_end = new_trackers.cend();
bool any_added = false;
int row = ROW_FIRST_TRACKER;
@ -181,7 +180,7 @@ void FilterBar::refreshTrackers()
refreshPref(Prefs::FILTER_TRACKERS);
}
tracker_counts_.swap(new_trackers);
sitename_counts_.swap(new_trackers);
}
FilterBarComboBox* FilterBar::createTrackerCombo(QStandardItemModel* model)

View File

@ -57,7 +57,7 @@ private:
TorrentModel const& torrents_;
TorrentFilter const& filter_;
std::map<FaviconCache::Key, int> tracker_counts_;
std::map<QString, int> sitename_counts_;
FilterBarComboBox* activity_combo_ = {};
FilterBarComboBox* tracker_combo_ = {};
QLabel* count_label_ = {};

View File

@ -52,9 +52,9 @@ std::optional<double> Torrent::getSeedRatioLimit() const
return {};
}
bool Torrent::includesTracker(FaviconCache::Key const& key) const
bool Torrent::includesTracker(QString const& sitename) const
{
return std::binary_search(std::begin(tracker_keys_), std::end(tracker_keys_), key);
return std::binary_search(std::begin(sitenames_), std::end(sitenames_), sitename);
}
int Torrent::compareSeedRatio(Torrent const& that) const
@ -269,19 +269,17 @@ Torrent::fields_t Torrent::update(tr_quark const* keys, tr_variant const* const*
{
files_[i].index = i;
}
break;
}
case TR_KEY_trackers:
{
std::set<FaviconCache::Key> tmp;
auto tmp = std::set<QString>{};
for (auto const& ts : tracker_stats_)
{
tmp.insert(ts.favicon_key);
tmp.insert(ts.sitename);
}
tracker_keys_ = FaviconCache::Keys(std::begin(tmp), std::end(tmp));
sitenames_ = std::vector<QString>{ std::begin(tmp), std::end(tmp) };
break;
}
}
@ -357,5 +355,5 @@ QString Torrent::getError() const
QPixmap TrackerStat::getFavicon() const
{
return trApp->faviconCache().find(favicon_key);
return trApp->faviconCache().find(sitename);
}

View File

@ -84,10 +84,10 @@ struct TrackerStat
int scrape_state;
int seeder_count;
int tier;
FaviconCache::Key favicon_key;
QString announce;
QString last_announce_result;
QString last_scrape_result;
QString sitename;
};
using TrackerStatsList = std::vector<TrackerStat>;
@ -406,11 +406,11 @@ public:
return recheck_progress_;
}
bool includesTracker(FaviconCache::Key const& key) const;
bool includesTracker(QString const& sitename) const;
FaviconCache::Keys const& trackerKeys() const
std::vector<QString> const& sitenames() const
{
return tracker_keys_;
return sitenames_;
}
Speed uploadLimit() const
@ -676,7 +676,7 @@ private:
PeerList peers_;
FileList files_;
FaviconCache::Keys tracker_keys_;
std::vector<QString> sitenames_;
TrackerStatsList tracker_stats_;
Speed upload_speed_;

View File

@ -244,8 +244,7 @@ bool TorrentFilter::filterAcceptsRow(int source_row, QModelIndex const& source_p
if (accepts)
{
auto const display_name = prefs_.getString(Prefs::FILTER_TRACKERS);
auto const key = FaviconCache::getKey(display_name);
accepts = key.isEmpty() || tor.includesTracker(key);
accepts = display_name.isEmpty() || tor.includesTracker(display_name.toLower());
}
if (accepts)

View File

@ -121,7 +121,8 @@ bool change(TorrentFile& setme, tr_variant const* value)
bool change(TrackerStat& setme, tr_variant const* value)
{
auto changed = bool{ false };
bool changed = false;
bool site_changed = false;
auto pos = size_t{ 0 };
auto key = tr_quark{};
@ -159,6 +160,7 @@ bool change(TrackerStat& setme, tr_variant const* value)
HANDLE_KEY(nextScrapeTime, next_scrape_time)
HANDLE_KEY(scrapeState, scrape_state)
HANDLE_KEY(seederCount, seeder_count)
HANDLE_KEY(sitename, sitename)
HANDLE_KEY(tier, tier)
#undef HANDLE_KEY
@ -168,14 +170,16 @@ bool change(TrackerStat& setme, tr_variant const* value)
if (field_changed)
{
if (key == TR_KEY_announce)
{
setme.announce = trApp->intern(setme.announce);
setme.favicon_key = trApp->faviconCache().add(setme.announce);
}
changed = true;
site_changed |= key == TR_KEY_announce || key == TR_KEY_sitename;
}
changed = true;
}
if (site_changed && !setme.sitename.isEmpty() && !setme.announce.isEmpty())
{
setme.announce = trApp->intern(setme.announce);
trApp->faviconCache().add(setme.sitename, setme.announce);
}
return changed;

View File

@ -29,6 +29,7 @@ TEST_F(WebUtilsTest, urlParse)
EXPECT_TRUE(parsed);
EXPECT_EQ("http"sv, parsed->scheme);
EXPECT_EQ("1"sv, parsed->host);
EXPECT_EQ("1"sv, parsed->sitename);
EXPECT_EQ(""sv, parsed->path);
EXPECT_EQ("80"sv, parsed->portstr);
EXPECT_EQ(""sv, parsed->query);
@ -40,6 +41,7 @@ TEST_F(WebUtilsTest, urlParse)
EXPECT_TRUE(parsed);
EXPECT_EQ("http"sv, parsed->scheme);
EXPECT_EQ("www.some-tracker.org"sv, parsed->host);
EXPECT_EQ("some-tracker"sv, parsed->sitename);
EXPECT_EQ("/some/path"sv, parsed->path);
EXPECT_EQ(""sv, parsed->query);
EXPECT_EQ(""sv, parsed->fragment);
@ -51,6 +53,7 @@ TEST_F(WebUtilsTest, urlParse)
EXPECT_TRUE(parsed);
EXPECT_EQ("http"sv, parsed->scheme);
EXPECT_EQ("www.some-tracker.org"sv, parsed->host);
EXPECT_EQ("some-tracker"sv, parsed->sitename);
EXPECT_EQ("/some/path"sv, parsed->path);
EXPECT_EQ(""sv, parsed->query);
EXPECT_EQ(""sv, parsed->fragment);
@ -62,6 +65,7 @@ TEST_F(WebUtilsTest, urlParse)
EXPECT_TRUE(parsed);
EXPECT_EQ("http"sv, parsed->scheme);
EXPECT_EQ("www.some-tracker.org"sv, parsed->host);
EXPECT_EQ("some-tracker"sv, parsed->sitename);
EXPECT_EQ("/some/path"sv, parsed->path);
EXPECT_EQ("key=val&foo=bar"sv, parsed->query);
EXPECT_EQ("fragment"sv, parsed->fragment);
@ -79,6 +83,7 @@ TEST_F(WebUtilsTest, urlParse)
EXPECT_TRUE(parsed);
EXPECT_EQ("magnet"sv, parsed->scheme);
EXPECT_EQ(""sv, parsed->host);
EXPECT_EQ(""sv, parsed->sitename);
EXPECT_EQ(""sv, parsed->path);
EXPECT_EQ(
"xt=urn:btih:14ffe5dd23188fd5cb53a1d47f1289db70abf31e"
@ -88,6 +93,39 @@ TEST_F(WebUtilsTest, urlParse)
"&ws=http%3A%2F%2Ftransmissionbt.com"sv,
parsed->query);
EXPECT_EQ(""sv, parsed->portstr);
// test a host whose public suffix contains >1 dot
url = "https://www.example.co.uk:8080/some/path"sv;
parsed = tr_urlParse(url);
EXPECT_TRUE(parsed);
EXPECT_EQ("https"sv, parsed->scheme);
EXPECT_EQ("example"sv, parsed->sitename);
EXPECT_EQ("www.example.co.uk"sv, parsed->host);
EXPECT_EQ("/some/path"sv, parsed->path);
EXPECT_EQ("8080"sv, parsed->portstr);
EXPECT_EQ(8080, parsed->port);
// test a host that lacks a subdomain
url = "http://some-tracker.co.uk/some/other/path"sv;
parsed = tr_urlParse(url);
EXPECT_TRUE(parsed);
EXPECT_EQ("http"sv, parsed->scheme);
EXPECT_EQ("some-tracker"sv, parsed->sitename);
EXPECT_EQ("some-tracker.co.uk"sv, parsed->host);
EXPECT_EQ("/some/other/path"sv, parsed->path);
EXPECT_EQ("80"sv, parsed->portstr);
EXPECT_EQ(80, parsed->port);
// test a host with an IP address
url = "https://127.0.0.1:8080/some/path"sv;
parsed = tr_urlParse(url);
EXPECT_TRUE(parsed);
EXPECT_EQ("https"sv, parsed->scheme);
EXPECT_EQ("127.0.0.1"sv, parsed->sitename);
EXPECT_EQ("127.0.0.1"sv, parsed->host);
EXPECT_EQ("/some/path"sv, parsed->path);
EXPECT_EQ("8080"sv, parsed->portstr);
EXPECT_EQ(8080, parsed->port);
}
TEST_F(WebUtilsTest, urlNextQueryPair)

1
third-party/libpsl vendored Submodule

@ -0,0 +1 @@
Subproject commit 692c69d2415e1b20378030c08607935a54434087

File diff suppressed because one or more lines are too long

View File

@ -738,7 +738,7 @@ export class Inspector extends EventTarget {
element.classList.add('tier-list-tracker');
setTextContent(
element,
`${tracker.domain || tracker.host || tracker.announce} - tier ${
`${tracker.sitename || tracker.host || tracker.announce} - tier ${
tracker.tier + 1
}`
);

View File

@ -7,46 +7,6 @@ import { Formatter } from './formatter.js';
import { Prefs } from './prefs.js';
import { deepEqual } from './utils.js';
/// DOMAINS
// example: "tracker.ubuntu.com" returns "ubuntu.com"
function getDomainName(host) {
const dot = host.indexOf('.');
if (dot !== host.lastIndexOf('.')) {
host = host.slice(dot + 1);
}
return host;
}
// example: "ubuntu.com" returns "Ubuntu"
function getReadableDomain(name) {
if (name.length > 0) {
name = name.charAt(0).toUpperCase() + name.slice(1);
}
const dot = name.indexOf('.');
if (dot !== -1) {
name = name.slice(0, dot);
}
return name;
}
// key: url string
// val: { domain, readable_domain }
const announce_to_domain_cache = {};
function getAnnounceDomain(announce) {
if (announce_to_domain_cache[announce]) {
return announce_to_domain_cache[announce];
}
const url = new URL(announce);
const domain = getDomainName(url.host);
const name = getReadableDomain(domain);
const o = { domain, name, url };
announce_to_domain_cache[announce] = o;
return o;
}
///
export class Torrent extends EventTarget {
@ -270,13 +230,7 @@ export class Torrent extends EventTarget {
return this.fields.totalSize;
}
getTrackers() {
const trackers = this.fields.trackers || [];
for (const tracker of trackers) {
if (tracker.announce && !tracker.domain) {
Object.assign(tracker, getAnnounceDomain(tracker.announce));
}
}
return this.fields.trackers;
return this.fields.trackers || [];
}
getUploadSpeed() {
return this.fields.rateUpload;

View File

@ -869,22 +869,29 @@ TODO: fix this when notifications get fixed
setTextContent(document.querySelector('#filter-count'), string);
}
static _displayName(hostname) {
let name = hostname;
if (name.length > 0) {
name = name.charAt(0).toUpperCase() + name.slice(1);
}
return name;
}
_updateFilterSelect() {
const trackers = this._getTrackers();
const names = Object.keys(trackers).sort();
const trackers = this._getTrackerCounts();
const sitenames = Object.keys(trackers).sort();
// build the new html
let string = '';
string += !this.filterTracker
? '<option value="all" selected="selected">All</option>'
: '<option value="all">All</option>';
for (const name of names) {
const o = trackers[name];
string += `<option value="${o.domain}"`;
if (trackers[name].domain === this.filterTracker) {
for (const sitename of sitenames) {
string += `<option value="${sitename}"`;
if (sitename === this.filterTracker) {
string += ' selected="selected"';
}
string += `>${name}</option>`;
string += `>${Transmission._displayName(sitename)}</option>`;
}
if (!this.filterTrackersStr || this.filterTrackersStr !== string) {
@ -1059,36 +1066,25 @@ TODO: fix this when notifications get fixed
}
}
setFilterTracker(domain) {
setFilterTracker(sitename) {
const e = document.querySelector('#filter-tracker');
e.value = domain ? Transmission._getReadableDomain(domain) : 'all';
e.value = sitename ? Transmission._getReadableDomain(sitename) : 'all';
this.filterTracker = domain;
this.filterTracker = sitename;
this.refilterAllSoon();
}
_getTrackers() {
const returnValue = {};
_getTrackerCounts() {
const counts = {};
for (const torrent of this._getAllTorrents()) {
const names = new Set();
for (const tracker of torrent.getTrackers()) {
const { domain, name } = tracker;
if (!returnValue[name]) {
returnValue[name] = { count: 0, domain };
}
names.add(name);
}
for (const name of names.values()) {
++returnValue[name].count;
const { sitename } = tracker;
counts[sitename] = (counts[sitename] || 0) + 1;
}
}
return returnValue;
return counts;
}
///