Tracker announce & scrape improvements 🥳 (#901)

* Tracker announce & scrape improvements

* Fixed null variables

Co-Authored-By: Constantine Kovalensky <45331093+kovalensky@users.noreply.github.com>

* Hybrid topics handling in announcer

* Updated

* Use correct data types

* Revert "Use correct data types"

This reverts commit bc9f0e4d6f.

* Update announce.php

* Showing downloaded times in stats

* Updated

* Updated

* Debugging variables

* Update scrape.php

* Updated

* Update announce.php

* Update announce.php

* Code re-formatting

* Code re-formatting

* Updated

* Update config.php

* Update tr_make_snapshot.php

* Update tr_complete_count.php

* Stopped event should be handled before cache check

* Include torrent download name in magnet link

* Magnet name re-formatting

* Smart cache management

---------

Co-authored-by: Constantine Kovalensky <45331093+kovalensky@users.noreply.github.com>
This commit is contained in:
Roman Kelesidis 2023-09-14 08:38:07 +07:00 committed by GitHub
parent cc07ecb56c
commit c654a9ae36
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 125 additions and 86 deletions

View file

@ -18,11 +18,6 @@ if (empty($_SERVER['HTTP_USER_AGENT'])) {
die;
}
// Ignore 'completed' event
if (isset($_GET['event']) && $_GET['event'] === 'completed') {
dummy_exit(random_int(600, 1200));
}
$announce_interval = $bb_cfg['announce_interval'];
$passkey_key = $bb_cfg['passkey_key'];
@ -31,6 +26,9 @@ if (isset($_GET['?info_hash']) && !isset($_GET['info_hash'])) {
$_GET['info_hash'] = $_GET['?info_hash'];
}
$is_bt_v2 = null;
$is_hybrid = $hybrid_v1_hash = $hybrid_v2_hash = $hybrid_tor_update = false;
// Initial request verification
if (strpos($_SERVER['REQUEST_URI'], 'scrape') !== false) {
msg_die('Please disable SCRAPE!');
@ -60,10 +58,10 @@ $passkey = ${$passkey_key} ?? null;
// Verify request
// Required params (info_hash, peer_id, port, uploaded, downloaded, left, passkey)
if (!isset($info_hash)) {
msg_die('info_hash does not exist');
msg_die('info_hash was not provided');
}
if (!isset($peer_id) || strlen($peer_id) != 20) {
msg_die('Invalid peer_id: ' . $peer_id);
msg_die('Invalid peer_id: ' . bin2hex($peer_id));
}
// Check info_hash version
@ -72,7 +70,7 @@ if (strlen($info_hash) == 32) {
} elseif (strlen($info_hash) == 20) {
$is_bt_v2 = false;
} else {
msg_die('Invalid info_hash: ' . $info_hash);
msg_die('Invalid info_hash: ' . bin2hex($info_hash));
}
if (!isset($port) || $port < 0 || $port > 0xFFFF) {
@ -119,30 +117,27 @@ if (!\TorrentPier\Helpers\IPHelper::isValid($ip)) {
$ip_sql = \TorrentPier\Helpers\IPHelper::ip2long($ip);
// Peer unique id
$peer_hash = md5(
rtrim($info_hash, ' ') . $passkey . $ip . $port
);
$peer_hash = md5(rtrim($info_hash, ' ') . $passkey . $ip . $port);
// Events
$stopped = ($event === 'stopped');
// Set seeder & complete
$complete = $seeder = ($left == 0) ? 1 : 0;
// Get cached peer info from previous announce (last peer info)
$lp_info = CACHE('tr_cache')->get(PEER_HASH_PREFIX . $peer_hash);
// Stopped event, slice peer's cache life to 30 seconds
if ($stopped && $lp_info) {
CACHE('tr_cache')->set(PEER_HASH_PREFIX . $peer_hash, $lp_info, 30);
}
// Drop fast announce
if ($lp_info && (!isset($event) || $event !== 'stopped')) {
drop_fast_announce($lp_info);
}
// Events
$seeder = ($left == 0) ? 1 : 0;
$stopped = ($event === 'stopped');
$completed = ($event === 'completed');
// Stopped event
if ($stopped) {
CACHE('tr_cache')->rm(PEER_HASH_PREFIX . $peer_hash);
}
// Completed event
if ($completed) {
if ($lp_info && (!isset($event) || !$stopped)) {
if ($lp_cached_peers = CACHE('tr_cache')->get(PEERS_LIST_PREFIX . $lp_info['topic_id'])) {
drop_fast_announce($lp_info, $lp_cached_peers); // Use cache but with new calculated interval and seed, peer count set
}
}
// Get last peer info from DB
@ -153,10 +148,6 @@ if (!CACHE('tr_cache')->used && !$lp_info) {
}
if ($lp_info) {
if (!$stopped) {
drop_fast_announce($lp_info);
}
$user_id = $lp_info['user_id'];
$topic_id = $lp_info['topic_id'];
$releaser = $lp_info['releaser'];
@ -179,24 +170,29 @@ if ($lp_info) {
";
$row = DB()->fetch_row($sql);
// Verify if torrent registered on tracker and user authorized
if (empty($row['topic_id'])) {
msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
}
if (empty($row['user_id'])) {
msg_die('Please LOG IN and RE-DOWNLOAD this torrent (user not found)');
}
// Assign variables
$user_id = $row['user_id'];
$topic_id = $row['topic_id'];
$releaser = (int)($user_id == $row['poster_id']);
$tor_type = $row['tor_type'];
// Check user and topic id
if (empty($topic_id)) {
msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
}
if (empty($user_id)) {
msg_die('Please LOG IN and RE-DOWNLOAD this torrent (user not found)');
}
// Check hybrid torrents
$is_hybrid = false;
if (!empty($row['info_hash']) && !empty($row['info_hash_v2'])) {
// Helpful dev variables
$is_hybrid = true;
$hybrid_v1_hash = &$row['info_hash'];
$hybrid_v2_hash = &$row['info_hash_v2'];
if ($info_hash === $hybrid_v1_hash) {
$hybrid_tor_update = true;
}
}
// Ratio limits
@ -296,38 +292,41 @@ if ($bb_cfg['tracker']['freeleech'] && $down_add) {
// Insert / update peer info
$peer_info_updated = false;
$update_time = ($stopped) ? 0 : TIMENOW;
if (($is_hybrid && $hybrid_tor_update) || !$is_hybrid) { // Update statistics only for one topic
if ($lp_info) {
$sql = "UPDATE " . BB_BT_TRACKER . " SET update_time = $update_time";
if ($lp_info) {
$sql = "UPDATE " . BB_BT_TRACKER . " SET update_time = $update_time";
$sql .= ", seeder = $seeder";
$sql .= ($releaser != $lp_info['releaser']) ? ", releaser = $releaser" : '';
$sql .= ", seeder = $seeder";
$sql .= ($releaser != $lp_info['releaser']) ? ", releaser = $releaser" : '';
$sql .= ($tor_type != $lp_info['tor_type']) ? ", tor_type = $tor_type" : '';
$sql .= ($tor_type != $lp_info['tor_type']) ? ", tor_type = $tor_type" : '';
$sql .= ($uploaded != $lp_info['uploaded']) ? ", uploaded = $uploaded" : '';
$sql .= ($downloaded != $lp_info['downloaded']) ? ", downloaded = $downloaded" : '';
$sql .= ", remain = $left";
$sql .= ($uploaded != $lp_info['uploaded']) ? ", uploaded = $uploaded" : '';
$sql .= ($downloaded != $lp_info['downloaded']) ? ", downloaded = $downloaded" : '';
$sql .= ", remain = $left";
$sql .= $up_add ? ", up_add = up_add + $up_add" : '';
$sql .= $down_add ? ", down_add = down_add + $down_add" : '';
$sql .= $up_add ? ", up_add = up_add + $up_add" : '';
$sql .= $down_add ? ", down_add = down_add + $down_add" : '';
$sql .= ", speed_up = $speed_up";
$sql .= ", speed_down = $speed_down";
$sql .= ", speed_up = $speed_up";
$sql .= ", speed_down = $speed_down";
$sql .= ", complete = $complete";
$sql .= " WHERE peer_hash = '$peer_hash'";
$sql .= " LIMIT 1";
$sql .= " WHERE peer_hash = '$peer_hash'";
$sql .= " LIMIT 1";
DB()->query($sql);
DB()->query($sql);
$peer_info_updated = DB()->affected_rows();
}
$peer_info_updated = DB()->affected_rows();
}
if (!$lp_info || !$peer_info_updated) {
$columns = 'peer_hash, topic_id, user_id, ip, port, seeder, releaser, tor_type, uploaded, downloaded, remain, speed_up, speed_down, up_add, down_add, update_time';
$values = "'$peer_hash', $topic_id, $user_id, '$ip_sql', $port, $seeder, $releaser, $tor_type, $uploaded, $downloaded, $left, $speed_up, $speed_down, $up_add, $down_add, $update_time";
if (!$lp_info || !$peer_info_updated) {
$columns = 'peer_hash, topic_id, user_id, ip, port, seeder, releaser, tor_type, uploaded, downloaded, remain, speed_up, speed_down, up_add, down_add, update_time, complete';
$values = "'$peer_hash', $topic_id, $user_id, '$ip_sql', $port, $seeder, $releaser, $tor_type, $uploaded, $downloaded, $left, $speed_up, $speed_down, $up_add, $down_add, $update_time, $complete";
DB()->query("REPLACE INTO " . BB_BT_TRACKER . " ($columns) VALUES ($values)");
DB()->query("REPLACE INTO " . BB_BT_TRACKER . " ($columns) VALUES ($values)");
}
}
// Exit if stopped
@ -345,6 +344,7 @@ $lp_info = [
'uploaded' => (float)$uploaded,
'user_id' => (int)$user_id,
'tor_type' => (int)$tor_type,
'complete' => (int)$complete,
];
$lp_info_cached = CACHE('tr_cache')->set(PEER_HASH_PREFIX . $peer_hash, $lp_info, PEER_HASH_EXPIRE);
@ -379,18 +379,19 @@ if (!$output) {
}
}
$seeders = $leechers = 0;
$seeders = $leechers = $client_completed = 0;
if ($bb_cfg['tracker']['scrape']) {
$row = DB()->fetch_row("
SELECT seeders, leechers
SELECT seeders, leechers, completed
FROM " . BB_BT_TRACKER_SNAP . "
WHERE topic_id = $topic_id
LIMIT 1
");
$seeders = $row['seeders'] ?? 0;
$leechers = $row['leechers'] ?? 0;
$seeders = $row['seeders'] ?? ($seeder ? 1 : 0);
$leechers = $row['leechers'] ?? (!$seeder ? 1 : 0);
$client_completed = $row['completed'] ?? 0;
}
$output = [
@ -398,6 +399,8 @@ if (!$output) {
'min interval' => (int)$announce_interval,
'complete' => (int)$seeders,
'incomplete' => (int)$leechers,
'downloaded' => (int)$client_completed,
'warning message' => 'Statistics were updated',
'peers' => $peers,
];

View file

@ -37,7 +37,7 @@ function error_exit($msg = '')
exit;
}
function drop_fast_announce($lp_info)
function drop_fast_announce($lp_info, $lp_cached_peers = [])
{
global $announce_interval;
@ -47,7 +47,7 @@ function drop_fast_announce($lp_info)
$new_ann_intrv = $lp_info['update_time'] + $announce_interval - TIMENOW;
dummy_exit($new_ann_intrv);
dummy_exit($new_ann_intrv, $lp_cached_peers);
}
function msg_die($msg)
@ -55,19 +55,28 @@ function msg_die($msg)
$output = \SandFox\Bencode\Bencode::encode([
'min interval' => (int)1800,
'failure reason' => (string)$msg,
'warning message' => (string)$msg,
]);
die($output);
}
function dummy_exit($interval = 1800)
function dummy_exit($interval = 1800, $cache_dict = [])
{
$output = \SandFox\Bencode\Bencode::encode([
$output = [
'interval' => (int)$interval,
'min interval' => (int)$interval,
'peers' => (string)DUMMY_PEER,
]);
];
if (!empty($cache_dict)) {
$output['complete'] = $cache_dict['complete'];
$output['incomplete'] = $cache_dict['incomplete'];
$output['downloaded'] = $cache_dict['downloaded'];
$output['warning message'] = 'Next statistics update in: ' . (floor($interval / 60) % 60) . ' minutes';
$output['peers'] = $cache_dict['peers'];
}
$output = \SandFox\Bencode\Bencode::encode($output);
die($output);
}

View file

@ -27,16 +27,23 @@ $info_hash = isset($_GET['info_hash']) ? (string)$_GET['info_hash'] : null;
// Verify info_hash
if (!isset($info_hash)) {
msg_die('info_hash does not exist');
msg_die('info_hash was not provided');
}
// Store info hash in hex format
$info_hash_hex = bin2hex($info_hash);
// Check info_hash version
if (strlen($info_hash) == 32) {
$is_bt_v2 = true;
} elseif (strlen($info_hash) == 20) {
$is_bt_v2 = false;
} else {
msg_die('Invalid info_hash: ' . $info_hash);
msg_die('Invalid info_hash: ' . $info_hash_hex);
}
if ($lp_scrape_info = CACHE('tr_cache')->get(SCRAPE_LIST_PREFIX . $info_hash_hex)) {
die(\SandFox\Bencode\Bencode::encode($lp_scrape_info));
}
$info_hash_sql = rtrim(DB()->escape($info_hash), ' ');
@ -55,7 +62,7 @@ $row = DB()->fetch_row("
");
if (!$row) {
msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
msg_die('Torrent not registered, info_hash = ' . $info_hash_hex);
}
$output['files'][$info_hash] = [
@ -64,6 +71,8 @@ $output['files'][$info_hash] = [
'incomplete' => (int)$row['leechers'],
];
$peers_list_cached = CACHE('tr_cache')->set(SCRAPE_LIST_PREFIX . $info_hash_hex, $output, SCRAPE_LIST_EXPIRE);
echo \SandFox\Bencode\Bencode::encode($output);
exit;

View file

@ -346,9 +346,11 @@ if (!defined('IN_TRACKER')) {
define('PEER_HASH_EXPIRE', round($bb_cfg['announce_interval'] * (0.85 * $bb_cfg['tracker']['expire_factor'])));
define('PEERS_LIST_EXPIRE', round($bb_cfg['announce_interval'] * 0.7));
define('SCRAPE_LIST_EXPIRE', round($bb_cfg['scrape_interval'] * 0.7));
define('PEER_HASH_PREFIX', 'peer_');
define('PEERS_LIST_PREFIX', 'peers_list_');
define('SCRAPE_LIST_PREFIX', 'scrape_list_');
header('Content-Type: text/plain');
header('Pragma: no-cache');

View file

@ -374,7 +374,7 @@ CREATE TABLE IF NOT EXISTS `bb_bt_tracker`
`down_add` BIGINT(20) UNSIGNED NOT NULL DEFAULT '0',
`update_time` INT(11) NOT NULL DEFAULT '0',
`complete_percent` BIGINT(20) NOT NULL DEFAULT '0',
`complete` INT(11) NOT NULL DEFAULT '0',
`complete` TINYINT(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`peer_hash`),
KEY `topic_id` (`topic_id`),
KEY `user_id` (`user_id`)
@ -397,6 +397,7 @@ CREATE TABLE IF NOT EXISTS `bb_bt_tracker_snap`
`leechers` MEDIUMINT(8) UNSIGNED NOT NULL DEFAULT '0',
`speed_up` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`speed_down` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`completed` INT(10) NOT NULL DEFAULT '0',
PRIMARY KEY (`topic_id`)
)
ENGINE = MyISAM

View file

@ -68,3 +68,5 @@ INSERT INTO `bb_cron` (`cron_active`, `cron_title`, `cron_script`, `schedule`, `
ALTER TABLE `bb_posts_text` CHANGE `post_text` `post_text` MEDIUMTEXT NOT NULL;
ALTER TABLE `bb_privmsgs_text` CHANGE `privmsgs_text` `privmsgs_text` MEDIUMTEXT NOT NULL;
ALTER TABLE `bb_bt_torrents` ADD COLUMN `info_hash_v2` VARBINARY(32) NOT NULL DEFAULT '';
ALTER TABLE `bb_bt_tracker_snap` ADD COLUMN `completed` INT(10) NOT NULL DEFAULT '0';
ALTER TABLE `bb_bt_tracker` CHANGE `complete` `complete` TINYINT(1) NOT NULL DEFAULT '0';

View file

@ -141,10 +141,9 @@ if ($tor_reged && $tor_info) {
$tor_completed_count = declension((int)$tor_info['complete_count'], 'times');
$tor_id = $tor_info['topic_id'];
$tor_type = $tor_info['tor_type'];
// Magnet link
$user_passkey = \TorrentPier\Legacy\Torrent::getPasskey($bt_user_id);
$tor_magnet = create_magnet($tor_info['info_hash'], $tor_info['info_hash_v2'], $user_passkey);
$tor_magnet = create_magnet($tor_info['info_hash'], $tor_info['info_hash_v2'], $user_passkey, wbr($t_data['topic_title']));
// ratio limits
$min_ratio_dl = $bb_cfg['bt_min_ratio_allow_dl_tor'];

View file

@ -66,7 +66,7 @@ $bb_cfg['cache'] = [
'redis' => [
'host' => '127.0.0.1',
'port' => 6379,
'pconnect' => true,
'pconnect' => PHP_ZTS ? false : true,
'con_required' => true,
],
// Available cache types: filecache, memcache, sqlite, redis, apcu (filecache by default)
@ -94,7 +94,8 @@ $bb_cfg['script_path'] = '/'; // The path where FORUM is located relative to the
$bb_cfg['gzip_compress'] = false; // compress output
// Tracker
$bb_cfg['announce_interval'] = 2400; // Announce interval (default: 2400)
$bb_cfg['announce_interval'] = 1800; // Announce interval (default: 1800)
$bb_cfg['scrape_interval'] = 60; // Scrape interval (default: 60)
$bb_cfg['passkey_key'] = 'uk'; // Passkey key name in GET request
$bb_cfg['ignore_reported_ip'] = false; // Ignore IP reported by client
$bb_cfg['verify_reported_ip'] = true; // Verify IP reported by client against $_SERVER['HTTP_X_FORWARDED_FOR']

View file

@ -13,6 +13,18 @@ if (!defined('BB_ROOT')) {
global $bb_cfg;
// Update TORRENT "completed" counters
DB()->query("
UPDATE
" . BB_BT_TORRENTS . " tor,
" . BB_BT_TRACKER_SNAP . " snap
SET
tor.complete_count = snap.completed
WHERE
tor.topic_id = snap.topic_id
");
/*
if ($bb_cfg['ocelot']['enabled']) {
// Update TORRENT "completed" counters
DB()->query("
@ -51,4 +63,4 @@ if ($bb_cfg['ocelot']['enabled']) {
// Drop tmp table
DB()->query("DROP TEMPORARY TABLE tmp_complete_count");
}
}*/

View file

@ -41,7 +41,7 @@ while (true) {
$sql = "
SELECT
topic_id, SUM(seeder) AS seeders, (COUNT(*) - SUM(seeder)) AS leechers,
SUM(speed_up) AS speed_up, SUM(speed_down) AS speed_down
SUM(speed_up) AS speed_up, SUM(speed_down) AS speed_down, SUM(complete) AS completed
FROM " . BB_BT_TRACKER . "
WHERE topic_id BETWEEN $start_id AND $end_id
GROUP BY topic_id
@ -64,7 +64,7 @@ while (true) {
if (!$bb_cfg['ocelot']['enabled']) {
DB()->query("
REPLACE INTO " . NEW_BB_BT_TRACKER_SNAP . "
(topic_id, seeders, leechers, speed_up, speed_down)
(topic_id, seeders, leechers, speed_up, speed_down, completed)
VALUES(" . implode('),(', $val) . ")
");
} else {

View file

@ -1773,10 +1773,11 @@ function decode_text_match($txt)
* @param string $infohash
* @param string $infohash_v2
* @param string $auth_key
* @param string $name
*
* @return string
*/
function create_magnet(string $infohash, string $infohash_v2, string $auth_key): string
function create_magnet(string $infohash, string $infohash_v2, string $auth_key, string $name): string
{
global $bb_cfg, $images;
@ -1798,7 +1799,7 @@ function create_magnet(string $infohash, string $infohash_v2, string $auth_key):
$magnet .= 'xt=urn:btmh:1220' . bin2hex($infohash_v2);
}
return '<a href="' . $magnet . '&tr=' . urlencode($bb_cfg['bt_announce_url'] . "?{$bb_cfg['passkey_key']}=$auth_key") . '"><img src="' . $images['icon_magnet'] . '" width="12" height="12" border="0" /></a>';
return '<a href="' . $magnet . '&tr=' . urlencode($bb_cfg['bt_announce_url'] . "?{$bb_cfg['passkey_key']}=$auth_key") . '&dn=' . urlencode($name) . '"><img src="' . $images['icon_magnet'] . '" width="12" height="12" border="0" /></a>';
}
function set_die_append_msg($forum_id = null, $topic_id = null, $group_id = null)

View file

@ -702,7 +702,7 @@ if ($allowed_forums) {
$s_last = $tor['seeder_last_seen'];
$att_id = $tor['attach_id'];
$size = $tor['size'];
$tor_magnet = create_magnet($tor['info_hash'], $tor['info_hash_v2'], \TorrentPier\Legacy\Torrent::getPasskey($user_id));
$tor_magnet = create_magnet($tor['info_hash'], $tor['info_hash_v2'], \TorrentPier\Legacy\Torrent::getPasskey($user_id), wbr($tor['topic_title']));
$compl = $tor['complete_count'];
$dl_sp = ($dl) ? humn_size($dl, 0, 'KB') . '/s' : '0 KB/s';
$ul_sp = ($ul) ? humn_size($ul, 0, 'KB') . '/s' : '0 KB/s';

View file

@ -466,7 +466,7 @@ foreach ($topic_rowset as $topic) {
));
if (isset($topic['tor_size'])) {
$tor_magnet = create_magnet($topic['info_hash'], $topic['info_hash_v2'], $topic['auth_key']);
$tor_magnet = create_magnet($topic['info_hash'], $topic['info_hash_v2'], $topic['auth_key'], wbr($topic['topic_title']));
$template->assign_block_vars('t.tor', array(
'SEEDERS' => (int)$topic['seeders'],