Tracker announce & scrape improvements 🥳 (#901)

* Tracker announce & scrape improvements

* Fixed null variables

Co-Authored-By: Constantine Kovalensky <45331093+kovalensky@users.noreply.github.com>

* Hybrid topics handling in announcer

* Updated

* Use correct data types

* Revert "Use correct data types"

This reverts commit bc9f0e4d6f.

* Update announce.php

* Showing downloaded times in stats

* Updated

* Updated

* Debugging variables

* Update scrape.php

* Updated

* Update announce.php

* Update announce.php

* Code re-formatting

* Code re-formatting

* Updated

* Update config.php

* Update tr_make_snapshot.php

* Update tr_complete_count.php

* Stopped event should be handled before cache check

* Include torrent download name in magnet link

* Magnet name re-formatting

* Smart cache management

---------

Co-authored-by: Constantine Kovalensky <45331093+kovalensky@users.noreply.github.com>

bt/announce.php

@@ -18,11 +18,6 @@ if (empty($_SERVER['HTTP_USER_AGENT'])) {
     die;
 }
 
-// Ignore 'completed' event
-if (isset($_GET['event']) && $_GET['event'] === 'completed') {
-    dummy_exit(random_int(600, 1200));
-}
-
 $announce_interval = $bb_cfg['announce_interval'];
 $passkey_key = $bb_cfg['passkey_key'];
 
@@ -31,6 +26,9 @@ if (isset($_GET['?info_hash']) && !isset($_GET['info_hash'])) {
     $_GET['info_hash'] = $_GET['?info_hash'];
 }
 
+$is_bt_v2 = null;
+$is_hybrid = $hybrid_v1_hash = $hybrid_v2_hash = $hybrid_tor_update = false;
+
 // Initial request verification
 if (strpos($_SERVER['REQUEST_URI'], 'scrape') !== false) {
     msg_die('Please disable SCRAPE!');
@@ -60,10 +58,10 @@ $passkey = ${$passkey_key} ?? null;
 // Verify request
 // Required params (info_hash, peer_id, port, uploaded, downloaded, left, passkey)
 if (!isset($info_hash)) {
-    msg_die('info_hash does not exist');
+    msg_die('info_hash was not provided');
 }
 if (!isset($peer_id) || strlen($peer_id) != 20) {
-    msg_die('Invalid peer_id: ' . $peer_id);
+    msg_die('Invalid peer_id: ' . bin2hex($peer_id));
 }
 
 // Check info_hash version
@@ -72,7 +70,7 @@ if (strlen($info_hash) == 32) {
 } elseif (strlen($info_hash) == 20) {
     $is_bt_v2 = false;
 } else {
-    msg_die('Invalid info_hash: ' . $info_hash);
+    msg_die('Invalid info_hash: ' . bin2hex($info_hash));
 }
 
 if (!isset($port) || $port < 0 || $port > 0xFFFF) {
@@ -119,30 +117,27 @@ if (!\TorrentPier\Helpers\IPHelper::isValid($ip)) {
 $ip_sql = \TorrentPier\Helpers\IPHelper::ip2long($ip);
 
 // Peer unique id
-$peer_hash = md5(
+$peer_hash = md5(rtrim($info_hash, ' ') . $passkey . $ip . $port);
-    rtrim($info_hash, ' ') . $passkey . $ip . $port
+
-);
+// Events
+$stopped = ($event === 'stopped');
+
+// Set seeder & complete
+$complete = $seeder = ($left == 0) ? 1 : 0;
 
 // Get cached peer info from previous announce (last peer info)
 $lp_info = CACHE('tr_cache')->get(PEER_HASH_PREFIX . $peer_hash);
 
+// Stopped event, slice peer's cache life to 30 seconds
+if ($stopped && $lp_info) {
+    CACHE('tr_cache')->set(PEER_HASH_PREFIX . $peer_hash, $lp_info, 30);
+}
+
 // Drop fast announce
-if ($lp_info && (!isset($event) || $event !== 'stopped')) {
+if ($lp_info && (!isset($event) || !$stopped)) {
-    drop_fast_announce($lp_info);
+    if ($lp_cached_peers = CACHE('tr_cache')->get(PEERS_LIST_PREFIX . $lp_info['topic_id'])) {
-}
+        drop_fast_announce($lp_info, $lp_cached_peers); // Use cache but with new calculated interval and seed, peer count set
-
+    }
-// Events
-$seeder = ($left == 0) ? 1 : 0;
-$stopped = ($event === 'stopped');
-$completed = ($event === 'completed');
-
-// Stopped event
-if ($stopped) {
-    CACHE('tr_cache')->rm(PEER_HASH_PREFIX . $peer_hash);
-}
-
-// Completed event
-if ($completed) {
 }
 
 // Get last peer info from DB
@@ -153,10 +148,6 @@ if (!CACHE('tr_cache')->used && !$lp_info) {
 }
 
 if ($lp_info) {
-    if (!$stopped) {
-        drop_fast_announce($lp_info);
-    }
-
     $user_id = $lp_info['user_id'];
     $topic_id = $lp_info['topic_id'];
     $releaser = $lp_info['releaser'];
@@ -179,24 +170,29 @@ if ($lp_info) {
 	";
     $row = DB()->fetch_row($sql);
 
+    // Verify if torrent registered on tracker and user authorized
+    if (empty($row['topic_id'])) {
+        msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
+    }
+    if (empty($row['user_id'])) {
+        msg_die('Please LOG IN and RE-DOWNLOAD this torrent (user not found)');
+    }
+
     // Assign variables
     $user_id = $row['user_id'];
     $topic_id = $row['topic_id'];
     $releaser = (int)($user_id == $row['poster_id']);
     $tor_type = $row['tor_type'];
 
-    // Check user and topic id
-    if (empty($topic_id)) {
-        msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
-    }
-    if (empty($user_id)) {
-        msg_die('Please LOG IN and RE-DOWNLOAD this torrent (user not found)');
-    }
-
     // Check hybrid torrents
-    $is_hybrid = false;
     if (!empty($row['info_hash']) && !empty($row['info_hash_v2'])) {
+        // Helpful dev variables
         $is_hybrid = true;
+        $hybrid_v1_hash = &$row['info_hash'];
+        $hybrid_v2_hash = &$row['info_hash_v2'];
+        if ($info_hash === $hybrid_v1_hash) {
+            $hybrid_tor_update = true;
+        }
     }
 
     // Ratio limits
@@ -296,38 +292,41 @@ if ($bb_cfg['tracker']['freeleech'] && $down_add) {
 // Insert / update peer info
 $peer_info_updated = false;
 $update_time = ($stopped) ? 0 : TIMENOW;
+if (($is_hybrid && $hybrid_tor_update) || !$is_hybrid) { // Update statistics only for one topic
+    if ($lp_info) {
+        $sql = "UPDATE " . BB_BT_TRACKER . " SET update_time = $update_time";
 
-if ($lp_info) {
+        $sql .= ", seeder = $seeder";
-    $sql = "UPDATE " . BB_BT_TRACKER . " SET update_time = $update_time";
+        $sql .= ($releaser != $lp_info['releaser']) ? ", releaser = $releaser" : '';
 
-    $sql .= ", seeder = $seeder";
+        $sql .= ($tor_type != $lp_info['tor_type']) ? ", tor_type = $tor_type" : '';
-    $sql .= ($releaser != $lp_info['releaser']) ? ", releaser = $releaser" : '';
 
-    $sql .= ($tor_type != $lp_info['tor_type']) ? ", tor_type = $tor_type" : '';
+        $sql .= ($uploaded != $lp_info['uploaded']) ? ", uploaded = $uploaded" : '';
+        $sql .= ($downloaded != $lp_info['downloaded']) ? ", downloaded = $downloaded" : '';
+        $sql .= ", remain = $left";
 
-    $sql .= ($uploaded != $lp_info['uploaded']) ? ", uploaded = $uploaded" : '';
+        $sql .= $up_add ? ", up_add = up_add + $up_add" : '';
-    $sql .= ($downloaded != $lp_info['downloaded']) ? ", downloaded = $downloaded" : '';
+        $sql .= $down_add ? ", down_add = down_add + $down_add" : '';
-    $sql .= ", remain = $left";
 
-    $sql .= $up_add ? ", up_add = up_add + $up_add" : '';
+        $sql .= ", speed_up = $speed_up";
-    $sql .= $down_add ? ", down_add = down_add + $down_add" : '';
+        $sql .= ", speed_down = $speed_down";
 
-    $sql .= ", speed_up = $speed_up";
+        $sql .= ", complete = $complete";
-    $sql .= ", speed_down = $speed_down";
 
-    $sql .= " WHERE peer_hash = '$peer_hash'";
+        $sql .= " WHERE peer_hash = '$peer_hash'";
-    $sql .= " LIMIT 1";
+        $sql .= " LIMIT 1";
 
-    DB()->query($sql);
+        DB()->query($sql);
 
-    $peer_info_updated = DB()->affected_rows();
+        $peer_info_updated = DB()->affected_rows();
-}
+    }
 
-if (!$lp_info || !$peer_info_updated) {
+    if (!$lp_info || !$peer_info_updated) {
-    $columns = 'peer_hash, topic_id, user_id, ip, port, seeder, releaser, tor_type, uploaded, downloaded, remain, speed_up, speed_down, up_add, down_add, update_time';
+        $columns = 'peer_hash, topic_id, user_id, ip, port, seeder, releaser, tor_type, uploaded, downloaded, remain, speed_up, speed_down, up_add, down_add, update_time, complete';
-    $values = "'$peer_hash', $topic_id, $user_id, '$ip_sql', $port, $seeder, $releaser, $tor_type, $uploaded, $downloaded, $left, $speed_up, $speed_down, $up_add, $down_add, $update_time";
+        $values = "'$peer_hash', $topic_id, $user_id, '$ip_sql', $port, $seeder, $releaser, $tor_type, $uploaded, $downloaded, $left, $speed_up, $speed_down, $up_add, $down_add, $update_time, $complete";
 
-    DB()->query("REPLACE INTO " . BB_BT_TRACKER . " ($columns) VALUES ($values)");
+        DB()->query("REPLACE INTO " . BB_BT_TRACKER . " ($columns) VALUES ($values)");
+    }
 }
 
 // Exit if stopped
@@ -345,6 +344,7 @@ $lp_info = [
     'uploaded' => (float)$uploaded,
     'user_id' => (int)$user_id,
     'tor_type' => (int)$tor_type,
+    'complete' => (int)$complete,
 ];
 
 $lp_info_cached = CACHE('tr_cache')->set(PEER_HASH_PREFIX . $peer_hash, $lp_info, PEER_HASH_EXPIRE);
@@ -379,18 +379,19 @@ if (!$output) {
         }
     }
 
-    $seeders = $leechers = 0;
+    $seeders = $leechers = $client_completed = 0;
 
     if ($bb_cfg['tracker']['scrape']) {
         $row = DB()->fetch_row("
-			SELECT seeders, leechers
+			SELECT seeders, leechers, completed
 			FROM " . BB_BT_TRACKER_SNAP . "
 			WHERE topic_id = $topic_id
 			LIMIT 1
 		");
 
-        $seeders = $row['seeders'] ?? 0;
+        $seeders = $row['seeders'] ?? ($seeder ? 1 : 0);
-        $leechers = $row['leechers'] ?? 0;
+        $leechers = $row['leechers'] ?? (!$seeder ? 1 : 0);
+        $client_completed = $row['completed'] ?? 0;
     }
 
     $output = [
@@ -398,6 +399,8 @@ if (!$output) {
         'min interval' => (int)$announce_interval,
         'complete' => (int)$seeders,
         'incomplete' => (int)$leechers,
+        'downloaded' => (int)$client_completed,
+        'warning message' => 'Statistics were updated',
         'peers' => $peers,
     ];
 

bt/includes/init_tr.php

@@ -37,7 +37,7 @@ function error_exit($msg = '')
     exit;
 }
 
-function drop_fast_announce($lp_info)
+function drop_fast_announce($lp_info, $lp_cached_peers = [])
 {
     global $announce_interval;
 
@@ -47,7 +47,7 @@ function drop_fast_announce($lp_info)
 
     $new_ann_intrv = $lp_info['update_time'] + $announce_interval - TIMENOW;
 
-    dummy_exit($new_ann_intrv);
+    dummy_exit($new_ann_intrv, $lp_cached_peers);
 }
 
 function msg_die($msg)
@@ -55,19 +55,28 @@ function msg_die($msg)
     $output = \SandFox\Bencode\Bencode::encode([
         'min interval' => (int)1800,
         'failure reason' => (string)$msg,
-        'warning message' => (string)$msg,
     ]);
 
     die($output);
 }
 
-function dummy_exit($interval = 1800)
+function dummy_exit($interval = 1800, $cache_dict = [])
 {
-    $output = \SandFox\Bencode\Bencode::encode([
+    $output = [
         'interval' => (int)$interval,
         'min interval' => (int)$interval,
         'peers' => (string)DUMMY_PEER,
-    ]);
+    ];
+
+    if (!empty($cache_dict)) {
+        $output['complete'] = $cache_dict['complete'];
+        $output['incomplete'] = $cache_dict['incomplete'];
+        $output['downloaded'] = $cache_dict['downloaded'];
+        $output['warning message'] = 'Next statistics update in: ' . (floor($interval / 60) % 60) . ' minutes';
+        $output['peers'] = $cache_dict['peers'];
+    }
+
+    $output = \SandFox\Bencode\Bencode::encode($output);
 
     die($output);
 }

bt/scrape.php

@@ -27,16 +27,23 @@ $info_hash = isset($_GET['info_hash']) ? (string)$_GET['info_hash'] : null;
 
 // Verify info_hash
 if (!isset($info_hash)) {
-    msg_die('info_hash does not exist');
+    msg_die('info_hash was not provided');
 }
 
+// Store info hash in hex format
+$info_hash_hex = bin2hex($info_hash);
+
 // Check info_hash version
 if (strlen($info_hash) == 32) {
     $is_bt_v2 = true;
 } elseif (strlen($info_hash) == 20) {
     $is_bt_v2 = false;
 } else {
-    msg_die('Invalid info_hash: ' . $info_hash);
+    msg_die('Invalid info_hash: ' . $info_hash_hex);
+}
+
+if ($lp_scrape_info = CACHE('tr_cache')->get(SCRAPE_LIST_PREFIX . $info_hash_hex)) {
+    die(\SandFox\Bencode\Bencode::encode($lp_scrape_info));
 }
 
 $info_hash_sql = rtrim(DB()->escape($info_hash), ' ');
@@ -55,7 +62,7 @@ $row = DB()->fetch_row("
 ");
 
 if (!$row) {
-    msg_die('Torrent not registered, info_hash = ' . bin2hex($info_hash));
+    msg_die('Torrent not registered, info_hash = ' . $info_hash_hex);
 }
 
 $output['files'][$info_hash] = [
@@ -64,6 +71,8 @@ $output['files'][$info_hash] = [
     'incomplete' => (int)$row['leechers'],
 ];
 
+$peers_list_cached = CACHE('tr_cache')->set(SCRAPE_LIST_PREFIX . $info_hash_hex, $output, SCRAPE_LIST_EXPIRE);
+
 echo \SandFox\Bencode\Bencode::encode($output);
 
 exit;

common.php

@@ -346,9 +346,11 @@ if (!defined('IN_TRACKER')) {
 
     define('PEER_HASH_EXPIRE', round($bb_cfg['announce_interval'] * (0.85 * $bb_cfg['tracker']['expire_factor'])));
     define('PEERS_LIST_EXPIRE', round($bb_cfg['announce_interval'] * 0.7));
+    define('SCRAPE_LIST_EXPIRE', round($bb_cfg['scrape_interval'] * 0.7));
 
     define('PEER_HASH_PREFIX', 'peer_');
     define('PEERS_LIST_PREFIX', 'peers_list_');
+    define('SCRAPE_LIST_PREFIX', 'scrape_list_');
 
     header('Content-Type: text/plain');
     header('Pragma: no-cache');

install/sql/mysql.sql

@@ -374,7 +374,7 @@ CREATE TABLE IF NOT EXISTS `bb_bt_tracker`
   `down_add`         BIGINT(20) UNSIGNED   NOT NULL DEFAULT '0',
   `update_time`      INT(11)               NOT NULL DEFAULT '0',
   `complete_percent` BIGINT(20)            NOT NULL DEFAULT '0',
-  `complete`         INT(11)               NOT NULL DEFAULT '0',
+  `complete`         TINYINT(1)            NOT NULL DEFAULT '0',
   PRIMARY KEY (`peer_hash`),
   KEY `topic_id` (`topic_id`),
   KEY `user_id` (`user_id`)
@@ -397,6 +397,7 @@ CREATE TABLE IF NOT EXISTS `bb_bt_tracker_snap`
   `leechers`   MEDIUMINT(8) UNSIGNED NOT NULL DEFAULT '0',
   `speed_up`   INT(10) UNSIGNED      NOT NULL DEFAULT '0',
   `speed_down` INT(10) UNSIGNED      NOT NULL DEFAULT '0',
+  `completed`  INT(10)               NOT NULL DEFAULT '0',
   PRIMARY KEY (`topic_id`)
 )
   ENGINE = MyISAM

install/upgrade/changes.txt

@@ -68,3 +68,5 @@ INSERT INTO `bb_cron` (`cron_active`, `cron_title`, `cron_script`, `schedule`, `
 ALTER TABLE `bb_posts_text` CHANGE `post_text` `post_text` MEDIUMTEXT NOT NULL;
 ALTER TABLE `bb_privmsgs_text` CHANGE `privmsgs_text` `privmsgs_text` MEDIUMTEXT NOT NULL;
 ALTER TABLE `bb_bt_torrents` ADD COLUMN `info_hash_v2` VARBINARY(32) NOT NULL DEFAULT '';
+ALTER TABLE `bb_bt_tracker_snap` ADD COLUMN `completed` INT(10) NOT NULL DEFAULT '0';
+ALTER TABLE `bb_bt_tracker` CHANGE `complete` `complete` TINYINT(1) NOT NULL DEFAULT '0';

library/attach_mod/displaying_torrent.php

@@ -141,10 +141,9 @@ if ($tor_reged && $tor_info) {
     $tor_completed_count = declension((int)$tor_info['complete_count'], 'times');
     $tor_id = $tor_info['topic_id'];
     $tor_type = $tor_info['tor_type'];
-
     // Magnet link
     $user_passkey = \TorrentPier\Legacy\Torrent::getPasskey($bt_user_id);
-    $tor_magnet = create_magnet($tor_info['info_hash'], $tor_info['info_hash_v2'], $user_passkey);
+    $tor_magnet = create_magnet($tor_info['info_hash'], $tor_info['info_hash_v2'], $user_passkey, wbr($t_data['topic_title']));
 
     // ratio limits
     $min_ratio_dl = $bb_cfg['bt_min_ratio_allow_dl_tor'];

library/config.php

@@ -66,7 +66,7 @@ $bb_cfg['cache'] = [
     'redis' => [
         'host' => '127.0.0.1',
         'port' => 6379,
-        'pconnect' => true,
+        'pconnect' => PHP_ZTS ? false : true,
         'con_required' => true,
     ],
     // Available cache types: filecache, memcache, sqlite, redis, apcu (filecache by default)
@@ -94,7 +94,8 @@ $bb_cfg['script_path'] = '/'; // The path where FORUM is located relative to the
 $bb_cfg['gzip_compress'] = false; // compress output
 
 // Tracker
-$bb_cfg['announce_interval'] = 2400; // Announce interval (default: 2400)
+$bb_cfg['announce_interval'] = 1800; // Announce interval (default: 1800)
+$bb_cfg['scrape_interval'] = 60; // Scrape interval (default: 60)
 $bb_cfg['passkey_key'] = 'uk'; // Passkey key name in GET request
 $bb_cfg['ignore_reported_ip'] = false; // Ignore IP reported by client
 $bb_cfg['verify_reported_ip'] = true; // Verify IP reported by client against $_SERVER['HTTP_X_FORWARDED_FOR']

library/includes/cron/jobs/tr_complete_count.php

@@ -13,6 +13,18 @@ if (!defined('BB_ROOT')) {
 
 global $bb_cfg;
 
+// Update TORRENT "completed" counters
+DB()->query("
+	UPDATE
+		" . BB_BT_TORRENTS . " tor,
+		" . BB_BT_TRACKER_SNAP . " snap
+	SET
+		tor.complete_count = snap.completed
+	WHERE
+		tor.topic_id = snap.topic_id
+");
+
+/*
 if ($bb_cfg['ocelot']['enabled']) {
     // Update TORRENT "completed" counters
     DB()->query("
@@ -51,4 +63,4 @@ if ($bb_cfg['ocelot']['enabled']) {
 
     // Drop tmp table
     DB()->query("DROP TEMPORARY TABLE tmp_complete_count");
-}
+}*/

library/includes/cron/jobs/tr_make_snapshot.php

@@ -41,7 +41,7 @@ while (true) {
         $sql = "
 			SELECT
 				topic_id, SUM(seeder) AS seeders, (COUNT(*) - SUM(seeder)) AS leechers,
-				SUM(speed_up) AS speed_up, SUM(speed_down) AS speed_down
+				SUM(speed_up) AS speed_up, SUM(speed_down) AS speed_down, SUM(complete) AS completed
 			FROM " . BB_BT_TRACKER . "
 			WHERE topic_id BETWEEN $start_id AND $end_id
 			GROUP BY topic_id
@@ -64,7 +64,7 @@ while (true) {
         if (!$bb_cfg['ocelot']['enabled']) {
             DB()->query("
 				REPLACE INTO " . NEW_BB_BT_TRACKER_SNAP . "
-				(topic_id, seeders, leechers, speed_up, speed_down)
+				(topic_id, seeders, leechers, speed_up, speed_down, completed)
 				VALUES(" . implode('),(', $val) . ")
 			");
         } else {

library/includes/functions.php

@@ -1773,10 +1773,11 @@ function decode_text_match($txt)
  * @param string $infohash
  * @param string $infohash_v2
  * @param string $auth_key
+ * @param string $name
  *
  * @return string
  */
-function create_magnet(string $infohash, string $infohash_v2, string $auth_key): string
+function create_magnet(string $infohash, string $infohash_v2, string $auth_key, string $name): string
 {
     global $bb_cfg, $images;
 
@@ -1798,7 +1799,7 @@ function create_magnet(string $infohash, string $infohash_v2, string $auth_key):
         $magnet .= 'xt=urn:btmh:1220' . bin2hex($infohash_v2);
     }
 
-    return '<a href="' . $magnet . '&tr=' . urlencode($bb_cfg['bt_announce_url'] . "?{$bb_cfg['passkey_key']}=$auth_key") . '"><img src="' . $images['icon_magnet'] . '" width="12" height="12" border="0" /></a>';
+    return '<a href="' . $magnet . '&tr=' . urlencode($bb_cfg['bt_announce_url'] . "?{$bb_cfg['passkey_key']}=$auth_key") . '&dn=' . urlencode($name) . '"><img src="' . $images['icon_magnet'] . '" width="12" height="12" border="0" /></a>';
 }
 
 function set_die_append_msg($forum_id = null, $topic_id = null, $group_id = null)

tracker.php

@@ -702,7 +702,7 @@ if ($allowed_forums) {
             $s_last = $tor['seeder_last_seen'];
             $att_id = $tor['attach_id'];
             $size = $tor['size'];
-            $tor_magnet = create_magnet($tor['info_hash'], $tor['info_hash_v2'], \TorrentPier\Legacy\Torrent::getPasskey($user_id));
+            $tor_magnet = create_magnet($tor['info_hash'], $tor['info_hash_v2'], \TorrentPier\Legacy\Torrent::getPasskey($user_id), wbr($tor['topic_title']));
             $compl = $tor['complete_count'];
             $dl_sp = ($dl) ? humn_size($dl, 0, 'KB') . '/s' : '0 KB/s';
             $ul_sp = ($ul) ? humn_size($ul, 0, 'KB') . '/s' : '0 KB/s';

viewforum.php

@@ -466,7 +466,7 @@ foreach ($topic_rowset as $topic) {
     ));
 
     if (isset($topic['tor_size'])) {
-        $tor_magnet = create_magnet($topic['info_hash'], $topic['info_hash_v2'], $topic['auth_key']);
+        $tor_magnet = create_magnet($topic['info_hash'], $topic['info_hash_v2'], $topic['auth_key'], wbr($topic['topic_title']));
 
         $template->assign_block_vars('t.tor', array(
             'SEEDERS' => (int)$topic['seeders'],