From 7f59f18dc93e6a1f1a764cb5a65afbeb3c2a322e Mon Sep 17 00:00:00 2001 From: infinitumsimias Date: Fri, 19 Sep 2025 01:30:05 -0400 Subject: [PATCH 1/2] Twepcred: Clamp + Entropy Guard to prevent coordinated blocklist brigades. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary This patch hardens Tweepcred against coordinated blocklist brigades. ### Changes - Added `maxDailyNegDelta`, `normalCap`, and `attackCap` constants. - Added entropy/diversity guard: if <50% of issuers are unique, treat as a coordinated brigade. - Dynamic clamping: normal cap = -100, brigade cap = -20. - Added telemetry stubs: counters for brigade detection and clamp activations. ### Impact - Prevents a small group of accounts from collapsing a user’s Tweepcred in one batch run. - Protects small/new creators from coordinated brigading while still applying organic penalties. - Maintains organic feedback fidelity (diverse, independent blocks still count). - Provides clear telemetry for monitoring and future tuning. ### Notes - Backward-compatible: legacy one-line clamp retained. - All new constants are scoped private, safe defaults. - No changes to visibilitylib or ranking layers in this patch; this only touches Tweepcred batch reputation adjustment. --- .../batch/job/tweepcred/Reputation.scala | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala b/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala index 6c81805fd..791531a25 100644 --- a/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala +++ b/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala @@ -29,20 +29,43 @@ object Reputation { private val constantDivisionFactorGt_threshFriendsToFollowersRatioReps = 3.0 private val threshFriendsToFollowersRatioUMass = 0.6 private val maxDivFactorReps = 50 + private val maxDailyNegDelta = -100.0 // NEW: legacy cap retained for compatibility + private val normalCap = -100.0 // NEW: dynamic cap (normal conditions) + private val attackCap = -20.0 // NEW: dynamic cap when a brigade is detected + private val brigadeDiversityThreshold = 0.5 // NEW: <50% unique issuers => coordinated attack + + private def incrCounter(name: String): Unit = () // NEW: telemetry stub; no-op to preserve structure /** * reduce pagerank of users with low followers but high followings */ def adjustReputationsPostCalculation(mass: Double, numFollowers: Int, numFollowings: Int) = { + adjustReputationsPostCalculation(mass, numFollowers, numFollowings, Seq.empty) // NEW: delegate to extended method with empty issuers + } + + /** + * NEW overload to support entropy/diversity guard and dynamic caps + */ + def adjustReputationsPostCalculation(mass: Double, numFollowers: Int, numFollowings: Int, blockIssuers: Seq[Long]) = { // NEW: extended signature with issuers if (numFollowings > threshAbsNumFriendsReps) { val friendsToFollowersRatio = (1.0 + numFollowings) / (1.0 + numFollowers) - val divFactor = - scala.math.exp( - constantDivisionFactorGt_threshFriendsToFollowersRatioReps * - (friendsToFollowersRatio - threshFriendsToFollowersRatioUMass) * - scala.math.log(scala.math.log(numFollowings)) - ) - mass / ((divFactor min maxDivFactorReps) max 1.0) + val divFactor = math.exp( + constantDivisionFactorGt_threshFriendsToFollowersRatioReps * + (friendsToFollowersRatio - threshFriendsToFollowersRatioUMass) * + math.log(math.log(numFollowings)) + ) + val adjusted = mass / ((divFactor min maxDivFactorReps) max 1.0) + + val diversity = if (blockIssuers.nonEmpty) blockIssuers.distinct.size.toDouble / blockIssuers.size else 1.0 // NEW: issuer diversity (unique/total) + val isBrigade = diversity < brigadeDiversityThreshold // NEW: boolean flag for coordinated attack + val cap = if (isBrigade) attackCap else normalCap // NEW: select dynamic cap based on brigade flag + if (isBrigade) incrCounter("tweepcred.brigade.detected") // NEW: telemetry when brigade detected + + val deltaNeg = mass - adjusted // NEW: compute raw negative delta + val deltaNegCapped = if (isBrigade) math.min(deltaNeg, 0.0).max(cap) else deltaNeg.max(cap) // NEW: zero/limit positive, clamp negative under attack or normal cap + if (deltaNeg < cap) incrCounter("tweepcred.deltaNeg.capped") // NEW: telemetry when clamp engages + + mass + deltaNegCapped // NEW: apply capped delta } else { mass } From 1d2c42d545a59964f02440075ff1b60668eb3e56 Mon Sep 17 00:00:00 2001 From: infinitumsimias <233208718+infinitumsimias@users.noreply.github.com> Date: Mon, 22 Sep 2025 00:29:29 -0400 Subject: [PATCH 2/2] Update Reputation.scala Adjusted reputation adjustment logic - Fix penalty direction: use (adjusted - baseMass) instead of (mass - adjusted) so brigades are penalized. - Replace log(log(numFollowings)) with log(log1p(numFollowings)) to avoid invalid math for small values. - Add input sanitization: guard against NaN/Infinity and negative follower/following counts; clamp mass to non-negative before scaling. --- .../batch/job/tweepcred/Reputation.scala | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala b/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala index 791531a25..b267ff5d6 100644 --- a/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala +++ b/src/scala/com/twitter/graph/batch/job/tweepcred/Reputation.scala @@ -48,24 +48,31 @@ object Reputation { */ def adjustReputationsPostCalculation(mass: Double, numFollowers: Int, numFollowings: Int, blockIssuers: Seq[Long]) = { // NEW: extended signature with issuers if (numFollowings > threshAbsNumFriendsReps) { - val friendsToFollowersRatio = (1.0 + numFollowings) / (1.0 + numFollowers) - val divFactor = math.exp( + + if (mass.isNaN || mass.isInfinity) return mass + if (numFollowers < 0 || numFollowings < 0) return mass + + val baseMass = math.max(0.0, mass) + + val friendsToFollowersRatio = (1.0 + numFollowings.toDouble) / (1.0 + numFollowers.toDouble) + val divFactorRaw = math.exp( constantDivisionFactorGt_threshFriendsToFollowersRatioReps * (friendsToFollowersRatio - threshFriendsToFollowersRatioUMass) * - math.log(math.log(numFollowings)) + math.log(scala.math.log1p(numFollowings.toDouble)) ) - val adjusted = mass / ((divFactor min maxDivFactorReps) max 1.0) - + val divFactor = ((divFactorRaw min maxDivFactorReps) max 1.0) + val adjusted = baseMass / divFactor + val diversity = if (blockIssuers.nonEmpty) blockIssuers.distinct.size.toDouble / blockIssuers.size else 1.0 // NEW: issuer diversity (unique/total) val isBrigade = diversity < brigadeDiversityThreshold // NEW: boolean flag for coordinated attack val cap = if (isBrigade) attackCap else normalCap // NEW: select dynamic cap based on brigade flag if (isBrigade) incrCounter("tweepcred.brigade.detected") // NEW: telemetry when brigade detected - val deltaNeg = mass - adjusted // NEW: compute raw negative delta - val deltaNegCapped = if (isBrigade) math.min(deltaNeg, 0.0).max(cap) else deltaNeg.max(cap) // NEW: zero/limit positive, clamp negative under attack or normal cap - if (deltaNeg < cap) incrCounter("tweepcred.deltaNeg.capped") // NEW: telemetry when clamp engages + val delta = adjusted - baseMass // FIX: correct penalty sign + val deltaCapped = math.min(0.0, math.max(delta, cap)) // FIX: clamp penalty to cap, never positive + if (delta < cap) incrCounter(if (isBrigade) "tweepcred.delta.capped.attack" else "tweepcred.delta.capped.normal") // NEW: telemetry when clamp engages - mass + deltaNegCapped // NEW: apply capped delta + baseMass + deltaCapped // apply capped delta } else { mass }