EditCounter: + approx autoedits count using tags

assets/js/editcounter.js

@@ -317,7 +317,7 @@ xtools.editcounter.setupMonthYearChart = function (id, datasets, labels, maxTota
 xtools.editcounter.setupSizeHistogram = function (data, colors, barLabels) {
     let bars = 11;
     // First sanitize input, to get array.
-    let total = Object.keys(data).length - 3; // -3 to exclude small edits, large edits and average
+    let total = Object.keys(data).length;
     data.length = total;
     data = Array.from(data)
     // Then make datasets

i18n/en.json

@@ -43,6 +43,7 @@
 	"auto-contribs-use-tool": "Show only edits using the tool:",
 	"auto-edits": "(Semi-)automated edits",
 	"auto-edits-bot": "This user is a bot. All edits may be automated. Statistics shown here are based on edits made using known tools.",
+    "auto-edits-tagged": "Tagged (semi-)automated edits",
 	"auto-edits-counts-desc": "Some tool usage overlaps (e.g. rolling back to a revision that is a redirect counts as Rollback and as Redirect), so the total of edits made with each tool may differ from the $1.",
 	"auto-edits-counts-desc-grand-total": "grand total",
 	"auto-edits-logged-out": "You must login to use the sandbox configuration. Using production configuration instead.",

i18n/qqq.json

@@ -62,6 +62,7 @@
 	"auto-contribs-use-tool": "Label for dropdown that limits results to edits made by the selected (semi-)automated tool. The dropdown is located immediately after the label.",
 	"auto-edits": "(semi-)automated edits with known tools like Huggle",
 	"auto-edits-bot": "Message shown at the top of the AutoEdits tool for users who are a bot, explaining that all edits made by the bot may be automated, even if XTools says they are not.",
+    "auto-edits-tagged": "Label for the very approximate count of tagged (semi-)automated edits in the last 5000 edits in the Edit Counter.",
 	"auto-edits-counts-desc": "In the AutoEdits tool, this message explains how the totals of individual tools (shown in the '(semi-)automated edits' section) may be different from the total shown in the 'Summary' section. $1 is a link to the 'Summary' section with the message 'auto-edits-counts-grand-total' as the link text.",
 	"auto-edits-counts-desc-grand-total": "The text 'grand total'. This is used as the text for the link in the 'auto-edits-counts-desc' message.",
 	"auto-edits-logged-out": "Message shown when the user attempts to use the AutoEdits tool in sandbox mode, but hasn't logged in.",

public/build/entrypoints.json

@@ -4,7 +4,7 @@
       "js": [
         "/build/runtime.c217f8c4.js",
         "/build/852.96913092.js",
-        "/build/app.a92fcf1a.js"
+        "/build/app.93b4ef2d.js"
       ],
       "css": [
         "/build/app.7692d209.css"

public/build/manifest.json

@@ -1,6 +1,6 @@
 {
   "build/app.css": "/build/app.7692d209.css",
-  "build/app.js": "/build/app.a92fcf1a.js",
+  "build/app.js": "/build/app.93b4ef2d.js",
   "build/runtime.js": "/build/runtime.c217f8c4.js",
   "build/852.96913092.js": "/build/852.96913092.js",
   "build/images/VPS-badge.svg": "/build/images/VPS-badge.svg",

src/Controller/EditCounterController.php

@@ -4,6 +4,7 @@
 
 namespace App\Controller;
 
+use App\Helper\AutomatedEditsHelper;
 use App\Model\EditCounter;
 use App\Model\GlobalContribs;
 use App\Model\UserRights;
@@ -92,7 +93,8 @@ public function restrictedApiActions(): array
     protected function setUpEditCounter(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): void {
         // Whether we're making a subrequest (the view makes a request to another action).
         // Subrequests to the same controller do not re-instantiate a new controller, and hence
@@ -119,7 +121,8 @@ protected function setUpEditCounter(
             $this->i18n,
             $this->userRights,
             $this->project,
-            $this->user
+            $this->user,
+            $autoEditsHelper
         );
     }
 
@@ -242,15 +245,17 @@ private function redirectFromSections(): RedirectResponse
      * @param EditCounterRepository $editCounterRepo
      * @param UserRightsRepository $userRightsRepo
      * @param RequestStack $requestStack
+     * @param AutomatedEditsHelper $autoEditsHelper
      * @return Response|RedirectResponse
      * @codeCoverageIgnore
      */
     public function resultAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ) {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         if (1 === count($this->sections)) {
             // Redirect to dedicated route.
@@ -297,9 +302,10 @@ public function generalStatsAction(
         UserRightsRepository $userRightsRepo,
         GlobalContribsRepository $globalContribsRepo,
         EditRepository $editRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $globalContribs = new GlobalContribs(
             $globalContribsRepo,
@@ -358,9 +364,10 @@ public function generalStatsIndexAction(): Response
     public function namespaceTotalsAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = [
             'xtTitle' => $this->user->getUsername(),
@@ -405,9 +412,10 @@ public function namespaceTotalsIndexAction(): Response
     public function timecardAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = [
             'xtTitle' => $this->user->getUsername(),
@@ -453,9 +461,10 @@ public function timecardIndexAction(): Response
     public function yearCountsAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = [
             'xtTitle' => $this->user->getUsername(),
@@ -500,9 +509,10 @@ public function yearCountsIndexAction(): Response
     public function monthCountsAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = [
             'xtTitle' => $this->user->getUsername(),
@@ -548,9 +558,10 @@ public function monthCountsIndexAction(): Response
     public function rightsChangesAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): Response {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = [
             'xtTitle' => $this->user->getUsername(),
@@ -628,9 +639,10 @@ public function rightsChangesIndexAction(): Response
     public function logCountsApiAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): JsonResponse {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         return $this->getFormattedApiResponse([
             'log_counts' => $this->editCounter->getLogCounts(),
@@ -674,9 +686,10 @@ public function logCountsApiAction(
     public function namespaceTotalsApiAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): JsonResponse {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         return $this->getFormattedApiResponse([
             'namespace_totals' => (object)$this->editCounter->namespaceTotals(),
@@ -730,9 +743,10 @@ public function namespaceTotalsApiAction(
     public function monthCountsApiAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): JsonResponse {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         $ret = $this->editCounter->monthCounts();
 
@@ -790,9 +804,10 @@ public function monthCountsApiAction(
     public function timecardApiAction(
         EditCounterRepository $editCounterRepo,
         UserRightsRepository $userRightsRepo,
-        RequestStack $requestStack
+        RequestStack $requestStack,
+        AutomatedEditsHelper $autoEditsHelper
     ): JsonResponse {
-        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack);
+        $this->setUpEditCounter($editCounterRepo, $userRightsRepo, $requestStack, $autoEditsHelper);
 
         return $this->getFormattedApiResponse([
             'timecard' => $this->editCounter->timeCard(),

src/Helper/AutomatedEditsHelper.php

@@ -187,6 +187,18 @@ public function getTools(Project $project, bool $useSandbox = false): array
         return $this->tools[$projectDomain];
     }
 
+    /**
+     * Get all the tags associated to automated edits on a given project.
+     * @param bool $useSandbox Whether to use the /sandbox version for testing (also bypasses caching).
+     * @return array Array with numeric keys and values being tag names (as in change_tag_def).
+     */
+    public function getTags(Project $project, bool $useSandbox = false): array
+    {
+        $tools = $this->getTools($project, $useSandbox);
+        $tags = array_merge(... array_map(fn($o) => $o["tags"], array_values($tools)));
+        return $tags;
+    }
+
     /**
      * Merges the given rule sets, giving priority to the local set. Regex is concatenated, not overridden.
      * @param string[] $globalRules The global rule set.

src/Model/EditCounter.php

@@ -4,6 +4,7 @@
 
 namespace App\Model;
 
+use App\Helper\AutomatedEditsHelper;
 use App\Helper\I18nHelper;
 use App\Repository\EditCounterRepository;
 use DateInterval;
@@ -67,20 +68,23 @@ class EditCounter extends Model
      * @param I18nHelper $i18n
      * @param UserRights $userRights
      * @param Project $project The base project to count edits
+     * @param AutomatedEditsHelper
      * @param User $user
      */
     public function __construct(
         EditCounterRepository $repository,
         I18nHelper $i18n,
         UserRights $userRights,
         Project $project,
-        User $user
+        User $user,
+        AutomatedEditsHelper $autoEditsHelper
     ) {
         $this->repository = $repository;
         $this->i18n = $i18n;
         $this->userRights = $userRights;
         $this->project = $project;
         $this->user = $user;
+        $this->autoEditsHelper = $autoEditsHelper;
     }
 
     /**
@@ -1057,6 +1061,32 @@ public function countLargeEdits(): int
         return isset($editSizeData['large_edits']) ? (int) $editSizeData['large_edits'] : 0;
     }
 
+    /**
+     * Get the number of edits that have automated tags in the user's past 5000 edits.
+     * @return int
+     */
+    public function countAutoEdits(): int
+    {
+        $editSizeData = $this->getEditSizeData();
+        if (!isset($editSizeData['tag_lists'])) {
+            return 0;
+        }
+        $tags = json_decode($editSizeData['tag_lists']);
+        $autoTags = $this->autoEditsHelper->getTags($this->project);
+        return count( // Number
+            array_filter(
+                $tags, // of revisions
+                fn($a) => null !== $a && // with tags
+                count( // where the number of tags
+                    array_filter(
+                        $a,
+                        fn($t) => in_array($t, $autoTags) // that mean these edits are auto
+                    )
+                ) > 0 // is greater than 0
+            )
+        );
+    }
+
     /**
      * Get the average size of the user's past 5000 edits.
      * @return float Size in bytes.

src/Repository/EditCounterRepository.php

@@ -572,6 +572,8 @@ public function getEditSizeData(Project $project, User $user): array
         // Prepare the queries and execute them.
         $revisionTable = $project->getTableName('revision');
         $pageTable = $project->getTableName('page');
+        $ctTable = $project->getTableName('change_tag');
+        $ctdTable = $project->getTableName('change_tag_def');
         $ipcJoin = '';
         $whereClause = 'revs.rev_actor = :actorId';
         $params = ['actorId' => $user->getActorId($project)];
@@ -583,21 +585,34 @@ public function getEditSizeData(Project $project, User $user): array
             $whereClause = 'ipc_hex BETWEEN :startIp AND :endIp';
         }
 
-        $sql = "SELECT (CAST(revs.rev_len AS SIGNED) - IFNULL(parentrevs.rev_len, 0)) AS size
-                FROM $revisionTable AS revs
-                JOIN $pageTable ON revs.rev_page = page_id
-                $ipcJoin
-                LEFT JOIN $revisionTable AS parentrevs ON (revs.rev_parent_id = parentrevs.rev_id)
-                WHERE $whereClause
-                ORDER BY revs.rev_timestamp DESC
-                LIMIT 5000";
-        $data = $this->executeProjectsQuery($project, $sql, $params)->fetchFirstColumn();
-        $results = $data;
-        $results['average_size'] = count($data) > 0 ? array_sum($data)/count($data) : 0;
+        $sql = "SELECT JSON_ARRAYAGG(data.size) as sizes,
+                JSON_ARRAYAGG(data.tags) as tag_lists
+                FROM (
+                    SELECT CAST(revs.rev_len AS SIGNED) - IFNULL(parentrevs.rev_len, 0) AS size,
+                    (
+                        SELECT JSON_ARRAYAGG(ctd_name)
+                        FROM $ctTable
+                        JOIN $ctdTable
+                        ON ct_tag_id = ctd_id
+                        WHERE ct_rev_id = revs.rev_id
+                    ) as tags
+                    FROM $revisionTable AS revs
+                    JOIN $pageTable ON revs.rev_page = page_id
+                    $ipcJoin
+                    LEFT JOIN $revisionTable AS parentrevs ON (revs.rev_parent_id = parentrevs.rev_id)
+                    WHERE $whereClause
+                    ORDER BY revs.rev_timestamp DESC
+                    LIMIT 5000
+                ) data";
+        $results = $this->executeProjectsQuery($project, $sql, $params)->fetchAssociative();
+        $results['sizes'] = json_decode($results['sizes']);
+        $results['average_size'] = count($results['sizes']) > 0
+            ? array_sum($results['sizes'])/count($results['sizes'])
+            : 0;
         $isSmall = fn($n) => abs(intval($n)) < 20;
         $isLarge = fn($n) => abs(intval($n)) > 1000;
-        $results['small_edits'] = count(array_filter($data, $isSmall));
-        $results['large_edits'] = count(array_filter($data, $isLarge));
+        $results['small_edits'] = count(array_filter($results['sizes'], $isSmall));
+        $results['large_edits'] = count(array_filter($results['sizes'], $isLarge));
 
         // Cache and return.
         return $this->setCache($cacheKey, $results);

templates/editCounter/general_stats.html.twig

@@ -170,6 +170,15 @@
                 {% endif %}
             </td>
         </tr>
+        <tr>
+            <td>{{ msg('auto-edits-tagged') }}*</td>
+            <td>
+                {{ ec.countAutoEdits|num_format }}
+                {% if ec.countLast5000 %}
+                    &middot;
+                    ({{ ((ec.countAutoEdits / ec.countLast5000) * 100)|percent_format }} )
+                {% endif %}
+            </td>
         <tr>
             <td>{{ msg('small-edits') }}*</td>
             <td>
@@ -522,7 +531,7 @@
         <script type="text/javascript">
             $(function() {
                 xtools.editcounter.setupSizeHistogram(
-                    {{ ec.editSizeData|json_encode()|raw }},
+                    {{ ec.editSizeData.sizes|json_encode()|raw }},
                     ["{{ color(1) }}", "{{ color(0) }}", "{{ color(4) }}"],
                     ["{{ msg('additions') }}", "{{ msg('removals') }}", "{{ msg('size-zero') }}"],
                 );

templates/editCounter/general_stats.wikitext.twig

@@ -90,6 +90,10 @@
 | {% verbatim %}{{FORMATNUM:{% endverbatim %}{{ ec.countMinorRevisions }}}}{% if ec.countLiveRevisions %} ({{ ((ec.countMinorRevisions / ec.countLiveRevisions) * 100)|percent_format }})
 {% endif %}
 |-
+| {{ msg('auto-edits-tagged') }}*
+| {% verbatim %}{{FORMATNUM:{% endverbatim %}{{ ec.countAutoEdits }}}}{% if ec.countLast5000 %} ({{ ((ec.countAutoEdits / ec.countLast5000) * 100)|percent_format }})
+{% endif %}
+|-
 | {{ msg('small-edits') }}*
 | {% verbatim %}{{FORMATNUM:{% endverbatim %}{{ ec.countSmallEdits }}}}{% if ec.countLast5000 %} ({{ ((ec.countSmallEdits / ec.countLast5000) * 100)|percent_format }})
 {% endif %}