Add metrics: total number of entites

app/Metrics/App/WikiMetrics.php

@@ -8,6 +8,7 @@
 use Illuminate\Support\Arr;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
+use PDO;
 
 class WikiMetrics
 {
@@ -32,7 +33,7 @@ public function saveMetrics(Wiki $wiki): void
         $weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY);
         $monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY);
         $quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY);
-
+        $numberOfEntities = $this->getNumberOfEntities();
         $monthlyNumberOfUsersPerActivityType = $this->getNumberOfUsersPerActivityType();
 
         $dailyMetrics = new WikiDailyMetrics([
@@ -46,6 +47,10 @@ public function saveMetrics(Wiki $wiki): void
             'weekly_actions' => $weeklyActions,
             'monthly_actions' => $monthlyActions,
             'quarterly_actions' => $quarterlyActions,
+            'item_count' => $numberOfEntities['120'],
+            'property_count' => $numberOfEntities['122'],
+            'lexeme_count' => $numberOfEntities['146'],
+            'entity_schema_count' => $numberOfEntities['640'],
             'monthly_casual_users' => $monthlyNumberOfUsersPerActivityType[0],
             'monthly_active_users' => $monthlyNumberOfUsersPerActivityType[1],
         ]);
@@ -171,4 +176,27 @@ private function getNumberOfUsersPerActivityType() : array
             Arr::get($result, 'monthly_active_users',null)
         ];
     }
+
+    private function getNumberOfEntities() : array
+    {
+        $wikiDb = $this->wiki->wikiDb;
+        $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page';
+        $query = "SELECT
+              page_namespace AS namespace,
+              COUNT(*) AS count
+            FROM $tablePage
+            WHERE page_namespace in (120, 146, 122, 640)
+                        AND page_is_redirect=0 -- non-redirects only
+            GROUP BY page_namespace";
+
+        $manager = app()->db;
+        $manager->purge('mw');
+        $conn = $manager->connection('mw');
+        $pdo = $conn->getPdo();
+        $result = $pdo->query($query)->fetchAll(PDO::FETCH_ASSOC);
+        if (count($result) === 0){
+            return [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ];
+        }
+        return array_column($result, 'count', 'namespace');
+    }
 }

app/WikiDailyMetrics.php

@@ -28,6 +28,10 @@ class WikiDailyMetrics extends Model
         'monthly_actions',
         'quarterly_actions',
         'number_of_triples',
+        'item_count',
+        'property_count',
+        'lexeme_count',
+        'entity_schema_count',
         'monthly_casual_users',
         'monthly_active_users',
 
@@ -42,6 +46,10 @@ class WikiDailyMetrics extends Model
         'monthly_actions',
         'quarterly_actions',
         'number_of_triples',
+        'item_count',
+        'property_count',
+        'lexeme_count',
+        'entity_schema_count',
         'monthly_casual_users',
         'monthly_active_users',
     ];

database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('item_count')->nullable()->default(null);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->dropColumn('item_count');
+        });
+    }
+};

database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('property_count')->nullable()->default(null);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->dropColumn('property_count');
+        });
+    }
+};

database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('lexeme_count')->nullable()->default(null);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->dropColumn('lexeme_count');
+        });
+    }
+};

database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('entity_schema_count')->nullable()->default(null);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->dropColumn('entity_schema_count');
+        });
+    }
+};

tests/Jobs/UpdateWikiDailyMetricJobTest.php

@@ -61,6 +61,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis()
             'weekly_actions' => null,
             'monthly_actions' => null,
             'quarterly_actions' => null,
+            'item_count' => 0,
+            'property_count' => 0,
+            'lexeme_count' => 0,
+            'entity_schema_count' => 0,
         ]);
 
         $this->assertDatabaseHas('wiki_daily_metrics', [
@@ -70,6 +74,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis()
             'weekly_actions' => null,
             'monthly_actions' => null,
             'quarterly_actions' => null,
+            'item_count' => 0,
+            'property_count' => 0,
+            'lexeme_count' => 0,
+            'entity_schema_count' => 0,
         ]);
     }
 

tests/Metrics/WikiMetricsTest.php

@@ -9,9 +9,11 @@
 use App\WikiDailyMetrics;
 use App\Jobs\ProvisionWikiDbJob;
 use Carbon\Carbon;
+use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Schema;
 use Tests\TestCase;
 
 class WikiMetricsTest extends TestCase
@@ -170,5 +172,108 @@ public function testSaveNullForFailedRequestOfTriplesCount()
             'number_of_triples' => null
         ]);
     }
+
+    public function testSavesEntityCountsCorrectly()
+    {
+        $wiki = Wiki::factory()->create([
+            'domain' => 'entitycounttest.wikibase.cloud'
+        ]);
+
+        $wikiDb = WikiDb::first();
+        $wikiDb->update(['wiki_id' => $wiki->id]);
+
+        $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page';
+
+        Schema::dropIfExists($tablePage);
+        Schema::create($tablePage, function (Blueprint $table) {
+            $table->increments('page_id');
+            $table->integer('page_namespace');
+            $table->boolean('page_is_redirect')->default(0);
+            $table->string('page_title', 255);
+            $table->double('page_random');
+            $table->binary('page_touched');
+            $table->integer('page_latest');
+            $table->integer('page_len');
+        });
+
+        // Insert dummy data
+        DB::table($tablePage)->insert([
+            [
+                'page_namespace' => 120,
+                'page_is_redirect' => 0,
+                'page_title' => 'foo',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 1,
+                'page_len' => 2
+            ], // item
+            [
+                'page_namespace' => 120,
+                'page_is_redirect' => 0,
+                'page_title' => 'bar',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 0,
+                'page_len' => 2
+            ], // item
+            [
+                'page_namespace' => 122,
+                'page_is_redirect' => 0,
+                'page_title' => 'foo',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 1,
+                'page_len' => 2]
+            , // property
+            [
+                'page_namespace' => 640,
+                'page_is_redirect' => 0,
+                'page_title' => 'bar',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 1,
+                'page_len' => 2
+            ], // entity schema
+            [
+                'page_namespace' => 146,
+                'page_is_redirect' => 0,
+                'page_title' => 'foo',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 1,
+                'page_len' => 2
+            ], // lexeme
+            [
+                'page_namespace' => 640,
+                'page_is_redirect' => 1,
+                'page_title' => 'foo',
+                'page_random' => 0,
+                'page_touched' => random_bytes(10),
+                'page_latest' => 1,
+                'page_len' => 2
+            ], // entity schema
+        ]);
+        WikiDailyMetrics::create([
+            'id' => $wiki->id . '_' . now()->subDay()->toDateString(),
+            'wiki_id' => $wiki->id,
+            'date' => now()->subDay()->toDateString(),
+            'pages' => 6,
+            'is_deleted' => 0
+        ]);
+
+        (new WikiMetrics())->saveMetrics($wiki);
+
+        //clean up after the test
+        $wiki->forceDelete();
+        Schema::dropIfExists($tablePage);
+
+        $this->assertDatabaseHas('wiki_daily_metrics', [
+            'wiki_id' => $wiki->id,
+            'item_count' => 2,
+            'property_count' => 1,
+            'lexeme_count' => 1,
+            'entity_schema_count' => 1 // the redirect should be ignored
+        ]);
+    }
 }