diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php index 5609782a4..e9f0e80af 100644 --- a/app/Metrics/App/WikiMetrics.php +++ b/app/Metrics/App/WikiMetrics.php @@ -8,6 +8,7 @@ use Illuminate\Support\Arr; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; +use PDO; class WikiMetrics { @@ -32,7 +33,7 @@ public function saveMetrics(Wiki $wiki): void $weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY); $monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY); $quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY); - + $numberOfEntities = $this->getNumberOfEntities(); $monthlyNumberOfUsersPerActivityType = $this->getNumberOfUsersPerActivityType(); $dailyMetrics = new WikiDailyMetrics([ @@ -46,6 +47,10 @@ public function saveMetrics(Wiki $wiki): void 'weekly_actions' => $weeklyActions, 'monthly_actions' => $monthlyActions, 'quarterly_actions' => $quarterlyActions, + 'item_count' => $numberOfEntities['120'], + 'property_count' => $numberOfEntities['122'], + 'lexeme_count' => $numberOfEntities['146'], + 'entity_schema_count' => $numberOfEntities['640'], 'monthly_casual_users' => $monthlyNumberOfUsersPerActivityType[0], 'monthly_active_users' => $monthlyNumberOfUsersPerActivityType[1], ]); @@ -171,4 +176,27 @@ private function getNumberOfUsersPerActivityType() : array Arr::get($result, 'monthly_active_users',null) ]; } + + private function getNumberOfEntities() : array + { + $wikiDb = $this->wiki->wikiDb; + $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; + $query = "SELECT + page_namespace AS namespace, + COUNT(*) AS count + FROM $tablePage + WHERE page_namespace in (120, 146, 122, 640) + AND page_is_redirect=0 -- non-redirects only + GROUP BY page_namespace"; + + $manager = app()->db; + $manager->purge('mw'); + $conn = $manager->connection('mw'); + $pdo = $conn->getPdo(); + $result = $pdo->query($query)->fetchAll(PDO::FETCH_ASSOC); + if (count($result) === 0){ + return [ 120 => 0, 122 => 0, 146 => 0, 640 => 0 ]; + } + return array_column($result, 'count', 'namespace'); + } } diff --git a/app/WikiDailyMetrics.php b/app/WikiDailyMetrics.php index d0114842d..60ad57718 100644 --- a/app/WikiDailyMetrics.php +++ b/app/WikiDailyMetrics.php @@ -28,6 +28,10 @@ class WikiDailyMetrics extends Model 'monthly_actions', 'quarterly_actions', 'number_of_triples', + 'item_count', + 'property_count', + 'lexeme_count', + 'entity_schema_count', 'monthly_casual_users', 'monthly_active_users', @@ -42,6 +46,10 @@ class WikiDailyMetrics extends Model 'monthly_actions', 'quarterly_actions', 'number_of_triples', + 'item_count', + 'property_count', + 'lexeme_count', + 'entity_schema_count', 'monthly_casual_users', 'monthly_active_users', ]; diff --git a/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..2beb8c96b --- /dev/null +++ b/database/migrations/2025_07_30_104606_add_item_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('item_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('item_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..3d2fa9848 --- /dev/null +++ b/database/migrations/2025_07_30_104622_add_properties_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('property_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('property_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..f617454e7 --- /dev/null +++ b/database/migrations/2025_07_30_104657_add_lexemes_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('lexeme_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('lexeme_count'); + }); + } +}; diff --git a/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php b/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php new file mode 100644 index 000000000..7a06e2bb2 --- /dev/null +++ b/database/migrations/2025_07_30_104719_add_entity_schemas_count_to_wiki_daily_merics_table.php @@ -0,0 +1,28 @@ +integer('entity_schema_count')->nullable()->default(null); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metrics', function (Blueprint $table) { + $table->dropColumn('entity_schema_count'); + }); + } +}; diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index 90123c2c7..f59cd0a3e 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -61,6 +61,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() 'weekly_actions' => null, 'monthly_actions' => null, 'quarterly_actions' => null, + 'item_count' => 0, + 'property_count' => 0, + 'lexeme_count' => 0, + 'entity_schema_count' => 0, ]); $this->assertDatabaseHas('wiki_daily_metrics', [ @@ -70,6 +74,10 @@ public function testRunJobForAllWikisIncludingDeletedWikis() 'weekly_actions' => null, 'monthly_actions' => null, 'quarterly_actions' => null, + 'item_count' => 0, + 'property_count' => 0, + 'lexeme_count' => 0, + 'entity_schema_count' => 0, ]); } diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index 19140aeb5..868498cf8 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -9,9 +9,11 @@ use App\WikiDailyMetrics; use App\Jobs\ProvisionWikiDbJob; use Carbon\Carbon; +use Illuminate\Database\Schema\Blueprint; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Schema; use Tests\TestCase; class WikiMetricsTest extends TestCase @@ -170,5 +172,108 @@ public function testSaveNullForFailedRequestOfTriplesCount() 'number_of_triples' => null ]); } + + public function testSavesEntityCountsCorrectly() + { + $wiki = Wiki::factory()->create([ + 'domain' => 'entitycounttest.wikibase.cloud' + ]); + + $wikiDb = WikiDb::first(); + $wikiDb->update(['wiki_id' => $wiki->id]); + + $tablePage = $wikiDb->name . '.' . $wikiDb->prefix . '_page'; + + Schema::dropIfExists($tablePage); + Schema::create($tablePage, function (Blueprint $table) { + $table->increments('page_id'); + $table->integer('page_namespace'); + $table->boolean('page_is_redirect')->default(0); + $table->string('page_title', 255); + $table->double('page_random'); + $table->binary('page_touched'); + $table->integer('page_latest'); + $table->integer('page_len'); + }); + + // Insert dummy data + DB::table($tablePage)->insert([ + [ + 'page_namespace' => 120, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // item + [ + 'page_namespace' => 120, + 'page_is_redirect' => 0, + 'page_title' => 'bar', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 0, + 'page_len' => 2 + ], // item + [ + 'page_namespace' => 122, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2] + , // property + [ + 'page_namespace' => 640, + 'page_is_redirect' => 0, + 'page_title' => 'bar', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // entity schema + [ + 'page_namespace' => 146, + 'page_is_redirect' => 0, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // lexeme + [ + 'page_namespace' => 640, + 'page_is_redirect' => 1, + 'page_title' => 'foo', + 'page_random' => 0, + 'page_touched' => random_bytes(10), + 'page_latest' => 1, + 'page_len' => 2 + ], // entity schema + ]); + WikiDailyMetrics::create([ + 'id' => $wiki->id . '_' . now()->subDay()->toDateString(), + 'wiki_id' => $wiki->id, + 'date' => now()->subDay()->toDateString(), + 'pages' => 6, + 'is_deleted' => 0 + ]); + + (new WikiMetrics())->saveMetrics($wiki); + + //clean up after the test + $wiki->forceDelete(); + Schema::dropIfExists($tablePage); + + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'wiki_id' => $wiki->id, + 'item_count' => 2, + 'property_count' => 1, + 'lexeme_count' => 1, + 'entity_schema_count' => 1 // the redirect should be ignored + ]); + } }