|
1 | | -#!/usr/bin/env node |
2 | | - |
3 | 1 | import 'dotenv-flow/config' |
| 2 | +import fs from 'fs-extra' |
| 3 | +import path from 'path' |
| 4 | +import Knex from 'knex' |
| 5 | +import moment from 'moment' |
| 6 | +import { fileURLToPath } from 'url' |
| 7 | +import { DeleteObjectCommand, S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3' |
4 | 8 |
|
5 | | -/** |
6 | | - * Clean World Script |
7 | | - * Main entry point for world cleaning operations. |
8 | | - * |
9 | | - * Automatically selects the appropriate cleanup script based on STORAGE_TYPE: |
10 | | - * - STORAGE_TYPE=s3: Uses clean-world-s3.mjs |
11 | | - * - STORAGE_TYPE=local or unset: Uses clean-world-local.mjs |
12 | | - * |
13 | | - * Usage: |
14 | | - * npm run world:clean |
15 | | - */ |
16 | | - |
| 9 | +const DRY_RUN = false |
17 | 10 | const storageType = process.env.STORAGE_TYPE || 'local' |
| 11 | +const world = process.env.WORLD || 'world' |
| 12 | + |
| 13 | +const __dirname = path.dirname(fileURLToPath(import.meta.url)) |
| 14 | +const rootDir = path.join(__dirname, '../') |
| 15 | +const worldDir = path.join(rootDir, world) |
| 16 | +const assetsDir = path.join(worldDir, '/assets') |
| 17 | + |
| 18 | +// Database configuration |
| 19 | +const { DB_TYPE = '', DB_URL = '' } = process.env |
| 20 | +let dbConfig |
| 21 | + |
| 22 | +if (!DB_TYPE && !DB_URL) { |
| 23 | + // Default: sqlite in world folder |
| 24 | + dbConfig = { |
| 25 | + client: 'better-sqlite3', |
| 26 | + connection: { filename: `./${world}/db.sqlite` }, |
| 27 | + useNullAsDefault: true, |
| 28 | + } |
| 29 | +} else if (DB_TYPE === 'pg' && DB_URL) { |
| 30 | + dbConfig = { |
| 31 | + client: 'pg', |
| 32 | + connection: DB_URL, |
| 33 | + pool: { min: 2, max: 10 }, |
| 34 | + } |
| 35 | +} else { |
| 36 | + throw new Error('Unsupported or incomplete DB configuration. Only sqlite (default) and postgres (pg) via DB_TYPE/DB_URL are supported.') |
| 37 | +} |
| 38 | + |
| 39 | +const db = Knex(dbConfig) |
| 40 | + |
| 41 | +// S3 configuration if needed |
| 42 | +let s3Client, bucketName, assetsPrefix |
| 43 | +if (storageType === 's3') { |
| 44 | + if (!process.env.S3_BUCKET_NAME) { |
| 45 | + console.error('Error: S3_BUCKET_NAME is required when STORAGE_TYPE=s3') |
| 46 | + process.exit(1) |
| 47 | + } |
| 48 | + |
| 49 | + s3Client = new S3Client({ |
| 50 | + region: process.env.S3_REGION || 'us-east-1', |
| 51 | + credentials: { |
| 52 | + accessKeyId: process.env.S3_ACCESS_KEY_ID, |
| 53 | + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, |
| 54 | + }, |
| 55 | + }) |
| 56 | + bucketName = process.env.S3_BUCKET_NAME |
| 57 | + assetsPrefix = process.env.S3_ASSETS_PREFIX || 'assets/' |
| 58 | +} |
| 59 | + |
| 60 | +console.log(`Using ${storageType.toUpperCase()} storage`) |
| 61 | + |
| 62 | +// TODO: run any missing migrations first? |
| 63 | + |
| 64 | +let blueprints = new Set() |
| 65 | +const blueprintRows = await db('blueprints') |
| 66 | +for (const row of blueprintRows) { |
| 67 | + const blueprint = JSON.parse(row.data) |
| 68 | + blueprints.add(blueprint) |
| 69 | +} |
| 70 | + |
| 71 | +const entities = [] |
| 72 | +const entityRows = await db('entities') |
| 73 | +for (const row of entityRows) { |
| 74 | + const entity = JSON.parse(row.data) |
| 75 | + entities.push(entity) |
| 76 | +} |
| 77 | + |
| 78 | +const vrms = new Set() |
| 79 | +const userRows = await db('users').select('avatar') |
| 80 | +for (const user of userRows) { |
| 81 | + if (!user.avatar) continue |
| 82 | + const avatar = user.avatar.replace('asset://', '') |
| 83 | + vrms.add(avatar) |
| 84 | +} |
| 85 | + |
| 86 | +// Get assets from storage (local or S3) |
| 87 | +const fileAssets = new Set() |
18 | 88 |
|
19 | 89 | if (storageType === 's3') { |
20 | | - console.log('Running S3 cleanup...') |
21 | | - await import('./clean-world/clean-world-s3.mjs') |
| 90 | + // List S3 assets |
| 91 | + console.log('Fetching S3 assets...') |
| 92 | + let continuationToken = undefined |
| 93 | + do { |
| 94 | + const command = new ListObjectsV2Command({ |
| 95 | + Bucket: bucketName, |
| 96 | + Prefix: assetsPrefix, |
| 97 | + ContinuationToken: continuationToken, |
| 98 | + }) |
| 99 | + |
| 100 | + const response = await s3Client.send(command) |
| 101 | + |
| 102 | + if (response.Contents) { |
| 103 | + for (const object of response.Contents) { |
| 104 | + const key = object.Key |
| 105 | + const filename = key.replace(assetsPrefix, '') |
| 106 | + // Check if it's a hashed asset (64 character hash) |
| 107 | + const isAsset = filename.split('.')[0].length === 64 |
| 108 | + if (isAsset) { |
| 109 | + fileAssets.add(filename) |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + |
| 114 | + continuationToken = response.NextContinuationToken |
| 115 | + } while (continuationToken) |
| 116 | + |
| 117 | + console.log(`Found ${fileAssets.size} S3 assets`) |
22 | 118 | } else { |
23 | | - console.log('Running local cleanup...') |
24 | | - await import('./clean-world/clean-world-local.mjs') |
25 | | -} |
| 119 | + // List local assets |
| 120 | + const files = fs.readdirSync(assetsDir) |
| 121 | + for (const file of files) { |
| 122 | + const filePath = path.join(assetsDir, file) |
| 123 | + const isDirectory = fs.statSync(filePath).isDirectory() |
| 124 | + if (isDirectory) continue |
| 125 | + const relPath = path.relative(assetsDir, filePath) |
| 126 | + // HACK: we only want to include uploaded assets (not core/assets/*) so we do a check |
| 127 | + // if its filename is a 64 character hash |
| 128 | + const isAsset = relPath.split('.')[0].length === 64 |
| 129 | + if (!isAsset) continue |
| 130 | + fileAssets.add(relPath) |
| 131 | + } |
| 132 | +} |
| 133 | + |
| 134 | +let worldImage |
| 135 | +let worldModel |
| 136 | +let worldAvatar |
| 137 | +let settings = await db('config').where('key', 'settings').first() |
| 138 | +if (settings) { |
| 139 | + settings = JSON.parse(settings.value) |
| 140 | + if (settings.image) worldImage = settings.image.url.replace('asset://', '') |
| 141 | + if (settings.model) worldModel = settings.model.url.replace('asset://', '') |
| 142 | + if (settings.avatar) worldAvatar = settings.avatar.url.replace('asset://', '') |
| 143 | +} |
| 144 | + |
| 145 | +/** |
| 146 | + * Phase 1: |
| 147 | + * Remove all blueprints that no entities reference any more. |
| 148 | + * The world doesn't need them, and we shouldn't be loading them in and sending dead blueprints to all the clients. |
| 149 | + */ |
| 150 | + |
| 151 | +const blueprintsToDelete = [] |
| 152 | +for (const blueprint of blueprints) { |
| 153 | + const canDelete = !entities.find(e => e.blueprint === blueprint.id) |
| 154 | + if (canDelete) { |
| 155 | + blueprintsToDelete.push(blueprint) |
| 156 | + } |
| 157 | +} |
| 158 | +console.log(`deleting ${blueprintsToDelete.length} blueprints`) |
| 159 | +for (const blueprint of blueprintsToDelete) { |
| 160 | + blueprints.delete(blueprint) |
| 161 | + if (!DRY_RUN) { |
| 162 | + await db('blueprints').where('id', blueprint.id).delete() |
| 163 | + } |
| 164 | + console.log('delete blueprint:', blueprint.id) |
| 165 | +} |
| 166 | + |
| 167 | +/** |
| 168 | + * Phase 2: |
| 169 | + * Remove all asset files that are not: |
| 170 | + * - referenced by a blueprint |
| 171 | + * - used as a player avatar |
| 172 | + * - used as the world image |
| 173 | + * - used as the world avatar |
| 174 | + * - used as the world model |
| 175 | + * The world no longer uses/needs them. |
| 176 | + * |
| 177 | + */ |
| 178 | + |
| 179 | +const blueprintAssets = new Set() |
| 180 | +for (const blueprint of blueprints) { |
| 181 | + if (blueprint.model && blueprint.model.startsWith('asset://')) { |
| 182 | + const asset = blueprint.model.replace('asset://', '') |
| 183 | + blueprintAssets.add(asset) |
| 184 | + } |
| 185 | + if (blueprint.script && blueprint.script.startsWith('asset://')) { |
| 186 | + const asset = blueprint.script.replace('asset://', '') |
| 187 | + blueprintAssets.add(asset) |
| 188 | + } |
| 189 | + if (blueprint.image?.url && blueprint.image.url.startsWith('asset://')) { |
| 190 | + const asset = blueprint.image.url.replace('asset://', '') |
| 191 | + blueprintAssets.add(asset) |
| 192 | + } |
| 193 | + for (const key in blueprint.props) { |
| 194 | + const url = blueprint.props[key]?.url |
| 195 | + if (!url) continue |
| 196 | + const asset = url.replace('asset://', '') |
| 197 | + blueprintAssets.add(asset) |
| 198 | + } |
| 199 | +} |
| 200 | + |
| 201 | +const filesToDelete = [] |
| 202 | +for (const fileAsset of fileAssets) { |
| 203 | + const isUsedByBlueprint = blueprintAssets.has(fileAsset) |
| 204 | + const isUsedByUser = vrms.has(fileAsset) |
| 205 | + const isWorldImage = fileAsset === worldImage |
| 206 | + const isWorldModel = fileAsset === worldModel |
| 207 | + const isWorldAvatar = fileAsset === worldAvatar |
| 208 | + if (!isUsedByBlueprint && !isUsedByUser && !isWorldModel && !isWorldAvatar && !isWorldImage) { |
| 209 | + filesToDelete.push(fileAsset) |
| 210 | + } |
| 211 | +} |
| 212 | + |
| 213 | +console.log(`deleting ${filesToDelete.length} assets`) |
| 214 | +for (const fileAsset of filesToDelete) { |
| 215 | + if (storageType === 's3') { |
| 216 | + // Delete from S3 |
| 217 | + const s3Key = `${assetsPrefix}${fileAsset}` |
| 218 | + if (!DRY_RUN) { |
| 219 | + const deleteCommand = new DeleteObjectCommand({ |
| 220 | + Bucket: bucketName, |
| 221 | + Key: s3Key, |
| 222 | + }) |
| 223 | + await s3Client.send(deleteCommand) |
| 224 | + } |
| 225 | + console.log('delete asset:', fileAsset) |
| 226 | + } else { |
| 227 | + // Delete from local filesystem |
| 228 | + const fullPath = path.join(assetsDir, fileAsset) |
| 229 | + if (!DRY_RUN) { |
| 230 | + fs.removeSync(fullPath) |
| 231 | + } |
| 232 | + console.log('delete asset:', fileAsset) |
| 233 | + } |
| 234 | +} |
| 235 | + |
| 236 | +console.log(`${storageType.toUpperCase()} cleanup completed`) |
| 237 | + |
| 238 | +// Close database connection before exiting |
| 239 | +await db.destroy() |
| 240 | +process.exit() |
0 commit comments