From 551170e831f92c66402c1b7deaf6c27ecd7202f0 Mon Sep 17 00:00:00 2001 From: Hunter Wilhelm <71348224+hunterwilhelm@users.noreply.github.com> Date: Fri, 23 May 2025 11:53:34 -0700 Subject: [PATCH] Adds in app search --- .../org/stypox/dicio/eval/SkillHandler.kt | 2 + .../dicio/skills/app_search/AppSearchInfo.kt | 32 +++++ .../skills/app_search/AppSearchOutput.kt | 94 ++++++++++++ .../dicio/skills/app_search/AppSearchSkill.kt | 136 ++++++++++++++++++ app/src/main/res/values-es/strings.xml | 6 + app/src/main/res/values/strings.xml | 6 + app/src/main/sentences/en/app_search.yml | 6 + app/src/main/sentences/es/app_search.yml | 7 + app/src/main/sentences/skill_definitions.yml | 10 ++ 9 files changed, 299 insertions(+) create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchInfo.kt create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchOutput.kt create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchSkill.kt create mode 100644 app/src/main/sentences/en/app_search.yml create mode 100644 app/src/main/sentences/es/app_search.yml diff --git a/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt b/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt index 299994c1..9e63dfb7 100644 --- a/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt +++ b/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt @@ -29,6 +29,7 @@ import org.stypox.dicio.skills.search.SearchInfo import org.stypox.dicio.skills.telephone.TelephoneInfo import org.stypox.dicio.skills.timer.TimerInfo import org.stypox.dicio.skills.weather.WeatherInfo +import org.stypox.dicio.skills.app_search.AppSearchInfo import javax.inject.Inject import javax.inject.Singleton @@ -42,6 +43,7 @@ class SkillHandler @Inject constructor( val allSkillInfoList = listOf( WeatherInfo, SearchInfo, + AppSearchInfo, LyricsInfo, OpenInfo, CalculatorInfo, diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchInfo.kt b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchInfo.kt new file mode 100644 index 00000000..9adc9861 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchInfo.kt @@ -0,0 +1,32 @@ +package org.stypox.dicio.skills.app_search + +import android.content.Context +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Search +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.vector.rememberVectorPainter +import org.dicio.skill.context.SkillContext +import org.dicio.skill.skill.Skill +import org.dicio.skill.skill.SkillInfo +import org.stypox.dicio.R +import org.stypox.dicio.sentences.Sentences + +object AppSearchInfo : SkillInfo("app_search") { + override fun name(context: Context) = + context.getString(R.string.skill_name_app_search) + + override fun sentenceExample(context: Context) = + context.getString(R.string.skill_sentence_example_app_search) + + @Composable + override fun icon() = + rememberVectorPainter(Icons.Filled.Search) + + override fun isAvailable(ctx: SkillContext): Boolean { + return Sentences.AppSearch[ctx.sentencesLanguage] != null + } + + override fun build(ctx: SkillContext): Skill<*> { + return AppSearchSkill(AppSearchInfo, Sentences.AppSearch[ctx.sentencesLanguage]!!) + } +} diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchOutput.kt b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchOutput.kt new file mode 100644 index 00000000..a308cab8 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchOutput.kt @@ -0,0 +1,94 @@ +package org.stypox.dicio.skills.app_search + +import android.content.pm.PackageManager +import android.util.Log +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.width +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import com.google.accompanist.drawablepainter.rememberDrawablePainter +import org.dicio.skill.context.SkillContext +import org.dicio.skill.skill.SkillOutput +import org.stypox.dicio.R +import org.stypox.dicio.io.graphical.Headline +import org.stypox.dicio.util.getString + +private val TAG = AppSearchOutput::class.simpleName + +class AppSearchOutput( + private val appName: String?, + private val packageName: String?, + private val searchQuery: String?, + private val success: Boolean +) : SkillOutput { + override fun getSpeechOutput(ctx: SkillContext): String = when { + appName == null -> ctx.getString(R.string.skill_app_search_could_not_understand) + packageName == null -> ctx.getString(R.string.skill_app_search_unknown_app, appName) + !success -> ctx.getString(R.string.skill_app_search_failed, appName, searchQuery ?: "") + else -> ctx.getString(R.string.skill_app_search_searching, appName, searchQuery ?: "") + } + + @Composable + override fun GraphicalOutput(ctx: SkillContext) { + if (appName == null || packageName == null || !success) { + Headline(text = getSpeechOutput(ctx)) + } else { + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + ) { + val context = LocalContext.current + val icon = remember { + try { + context.packageManager.getApplicationIcon(packageName) + } catch (e: PackageManager.NameNotFoundException) { + Log.e(TAG, "Could not load icon for $packageName", e) + null + } + } + + if (icon != null) { + Image( + painter = rememberDrawablePainter(icon), + contentDescription = appName, + modifier = Modifier + .fillMaxWidth(0.2f) + .aspectRatio(1.0f), + ) + + Spacer(modifier = Modifier.width(8.dp)) + } + + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + text = getSpeechOutput(ctx), + style = MaterialTheme.typography.headlineMedium, + textAlign = TextAlign.Center, + modifier = Modifier.fillMaxWidth(), + ) + + Text( + text = packageName, + textAlign = TextAlign.Center, + modifier = Modifier.fillMaxWidth(), + ) + } + } + } + } +} diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchSkill.kt b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchSkill.kt new file mode 100644 index 00000000..787944f0 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/app_search/AppSearchSkill.kt @@ -0,0 +1,136 @@ +package org.stypox.dicio.skills.app_search + +import android.annotation.SuppressLint +import android.content.Intent +import android.content.pm.ApplicationInfo +import android.content.pm.PackageManager +import android.content.pm.ResolveInfo +import android.net.Uri +import org.dicio.skill.context.SkillContext +import org.dicio.skill.skill.SkillInfo +import org.dicio.skill.skill.SkillOutput +import org.dicio.skill.standard.StandardRecognizerData +import org.dicio.skill.standard.StandardRecognizerSkill +import org.stypox.dicio.sentences.Sentences.AppSearch +import org.stypox.dicio.util.StringUtils + +class AppSearchSkill(correspondingSkillInfo: SkillInfo, data: StandardRecognizerData) + : StandardRecognizerSkill(correspondingSkillInfo, data) { + + override suspend fun generateOutput(ctx: SkillContext, inputData: AppSearch): SkillOutput { + val userAppName = when (inputData) { + is AppSearch.Query -> inputData.app?.trim { it <= ' ' } + else -> null + } + val userQuery = when (inputData) { + is AppSearch.Query -> inputData.query?.trim { it <= ' ' } + else -> null + } + + val packageManager: PackageManager = ctx.android.packageManager + val applicationInfo = userAppName?.let { getMostSimilarApp(packageManager, it) } + var success = false + + if (applicationInfo != null && userQuery != null) { + success = launchAppWithSearch(ctx, packageManager, applicationInfo, userQuery) + } + + return AppSearchOutput( + appName = applicationInfo?.loadLabel(packageManager)?.toString() ?: userAppName, + packageName = applicationInfo?.packageName, + searchQuery = userQuery, + success = success + ) + } + + private fun launchAppWithSearch(ctx: SkillContext, packageManager: PackageManager, applicationInfo: ApplicationInfo, query: String): Boolean { + val packageName = applicationInfo.packageName + try { + // Handle different apps with their specific search intents + when { + // YouTube + packageName.contains("youtube", ignoreCase = true) -> { + val intent = Intent(Intent.ACTION_SEARCH) + intent.setPackage(packageName) + intent.putExtra("query", query) + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + ctx.android.startActivity(intent) + return true + } + // Spotify + packageName.contains("spotify", ignoreCase = true) -> { + val intent = Intent(Intent.ACTION_VIEW) + intent.setData(Uri.parse("spotify:search:$query")) + intent.putExtra(Intent.EXTRA_REFERRER, Uri.parse("android-app://${ctx.android.packageName}")) + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + ctx.android.startActivity(intent) + return true + } + // Generic search intent as fallback + else -> { + val intent = Intent(Intent.ACTION_SEARCH) + intent.setPackage(packageName) + intent.putExtra("query", query) + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + ctx.android.startActivity(intent) + return true + } + } + } catch (e: Exception) { + // If the specific intent fails, try a web search as fallback + try { + val intent = Intent(Intent.ACTION_VIEW) + when { + packageName.contains("youtube", ignoreCase = true) -> { + intent.data = Uri.parse("https://www.youtube.com/results?search_query=${Uri.encode(query)}") + } + packageName.contains("spotify", ignoreCase = true) -> { + intent.data = Uri.parse("https://open.spotify.com/search/${Uri.encode(query)}") + } + else -> { + return false + } + } + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + ctx.android.startActivity(intent) + return true + } catch (e: Exception) { + return false + } + } + } + + companion object { + private fun getMostSimilarApp( + packageManager: PackageManager, + appName: String + ): ApplicationInfo? { + val resolveInfosIntent = Intent(Intent.ACTION_MAIN, null) + resolveInfosIntent.addCategory(Intent.CATEGORY_LAUNCHER) + + @SuppressLint("QueryPermissionsNeeded") // we need to query all apps + val resolveInfos: List = + packageManager.queryIntentActivities(resolveInfosIntent, 0) + var bestDistance = Int.MAX_VALUE + var bestApplicationInfo: ApplicationInfo? = null + + for (resolveInfo in resolveInfos) { + try { + val currentApplicationInfo: ApplicationInfo = packageManager.getApplicationInfo( + resolveInfo.activityInfo.packageName, PackageManager.GET_META_DATA + ) + val currentDistance = StringUtils.customStringDistance( + appName, + packageManager.getApplicationLabel(currentApplicationInfo).toString() + ) + if (currentDistance < bestDistance) { + bestDistance = currentDistance + bestApplicationInfo = currentApplicationInfo + } + } catch (ignored: PackageManager.NameNotFoundException) { + } + } + return if (bestDistance > 5) null else bestApplicationInfo + } + } +} diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index ee1bfcc0..1d748d69 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -49,6 +49,8 @@ ¿Cuál es el clima en Roma\? Letras de canciones ¿Cuál es la canción que dice we will we will rock you\? + Búsqueda en Aplicaciones + En YouTube buscar gatos lindos Abrir aplicaciones Abrir newpipe ¿Cuánto es cinco por cuatro menos un millón\? @@ -61,6 +63,10 @@ Se ha encontrado la canción %1$s de %2$s Aplicación desconocida %1$s Abriendo %1$s… + No pude entender la aplicación + Aplicación desconocida %1$s + Buscando %2$s en %1$s + No se pudo buscar %2$s en %1$s No pude calcular tu solicitud Error de la red Cerrar la caja de navegación diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 08873089..2c141967 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -149,6 +149,8 @@ Timer Set a timer for five minutes Current time + App Search + In YouTube search for cute cats What time is it? Text message Here is what I found @@ -164,6 +166,10 @@ Could not understand app Unknown app %1$s Opening %1$s… + Could not understand app + Unknown app %1$s + Searching for %2$s on %1$s + Failed to search for %2$s on %1$s Playing media… Pausing media… Previous media… diff --git a/app/src/main/sentences/en/app_search.yml b/app/src/main/sentences/en/app_search.yml new file mode 100644 index 00000000..f159e4f1 --- /dev/null +++ b/app/src/main/sentences/en/app_search.yml @@ -0,0 +1,6 @@ +query: + - in .app. look up .query. + - in .app. find .query. + - in .app. search .query. + - in .app. search for .query. + diff --git a/app/src/main/sentences/es/app_search.yml b/app/src/main/sentences/es/app_search.yml new file mode 100644 index 00000000..423044de --- /dev/null +++ b/app/src/main/sentences/es/app_search.yml @@ -0,0 +1,7 @@ +query: + - en .app. buscar .query. + - en .app. busca .query. + - en .app. busque .query. + - en .app. encontrar .query. + - en .app. encuentra .query. + - en .app. encuentre .query. diff --git a/app/src/main/sentences/skill_definitions.yml b/app/src/main/sentences/skill_definitions.yml index 4785b7d5..340bfda6 100644 --- a/app/src/main/sentences/skill_definitions.yml +++ b/app/src/main/sentences/skill_definitions.yml @@ -1,4 +1,14 @@ skills: + - id: app_search + specificity: high + sentences: + - id: query + captures: + - id: app + type: string + - id: query + type: string + - id: current_time specificity: high sentences: