Skip to content

feat(firebaseai): add think feature #17409

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(firebaseai): add think feature #17409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 19 additions & 17 deletions packages/firebase_ai/firebase_ai/example/lib/pages/function_calling_page.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,24 +170,26 @@ class _FunctionCallingPageState extends State<FunctionCallingPage> {
final functionCalls = response.functionCalls.toList();
// When the model response with a function call, invoke the function.
if (functionCalls.isNotEmpty) {
final functionCall = functionCalls.first;
if (functionCall.name == 'fetchWeather') {
Map<String, dynamic> location =
functionCall.args['location']! as Map<String, dynamic>;
var date = functionCall.args['date']! as String;
var city = location['city'] as String;
var state = location['state'] as String;
final functionResult = await fetchWeather(Location(city, state), date);
// Send the response to the model so that it can use the result to
// generate text for the user.
response = await functionCallChat.sendMessage(
Content.functionResponse(functionCall.name, functionResult),
);
} else {
throw UnimplementedError(
'Function not declared to the model: ${functionCall.name}',
);
for (final functionCall in functionCalls) {
if (functionCall.name == 'fetchWeather') {
Map<String, dynamic> location =
functionCall.args['location']! as Map<String, dynamic>;
var date = functionCall.args['date']! as String;
var city = location['city'] as String;
var state = location['state'] as String;
final functionResult =
await fetchWeather(Location(city, state), date);
// Send the response to the model so that it can use the result to
// generate text for the user.
response = await functionCallChat.sendMessage(
Content.functionResponse(functionCall.name, functionResult),
);
}
}
} else {
throw UnimplementedError(
'Function not declared to the model: fetchWeather',
);
}
// When the model responds with non-null text content, print it.
if (response.text case final text?) {
Expand Down
33 changes: 32 additions & 1 deletion packages/firebase_ai/firebase_ai/lib/src/api.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -697,10 +697,32 @@ enum ResponseModalities {
const ResponseModalities(this._jsonString);
final String _jsonString;

/// Convert to json format
// ignore: public_member_api_docs
String toJson() => _jsonString;
}

/// Config for thinking features.
class ThinkingConfig {
// ignore: public_member_api_docs
ThinkingConfig({this.includeThoughts, this.thinkingBudget});

/// Whether to include thoughts in the response.
///
/// If true, thoughts are returned only when available.
bool? includeThoughts;

/// The number of thoughts tokens that the model should generate.
int? thinkingBudget;

// ignore: public_member_api_docs
Map<String, Object?> toJson() => {
if (includeThoughts case final includeThoughts?)
'includeThoughts': includeThoughts,
if (thinkingBudget case final thinkingBudget?)
'thinkingBudget': thinkingBudget,
};
}

/// Configuration options for model generation and outputs.
abstract class BaseGenerationConfig {
// ignore: public_member_api_docs
Expand All @@ -713,6 +735,7 @@ abstract class BaseGenerationConfig {
this.presencePenalty,
this.frequencyPenalty,
this.responseModalities,
this.thinkingConfig,
});

/// Number of generated responses to return.
Expand Down Expand Up @@ -792,6 +815,12 @@ abstract class BaseGenerationConfig {
/// The list of desired response modalities.
final List<ResponseModalities>? responseModalities;

/// Config for thinking features.
///
/// An error will be returned if this field is set for models that don't
/// support thinking.
final ThinkingConfig? thinkingConfig;

// ignore: public_member_api_docs
Map<String, Object?> toJson() => {
if (candidateCount case final candidateCount?)
Expand All @@ -808,6 +837,8 @@ abstract class BaseGenerationConfig {
if (responseModalities case final responseModalities?)
'responseModalities':
responseModalities.map((modality) => modality.toJson()).toList(),
if (thinkingConfig case final thinkingConfig?)
'thinkingConfig': thinkingConfig.toJson(),
};
}

Expand Down
82 changes: 66 additions & 16 deletions packages/firebase_ai/firebase_ai/lib/src/content.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,15 @@ Content parseContent(Object jsonObject) {

/// Parse the [Part] from json object.
Part parsePart(Object? jsonObject) {
if (jsonObject is Map && jsonObject.containsKey('functionCall')) {
if (jsonObject is! Map<String, Object?>) {
throw unhandledFormat('Part', jsonObject);
}
// Extract common thought-related fields from the top-level JSON object.
final bool? thought = jsonObject['thought'] as bool?;
final Uint8List? thoughtSignature = jsonObject.containsKey('thoughtSignature')
? base64Decode(jsonObject['thoughtSignature']! as String)
: null;
if (jsonObject.containsKey('functionCall')) {
final functionCall = jsonObject['functionCall'];
if (functionCall is Map &&
functionCall.containsKey('name') &&
Expand All @@ -90,51 +98,86 @@ Part parsePart(Object? jsonObject) {
functionCall['name'] as String,
functionCall['args'] as Map<String, Object?>,
id: functionCall['id'] as String?,
thought: thought,
thoughtSignature: thoughtSignature,
);
} else {
throw unhandledFormat('functionCall', functionCall);
}
}
return switch (jsonObject) {
{'text': final String text} => TextPart(text),
{'text': final String text} => TextPart(
text,
thought: thought,
thoughtSignature: thoughtSignature,
),
{
'file_data': {
'file_uri': final String fileUri,
'mime_type': final String mimeType
}
} =>
FileData(mimeType, fileUri),
FileData(
mimeType,
fileUri,
thought: thought,
thoughtSignature: thoughtSignature,
),
{
'functionResponse': {'name': String _, 'response': Map<String, Object?> _}
} =>
throw UnimplementedError('FunctionResponse part not yet supported'),
{'inlineData': {'mimeType': String mimeType, 'data': String bytes}} =>
InlineDataPart(mimeType, base64Decode(bytes)),
{
'inlineData': {
'mimeType': String mimeType,
'data': String bytes,
}
} =>
InlineDataPart(
mimeType,
base64Decode(bytes),
thought: thought,
thoughtSignature: thoughtSignature,
),
_ => throw unhandledFormat('Part', jsonObject),
};
}

/// A datatype containing media that is part of a multi-part [Content] message.
sealed class Part {
// ignore: public_member_api_docs
Part({this.thought, this.thoughtSignature});

/// Indicates if the part is thought from the model.
final bool? thought;

/// An opaque signature for the thought.
///
/// So it can be reused in subsequent requests.
final Uint8List? thoughtSignature;

/// Convert the [Part] content to json format.
Object toJson();
}

/// A [Part] with the text content.
final class TextPart implements Part {
final class TextPart extends Part {
// ignore: public_member_api_docs
TextPart(this.text);
TextPart(this.text, {super.thought, super.thoughtSignature});

/// The text content of the [Part]
final String text;
@override
Object toJson() => {'text': text};
Object toJson() => {
'text': text,
};
}

/// A [Part] with the byte content of a file.
final class InlineDataPart implements Part {
final class InlineDataPart extends Part {
// ignore: public_member_api_docs
InlineDataPart(this.mimeType, this.bytes, {this.willContinue});
InlineDataPart(this.mimeType, this.bytes,
{this.willContinue, super.thought, super.thoughtSignature});

/// File type of the [InlineDataPart].
/// https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#media_requirements
Expand Down Expand Up @@ -165,9 +208,10 @@ final class InlineDataPart implements Part {
/// A predicted `FunctionCall` returned from the model that contains
/// a string representing the `FunctionDeclaration.name` with the
/// arguments and their values.
final class FunctionCall implements Part {
final class FunctionCall extends Part {
// ignore: public_member_api_docs
FunctionCall(this.name, this.args, {this.id});
FunctionCall(this.name, this.args,
{this.id, super.thought, super.thoughtSignature});

/// The name of the function to call.
final String name;
Expand All @@ -192,7 +236,9 @@ final class FunctionCall implements Part {
}

/// The response class for [FunctionCall]
final class FunctionResponse implements Part {
///
/// note: this part will not extends [thought] and [thoughtSignature]
final class FunctionResponse extends Part {
// ignore: public_member_api_docs
FunctionResponse(this.name, this.response, {this.id});

Expand Down Expand Up @@ -221,9 +267,10 @@ final class FunctionResponse implements Part {
}

/// A [Part] with Firebase Storage uri as prompt content
final class FileData implements Part {
final class FileData extends Part {
// ignore: public_member_api_docs
FileData(this.mimeType, this.fileUri);
FileData(this.mimeType, this.fileUri,
{super.thought, super.thoughtSignature});

/// File type of the [FileData].
/// https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#media_requirements
Expand All @@ -234,6 +281,9 @@ final class FileData implements Part {

@override
Object toJson() => {
'file_data': {'file_uri': fileUri, 'mime_type': mimeType}
'file_data': {
'file_uri': fileUri,
'mime_type': mimeType,
}
};
}
Loading
Loading