diff --git a/firebase-ai-ksp-processor/README.md b/firebase-ai-ksp-processor/README.md new file mode 100644 index 00000000000..50ecd9372d5 --- /dev/null +++ b/firebase-ai-ksp-processor/README.md @@ -0,0 +1,14 @@ +To build run `./gradlew :publishToMavenLocal` + +To integrate: add the following to your app's gradle file: + +```kotlin +plugins { + id("com.google.devtools.ksp") +} +dependencies { + implementation("com.google.firebase:firebase-ai:") + ksp("com.google.firebase:firebase-ai-processor:1.0.0") +} +``` + diff --git a/firebase-ai-ksp-processor/build.gradle.kts b/firebase-ai-ksp-processor/build.gradle.kts new file mode 100644 index 00000000000..16df3455759 --- /dev/null +++ b/firebase-ai-ksp-processor/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + kotlin("jvm") + id("java-library") + id("maven-publish") +} + +dependencies { + testImplementation(kotlin("test")) + implementation(libs.symbol.processing.api) + implementation(libs.kotlinpoet.ksp) +} + +tasks.test { useJUnitPlatform() } + +kotlin { jvmToolchain(21) } + +publishing { + publications { + create("mavenKotlin") { + from(components["kotlin"]) + groupId = "com.google.firebase" + artifactId = "firebase-ai-processor" + version = "1.0.0" + } + } + repositories { + maven { url = uri("m2/") } + mavenLocal() + } +} diff --git a/firebase-ai-ksp-processor/gradle.properties b/firebase-ai-ksp-processor/gradle.properties new file mode 100644 index 00000000000..7fc6f1ff272 --- /dev/null +++ b/firebase-ai-ksp-processor/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official diff --git a/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessor.kt b/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessor.kt new file mode 100644 index 00000000000..0384846d8f5 --- /dev/null +++ b/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessor.kt @@ -0,0 +1,342 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.ksp + +import com.google.devtools.ksp.KspExperimental +import com.google.devtools.ksp.processing.CodeGenerator +import com.google.devtools.ksp.processing.Dependencies +import com.google.devtools.ksp.processing.KSPLogger +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.symbol.ClassKind +import com.google.devtools.ksp.symbol.KSAnnotated +import com.google.devtools.ksp.symbol.KSAnnotation +import com.google.devtools.ksp.symbol.KSClassDeclaration +import com.google.devtools.ksp.symbol.KSType +import com.google.devtools.ksp.symbol.KSVisitorVoid +import com.google.devtools.ksp.symbol.Modifier +import com.squareup.kotlinpoet.ClassName +import com.squareup.kotlinpoet.CodeBlock +import com.squareup.kotlinpoet.FileSpec +import com.squareup.kotlinpoet.KModifier +import com.squareup.kotlinpoet.ParameterizedTypeName +import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy +import com.squareup.kotlinpoet.PropertySpec +import com.squareup.kotlinpoet.TypeSpec +import com.squareup.kotlinpoet.ksp.toClassName +import com.squareup.kotlinpoet.ksp.toTypeName +import com.squareup.kotlinpoet.ksp.writeTo +import javax.annotation.processing.Generated + +public class SchemaSymbolProcessor( + private val codeGenerator: CodeGenerator, + private val logger: KSPLogger, +) : SymbolProcessor { + override fun process(resolver: Resolver): List { + resolver + .getSymbolsWithAnnotation("com.google.firebase.ai.annotations.Generable") + .filterIsInstance() + .map { it to SchemaSymbolProcessorVisitor() } + .forEach { (klass, visitor) -> visitor.visitClassDeclaration(klass, Unit) } + + return emptyList() + } + + private inner class SchemaSymbolProcessorVisitor() : KSVisitorVoid() { + private val numberTypes = setOf("kotlin.Int", "kotlin.Long", "kotlin.Double", "kotlin.Float") + private val baseKdocRegex = Regex("^\\s*(.*?)((@\\w* .*)|\\z)", RegexOption.DOT_MATCHES_ALL) + private val propertyKdocRegex = + Regex("\\s*@property (\\w*) (.*?)(?=@\\w*|\\z)", RegexOption.DOT_MATCHES_ALL) + + override fun visitClassDeclaration(classDeclaration: KSClassDeclaration, data: Unit) { + val isDataClass = classDeclaration.modifiers.contains(Modifier.DATA) + if (!isDataClass) { + logger.error("${classDeclaration.qualifiedName} is not a data class") + } + val generatedSchemaFile = generateFileSpec(classDeclaration) + generatedSchemaFile.writeTo( + codeGenerator, + Dependencies(true, classDeclaration.containingFile!!), + ) + } + + fun generateFileSpec(classDeclaration: KSClassDeclaration): FileSpec { + return FileSpec.builder( + classDeclaration.packageName.asString(), + "${classDeclaration.simpleName.asString()}GeneratedSchema", + ) + .addImport("com.google.firebase.ai.type", "JsonSchema") + .addType( + TypeSpec.classBuilder("${classDeclaration.simpleName.asString()}GeneratedSchema") + .addAnnotation(Generated::class) + .addType( + TypeSpec.companionObjectBuilder() + .addProperty( + PropertySpec.builder( + "SCHEMA", + ClassName("com.google.firebase.ai.type", "JsonSchema") + .parameterizedBy( + ClassName( + classDeclaration.packageName.asString(), + classDeclaration.simpleName.asString() + ) + ), + KModifier.PUBLIC, + ) + .mutable(false) + .initializer( + CodeBlock.builder() + .add( + generateCodeBlockForSchema(type = classDeclaration.asType(emptyList())) + ) + .build() + ) + .build() + ) + .build() + ) + .build() + ) + .build() + } + + @OptIn(KspExperimental::class) + fun generateCodeBlockForSchema( + name: String? = null, + description: String? = null, + type: KSType, + parentType: KSType? = null, + guideAnnotation: KSAnnotation? = null, + ): CodeBlock { + val parameterizedName = type.toTypeName() as? ParameterizedTypeName + val className = parameterizedName?.rawType ?: type.toClassName() + val kdocString = type.declaration.docString ?: "" + val baseKdoc = extractBaseKdoc(kdocString) + val propertyDocs = extractPropertyKdocs(kdocString) + val generableClassAnnotation = + type.annotations.firstOrNull() { it.shortName.getShortName() == "Generable" } + val description = + getDescriptionFromAnnotations( + guideAnnotation, + generableClassAnnotation, + description, + baseKdoc + ) + val minimum = getDoubleFromAnnotation(guideAnnotation, "minimum") + val maximum = getDoubleFromAnnotation(guideAnnotation, "maximum") + val minItems = getIntFromAnnotation(guideAnnotation, "minItems") + val maxItems = getIntFromAnnotation(guideAnnotation, "maxItems") + val format = getStringFromAnnotation(guideAnnotation, "format") + val pattern = getStringFromAnnotation(guideAnnotation, "pattern") + val builder = CodeBlock.builder() + when (className.canonicalName) { + "kotlin.Int" -> { + builder.addStatement("JsonSchema.integer(").indent() + } + "kotlin.Long" -> { + builder.addStatement("JsonSchema.long(").indent() + } + "kotlin.Boolean" -> { + builder.addStatement("JsonSchema.boolean(").indent() + } + "kotlin.Float" -> { + builder.addStatement("JsonSchema.float(").indent() + } + "kotlin.Double" -> { + builder.addStatement("JsonSchema.double(").indent() + } + "kotlin.String" -> { + builder.addStatement("JsonSchema.string(").indent() + } + "kotlin.collections.List" -> { + val listTypeParam = type.arguments.first().type!!.resolve() + val listParamCodeBlock = + generateCodeBlockForSchema(type = listTypeParam, parentType = type) + builder + .addStatement("JsonSchema.array(") + .indent() + .addStatement("items = ") + .add(listParamCodeBlock) + .addStatement(",") + } + else -> { + if ((type.declaration as? KSClassDeclaration)?.classKind == ClassKind.ENUM_CLASS) { + val enumValues = + (type.declaration as KSClassDeclaration) + .declarations + .filterIsInstance(KSClassDeclaration::class.java) + .map { it.simpleName.asString() } + .toList() + builder + .addStatement("JsonSchema.enumeration(") + .indent() + .addStatement("clazz = ${type.declaration.qualifiedName!!.asString()}::class,") + .addStatement("values = listOf(") + .indent() + .addStatement(enumValues.joinToString { "\"$it\"" }) + .unindent() + .addStatement("),") + } else { + builder + .addStatement("JsonSchema.obj(") + .indent() + .addStatement("clazz = ${type.declaration.qualifiedName!!.asString()}::class,") + .addStatement("properties = mapOf(") + .indent() + val properties = + (type.declaration as KSClassDeclaration).getAllProperties().associate { property -> + val propertyName = property.simpleName.asString() + propertyName to + generateCodeBlockForSchema( + type = property.type.resolve(), + parentType = type, + description = propertyDocs[propertyName], + name = propertyName, + guideAnnotation = + property.annotations.firstOrNull() { it.shortName.getShortName() == "Guide" }, + ) + } + properties.entries.forEach { + builder + .addStatement("%S to ", it.key) + .indent() + .add(it.value) + .unindent() + .addStatement(", ") + } + builder.unindent().addStatement("),") + } + } + } + if (name != null) { + builder.addStatement("title = %S,", name) + } + if (description != null) { + builder.addStatement("description = %S,", description) + } + if ((minimum != null || maximum != null) && !numberTypes.contains(className.canonicalName)) { + logger.warn( + "${parentType?.toClassName()?.simpleName?.let { "$it." }}$name is not a number type, " + + "minimum and maximum are not valid parameters to specify in @Guide" + ) + } + if ( + (minItems != null || maxItems != null) && + className.canonicalName != "kotlin.collections.List" + ) { + logger.warn( + "${parentType?.toClassName()?.simpleName?.let { "$it." }}$name is not a List type, " + + "minItems and maxItems are not valid parameters to specify in @Guide" + ) + } + if ((format != null || pattern != null) && className.canonicalName != "kotlin.String") { + logger.warn( + "${parentType?.toClassName()?.simpleName?.let { "$it." }}$name is not a String type, " + + "format and pattern are not a valid parameter to specify in @Guide" + ) + } + if (minimum != null) { + builder.addStatement("minimum = %L,", minimum) + } + if (maximum != null) { + builder.addStatement("maximum = %L,", maximum) + } + if (minItems != null) { + builder.addStatement("minItems = %L,", minItems) + } + if (maxItems != null) { + builder.addStatement("maxItems = %L,", maxItems) + } + if (format != null) { + builder.addStatement("format = %S,", format) + } + if (pattern != null) { + builder.addStatement("pattern = %S,", pattern) + } + builder.addStatement("nullable = %L)", className.isNullable).unindent() + return builder.build() + } + + private fun getDescriptionFromAnnotations( + guideAnnotation: KSAnnotation?, + generableClassAnnotation: KSAnnotation?, + description: String?, + baseKdoc: String?, + ): String? { + val guidePropertyDescription = getStringFromAnnotation(guideAnnotation, "description") + + val guideClassDescription = getStringFromAnnotation(generableClassAnnotation, "description") + + return guidePropertyDescription ?: guideClassDescription ?: description ?: baseKdoc + } + + private fun getDoubleFromAnnotation( + guideAnnotation: KSAnnotation?, + doubleName: String, + ): Double? { + val guidePropertyDoubleValue = + guideAnnotation + ?.arguments + ?.firstOrNull { it.name?.getShortName()?.equals(doubleName) == true } + ?.value as? Double + if (guidePropertyDoubleValue == null || guidePropertyDoubleValue == -1.0) { + return null + } + return guidePropertyDoubleValue + } + + private fun getIntFromAnnotation(guideAnnotation: KSAnnotation?, intName: String): Int? { + val guidePropertyIntValue = + guideAnnotation + ?.arguments + ?.firstOrNull { it.name?.getShortName()?.equals(intName) == true } + ?.value as? Int + if (guidePropertyIntValue == null || guidePropertyIntValue == -1) { + return null + } + return guidePropertyIntValue + } + + private fun getStringFromAnnotation( + guideAnnotation: KSAnnotation?, + stringName: String, + ): String? { + val guidePropertyStringValue = + guideAnnotation + ?.arguments + ?.firstOrNull { it.name?.getShortName()?.equals(stringName) == true } + ?.value as? String + if (guidePropertyStringValue.isNullOrEmpty()) { + return null + } + return guidePropertyStringValue + } + + private fun extractBaseKdoc(kdoc: String): String? { + return baseKdocRegex.matchEntire(kdoc)?.groups?.get(1)?.value?.trim().let { + if (it.isNullOrEmpty()) null else it + } + } + + private fun extractPropertyKdocs(kdoc: String): Map { + return propertyKdocRegex + .findAll(kdoc) + .map { it.groups[1]!!.value to it.groups[2]!!.value.replace("\n", "").trim() } + .toMap() + } + } +} diff --git a/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessorProvider.kt b/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessorProvider.kt new file mode 100644 index 00000000000..2c8015bc8a9 --- /dev/null +++ b/firebase-ai-ksp-processor/src/main/kotlin/com/google/firebase/ai/ksp/SchemaSymbolProcessorProvider.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.ksp + +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider + +public class SchemaSymbolProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor { + return SchemaSymbolProcessor(environment.codeGenerator, environment.logger) + } +} diff --git a/firebase-ai-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/firebase-ai-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 00000000000..83d92f28c7e --- /dev/null +++ b/firebase-ai-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1 @@ +com.google.firebase.ai.ksp.SchemaSymbolProcessorProvider \ No newline at end of file diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index 7c4c924ab9f..7b78b0df0a3 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,5 +1,8 @@ # Unreleased +- [feature] Added `JsonSchema`, `AutoFunctionDeclaration`, support for automatic function calling, + and the firebase-ai-ksp processor's annotation (`Generable`, `Tool`, and `Guide`) + # 17.7.0 - [changed] Added `LiveAudioConversationConfig` to control different aspects of the conversation diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 357fb1c2a8b..e8b4d45dbf8 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -73,6 +73,8 @@ package com.google.firebase.ai { method public kotlinx.coroutines.flow.Flow generateContentStream(com.google.firebase.ai.type.Content prompt, com.google.firebase.ai.type.Content... prompts); method public kotlinx.coroutines.flow.Flow generateContentStream(String prompt); method public kotlinx.coroutines.flow.Flow generateContentStream(java.util.List prompt); + method public suspend Object? generateObject(com.google.firebase.ai.type.JsonSchema jsonSchema, com.google.firebase.ai.type.Content prompt, com.google.firebase.ai.type.Content[] prompts, kotlin.coroutines.Continuation>); + method public suspend Object? generateObject(com.google.firebase.ai.type.JsonSchema jsonSchema, String prompt, kotlin.coroutines.Continuation>); method public com.google.firebase.ai.Chat startChat(java.util.List history = emptyList()); } @@ -98,6 +100,32 @@ package com.google.firebase.ai { } +package com.google.firebase.ai.annotations { + + @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.SOURCE) @kotlin.annotation.Target(allowedTargets=kotlin.annotation.AnnotationTarget.CLASS) public @interface Generable { + method public abstract String description() default ""; + property public abstract String description; + } + + @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.SOURCE) @kotlin.annotation.Target(allowedTargets=kotlin.annotation.AnnotationTarget.PROPERTY) public @interface Guide { + method public abstract String description() default ""; + method public abstract String format() default ""; + method public abstract int maxItems() default -1; + method public abstract double maximum() default -1.0; + method public abstract int minItems() default -1; + method public abstract double minimum() default -1.0; + method public abstract String pattern() default ""; + property public abstract String description; + property public abstract String format; + property public abstract int maxItems; + property public abstract double maximum; + property public abstract int minItems; + property public abstract double minimum; + property public abstract String pattern; + } + +} + package com.google.firebase.ai.java { public abstract class ChatFutures { @@ -218,6 +246,25 @@ package com.google.firebase.ai.type { ctor public AudioTranscriptionConfig(); } + public final class AutoFunctionDeclaration { + method public String getDescription(); + method public kotlin.jvm.functions.Function2,java.lang.Object?>? getFunctionReference(); + method public com.google.firebase.ai.type.JsonSchema getInputSchema(); + method public String getName(); + method public com.google.firebase.ai.type.JsonSchema? getOutputSchema(); + property public final String description; + property public final kotlin.jvm.functions.Function2,java.lang.Object?>? functionReference; + property public final com.google.firebase.ai.type.JsonSchema inputSchema; + property public final String name; + property public final com.google.firebase.ai.type.JsonSchema? outputSchema; + field public static final com.google.firebase.ai.type.AutoFunctionDeclaration.Companion Companion; + } + + public static final class AutoFunctionDeclaration.Companion { + method public com.google.firebase.ai.type.AutoFunctionDeclaration create(String functionName, String description, com.google.firebase.ai.type.JsonSchema inputSchema, com.google.firebase.ai.type.JsonSchema outputSchema, kotlin.jvm.functions.Function1? functionReference = null); + method public com.google.firebase.ai.type.AutoFunctionDeclaration create(String functionName, String description, com.google.firebase.ai.type.JsonSchema inputSchema, kotlin.jvm.functions.Function1? functionReference = null); + } + public final class BlockReason { method public String getName(); method public int getOrdinal(); @@ -454,12 +501,18 @@ package com.google.firebase.ai.type { property public final com.google.firebase.ai.type.UsageMetadata? usageMetadata; } + public final class GenerateObjectResponse { + method public T? getObject(int candidateIndex = 0); + method public com.google.firebase.ai.type.GenerateContentResponse getResponse(); + property public final com.google.firebase.ai.type.GenerateContentResponse response; + } + public final class GenerationConfig { + method public com.google.firebase.ai.type.GenerationConfig.Builder toBuilder(); field public static final com.google.firebase.ai.type.GenerationConfig.Companion Companion; } public static final class GenerationConfig.Builder { - ctor public GenerationConfig.Builder(); method public com.google.firebase.ai.type.GenerationConfig build(); method public com.google.firebase.ai.type.GenerationConfig.Builder setCandidateCount(Integer? candidateCount); method public com.google.firebase.ai.type.GenerationConfig.Builder setFrequencyPenalty(Float? frequencyPenalty); @@ -468,6 +521,7 @@ package com.google.firebase.ai.type { method public com.google.firebase.ai.type.GenerationConfig.Builder setResponseMimeType(String? responseMimeType); method public com.google.firebase.ai.type.GenerationConfig.Builder setResponseModalities(java.util.List? responseModalities); method public com.google.firebase.ai.type.GenerationConfig.Builder setResponseSchema(com.google.firebase.ai.type.Schema? responseSchema); + method public com.google.firebase.ai.type.GenerationConfig.Builder setResponseSchemaJson(com.google.firebase.ai.type.JsonSchema? responseSchemaJson); method public com.google.firebase.ai.type.GenerationConfig.Builder setStopSequences(java.util.List? stopSequences); method public com.google.firebase.ai.type.GenerationConfig.Builder setTemperature(Float? temperature); method public com.google.firebase.ai.type.GenerationConfig.Builder setThinkingConfig(com.google.firebase.ai.type.ThinkingConfig? thinkingConfig); @@ -477,6 +531,7 @@ package com.google.firebase.ai.type { field public Float? frequencyPenalty; field public Integer? maxOutputTokens; field public Float? presencePenalty; + field public com.google.firebase.ai.type.JsonSchema? responseJsonSchema; field public String? responseMimeType; field public java.util.List? responseModalities; field public com.google.firebase.ai.type.Schema? responseSchema; @@ -875,6 +930,161 @@ package com.google.firebase.ai.type { public final class InvalidStateException extends com.google.firebase.ai.type.FirebaseAIException { } + public final class JsonSchema { + method public static com.google.firebase.ai.type.JsonSchema anyOf(java.util.List> schemas); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null, Integer? minItems = null); + method public static com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null, Integer? minItems = null, Integer? maxItems = null); + method public static com.google.firebase.ai.type.JsonSchema boolean(); + method public static com.google.firebase.ai.type.JsonSchema boolean(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema boolean(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema boolean(String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null, boolean nullable = false, String? title = null); + method public java.util.List>? getAnyOf(); + method public kotlin.reflect.KClass getClazz(); + method public String? getDescription(); + method public java.util.List? getEnum(); + method public String? getFormat(); + method public com.google.firebase.ai.type.JsonSchema? getItems(); + method public Integer? getMaxItems(); + method public Double? getMaximum(); + method public Integer? getMinItems(); + method public Double? getMinimum(); + method public Boolean? getNullable(); + method public String? getPattern(); + method public java.util.Map>? getProperties(); + method public java.util.List? getRequired(); + method public String? getTitle(); + method public String getType(); + method public static com.google.firebase.ai.type.JsonSchema numDouble(); + method public static com.google.firebase.ai.type.JsonSchema numDouble(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public static com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public static com.google.firebase.ai.type.JsonSchema numFloat(); + method public static com.google.firebase.ai.type.JsonSchema numFloat(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public static com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public static com.google.firebase.ai.type.JsonSchema numInt(); + method public static com.google.firebase.ai.type.JsonSchema numInt(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public static com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public static com.google.firebase.ai.type.JsonSchema numLong(); + method public static com.google.firebase.ai.type.JsonSchema numLong(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public static com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList()); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList()); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false, String? title = null); + method public static com.google.firebase.ai.type.JsonSchema str(); + method public static com.google.firebase.ai.type.JsonSchema str(String? description = null); + method public static com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false); + method public static com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null); + method public static com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null, String? title = null); + property public final java.util.List>? anyOf; + property public final kotlin.reflect.KClass clazz; + property public final String? description; + property public final java.util.List? enum; + property public final String? format; + property public final com.google.firebase.ai.type.JsonSchema? items; + property public final Integer? maxItems; + property public final Double? maximum; + property public final Integer? minItems; + property public final Double? minimum; + property public final Boolean? nullable; + property public final String? pattern; + property public final java.util.Map>? properties; + property public final java.util.List? required; + property public final String? title; + property public final String type; + field public static final com.google.firebase.ai.type.JsonSchema.Companion Companion; + } + + public static final class JsonSchema.Companion { + method public com.google.firebase.ai.type.JsonSchema anyOf(java.util.List> schemas); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null, Integer? minItems = null); + method public com.google.firebase.ai.type.JsonSchema> array(com.google.firebase.ai.type.JsonSchema items, String? description = null, boolean nullable = false, String? title = null, Integer? minItems = null, Integer? maxItems = null); + method public com.google.firebase.ai.type.JsonSchema boolean(); + method public com.google.firebase.ai.type.JsonSchema boolean(String? description = null); + method public com.google.firebase.ai.type.JsonSchema boolean(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema boolean(String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema enumeration(java.util.List values, kotlin.reflect.KClass clazz, String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema numDouble(); + method public com.google.firebase.ai.type.JsonSchema numDouble(String? description = null); + method public com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public com.google.firebase.ai.type.JsonSchema numDouble(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public com.google.firebase.ai.type.JsonSchema numFloat(); + method public com.google.firebase.ai.type.JsonSchema numFloat(String? description = null); + method public com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public com.google.firebase.ai.type.JsonSchema numFloat(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public com.google.firebase.ai.type.JsonSchema numInt(); + method public com.google.firebase.ai.type.JsonSchema numInt(String? description = null); + method public com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public com.google.firebase.ai.type.JsonSchema numInt(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public com.google.firebase.ai.type.JsonSchema numLong(); + method public com.google.firebase.ai.type.JsonSchema numLong(String? description = null); + method public com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null); + method public com.google.firebase.ai.type.JsonSchema numLong(String? description = null, boolean nullable = false, String? title = null, Double? minimum = null, Double? maximum = null); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList()); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList()); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema obj(java.util.Map> properties, kotlin.reflect.KClass clazz, java.util.List optionalProperties = emptyList(), String? description = null, boolean nullable = false, String? title = null); + method public com.google.firebase.ai.type.JsonSchema str(); + method public com.google.firebase.ai.type.JsonSchema str(String? description = null); + method public com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false); + method public com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null); + method public com.google.firebase.ai.type.JsonSchema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null, String? title = null); + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveAudioConversationConfig { field public static final com.google.firebase.ai.type.LiveAudioConversationConfig.Companion Companion; } @@ -1297,7 +1507,8 @@ package com.google.firebase.ai.type { public final class Tool { method public static com.google.firebase.ai.type.Tool codeExecution(); - method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List functionDeclarations); + method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List? functionDeclarations = null); + method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List? functionDeclarations = null, java.util.List>? autoFunctionDeclarations); method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch()); method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext()); field public static final com.google.firebase.ai.type.Tool.Companion Companion; @@ -1305,7 +1516,8 @@ package com.google.firebase.ai.type { public static final class Tool.Companion { method public com.google.firebase.ai.type.Tool codeExecution(); - method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List functionDeclarations); + method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List? functionDeclarations = null); + method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List? functionDeclarations = null, java.util.List>? autoFunctionDeclarations); method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch()); method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext()); } diff --git a/firebase-ai/gradle.properties b/firebase-ai/gradle.properties index 215d4a50f32..2181d491391 100644 --- a/firebase-ai/gradle.properties +++ b/firebase-ai/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=17.7.1 +version=17.8.0 latestReleasedVersion=17.7.0 diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt index 4ff2c3c8e82..65e335e9aa6 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt @@ -18,18 +18,18 @@ package com.google.firebase.ai import android.graphics.Bitmap import com.google.firebase.ai.type.Content +import com.google.firebase.ai.type.FunctionCallPart +import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.GenerateContentResponse -import com.google.firebase.ai.type.ImagePart -import com.google.firebase.ai.type.InlineDataPart import com.google.firebase.ai.type.InvalidStateException -import com.google.firebase.ai.type.Part import com.google.firebase.ai.type.TextPart import com.google.firebase.ai.type.content import java.util.LinkedList import java.util.concurrent.Semaphore import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.transform /** * Representation of a multi-turn interaction with a model. @@ -66,11 +66,22 @@ public class Chat( public suspend fun sendMessage(prompt: Content): GenerateContentResponse { prompt.assertComesFromUser() attemptLock() + var response: GenerateContentResponse + var tempPrompt = prompt try { - val fullPrompt = history + prompt - val response = model.generateContent(fullPrompt.first(), *fullPrompt.drop(1).toTypedArray()) - history.add(prompt) - history.add(response.candidates.first().content) + while (true) { + response = model.generateContent(listOf(*history.toTypedArray(), tempPrompt)) + val responsePart = response.candidates.first().content.parts.first() + + history.add(tempPrompt) + history.add(response.candidates.first().content) + if (responsePart is FunctionCallPart && model.hasFunction(responsePart)) { + val output = model.executeFunction(responsePart) + tempPrompt = Content("function", listOf(FunctionResponsePart(responsePart.name, output))) + } else { + break + } + } return response } finally { lock.release() @@ -131,10 +142,8 @@ public class Chat( val fullPrompt = history + prompt val flow = model.generateContentStream(fullPrompt.first(), *fullPrompt.drop(1).toTypedArray()) - val bitmaps = LinkedList() - val inlineDataParts = LinkedList() - val text = StringBuilder() - val parts = mutableListOf() + val tempHistory = LinkedList() + tempHistory.add(prompt) /** * TODO: revisit when images and inline data are returned. This will cause issues with how @@ -142,28 +151,11 @@ public class Chat( * represented as image/text */ return flow - .onEach { - for (part in it.candidates.first().content.parts) { - when (part) { - is TextPart -> text.append(part.text) - is ImagePart -> bitmaps.add(part.image) - is InlineDataPart -> inlineDataParts.add(part) - } - parts.add(part) - } - } + .transform { response -> automaticFunctionExecutingTransform(this, tempHistory, response) } .onCompletion { lock.release() if (it == null) { - val content = - content("model") { - setParts( - parts.filterNot { part -> part is TextPart && !part.hasContent() }.toMutableList() - ) - } - - history.add(prompt) - history.add(content) + history.addAll(tempHistory) } } } @@ -206,6 +198,67 @@ public class Chat( return sendMessageStream(content) } + private suspend fun automaticFunctionExecutingTransform( + transformer: FlowCollector, + tempHistory: LinkedList, + response: GenerateContentResponse + ) { + for (part in response.candidates.first().content.parts) { + when (part) { + is TextPart -> { + transformer.emit(response) + addTextToHistory(tempHistory, part) + } + is FunctionCallPart -> { + val functionCall = + response.candidates.first().content.parts.first { it is FunctionCallPart } + as FunctionCallPart + if (model.hasFunction(functionCall)) { + val output = model.executeFunction(functionCall) + val functionResponse = + Content("function", listOf(FunctionResponsePart(functionCall.name, output))) + tempHistory.add(response.candidates.first().content) + tempHistory.add(functionResponse) + model + .generateContentStream(listOf(*history.toTypedArray(), *tempHistory.toTypedArray())) + .collect { automaticFunctionExecutingTransform(transformer, tempHistory, it) } + } else { + transformer.emit(response) + tempHistory.add(Content("model", listOf(part))) + } + } + else -> { + transformer.emit(response) + tempHistory.add(Content("model", listOf(part))) + } + } + } + } + + private fun addTextToHistory(tempHistory: LinkedList, textPart: TextPart) { + val lastContent = tempHistory.lastOrNull() + if (lastContent?.role == "model" && lastContent.parts.any { it is TextPart }) { + tempHistory.removeLast() + val editedContent = + Content( + "model", + lastContent.parts.map { + when (it) { + is TextPart -> { + TextPart(it.text + textPart.text) + } + else -> { + it + } + } + } + ) + tempHistory.add(editedContent) + return + } + tempHistory.add(Content("model", listOf(textPart))) + } + private fun Content.assertComesFromUser() { if (role !in listOf("user", "function")) { throw InvalidStateException("Chat prompts should come from the 'user' or 'function' role.") diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt index 45aa1e567e3..1eef17e7aa5 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt @@ -22,15 +22,19 @@ import com.google.firebase.ai.common.APIController import com.google.firebase.ai.common.AppCheckHeaderProvider import com.google.firebase.ai.common.CountTokensRequest import com.google.firebase.ai.common.GenerateContentRequest +import com.google.firebase.ai.type.AutoFunctionDeclaration import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.CountTokensResponse import com.google.firebase.ai.type.FinishReason import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.FunctionCallPart import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.GenerateObjectResponse import com.google.firebase.ai.type.GenerationConfig import com.google.firebase.ai.type.GenerativeBackend import com.google.firebase.ai.type.GenerativeBackendEnum import com.google.firebase.ai.type.InvalidStateException +import com.google.firebase.ai.type.JsonSchema import com.google.firebase.ai.type.PromptBlockedException import com.google.firebase.ai.type.RequestOptions import com.google.firebase.ai.type.ResponseStoppedException @@ -45,6 +49,11 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.map import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.InternalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.serializerOrNull /** * Represents a multimodal model (like Gemini), capable of generating content based on various input @@ -111,11 +120,43 @@ internal constructor( vararg prompts: Content ): GenerateContentResponse = try { - controller.generateContent(constructRequest(prompt, *prompts)).toPublic().validate() + controller.generateContent(constructRequest(null, prompt, *prompts)).toPublic().validate() } catch (e: Throwable) { throw FirebaseAIException.from(e) } + /** + * Generates an object from the input [Content] given to the model as a prompt. + * + * @param jsonSchema A schema for the output + * @param prompt The input(s) given to the model as a prompt. + * @return The content generated by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public suspend fun generateObject( + jsonSchema: JsonSchema, + prompt: Content, + vararg prompts: Content + ): GenerateObjectResponse { + try { + val config = + (generationConfig?.toBuilder() ?: GenerationConfig.builder()) + .setResponseSchemaJson(jsonSchema) + .setResponseMimeType("application/json") + .build() + return GenerateObjectResponse( + controller + .generateContent(constructRequest(config, prompt, *prompts)) + .toPublic() + .validate(), + jsonSchema.clazz + ) + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + } + /** * Generates new content from the input [Content] given to the model as a prompt. * @@ -144,7 +185,7 @@ internal constructor( vararg prompts: Content ): Flow = controller - .generateContentStream(constructRequest(prompt, *prompts)) + .generateContentStream(constructRequest(null, prompt, *prompts)) .catch { throw FirebaseAIException.from(it) } .map { it.toPublic().validate() } @@ -173,6 +214,20 @@ internal constructor( public suspend fun generateContent(prompt: String): GenerateContentResponse = generateContent(content { text(prompt) }) + /** + * Generates an object from the text input given to the model as a prompt. + * + * @param jsonSchema A schema for the output + * @param prompt The text to be send to the model as a prompt. + * @return The content generated by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public suspend fun generateObject( + jsonSchema: JsonSchema, + prompt: String + ): GenerateObjectResponse = generateObject(jsonSchema, content { text(prompt) }) + /** * Generates new content as a stream from the text input given to the model as a prompt. * @@ -266,8 +321,51 @@ internal constructor( return countTokens(content { image(prompt) }) } + internal fun hasFunction(call: FunctionCallPart): Boolean { + return tools + ?.flatMap { it.autoFunctionDeclarations?.filterNotNull() ?: emptyList() } + ?.firstOrNull { it.name == call.name } != null + } + + @OptIn(InternalSerializationApi::class) + internal suspend fun executeFunction(call: FunctionCallPart): JsonObject { + if (tools == null) { + throw RuntimeException("No registered tools") + } + val tool = tools.flatMap { it.autoFunctionDeclarations?.filterNotNull() ?: emptyList() } + val declaration = + tool.firstOrNull() { it.name == call.name } + ?: throw RuntimeException("No registered function named ${call.name}") + return executeFunction( + declaration as AutoFunctionDeclaration, + call.args["param"].toString() + ) + } + + @OptIn(InternalSerializationApi::class) + internal suspend fun executeFunction( + functionDeclaration: AutoFunctionDeclaration, + parameter: String + ): JsonObject { + val inputDeserializer = + functionDeclaration.inputSchema.clazz.serializerOrNull() + ?: throw RuntimeException( + "Function input type ${functionDeclaration.inputSchema.clazz.qualifiedName} is not @Serializable" + ) + val input = Json.decodeFromString(inputDeserializer, parameter) + val functionReference = + functionDeclaration.functionReference + ?: throw RuntimeException("Function reference for ${functionDeclaration.name} is missing") + val output = functionReference.invoke(input) + val outputSerializer = functionDeclaration.outputSchema?.clazz?.serializerOrNull() + if (outputSerializer != null) { + return Json.encodeToJsonElement(outputSerializer, output).jsonObject + } + return output as JsonObject + } + @OptIn(ExperimentalSerializationApi::class) - private fun constructRequest(vararg prompt: Content) = + private fun constructRequest(overrideConfig: GenerationConfig? = null, vararg prompt: Content) = GenerateContentRequest( modelName, prompt.map { it.toInternal() }, @@ -283,18 +381,21 @@ internal constructor( } } ?.map { it.toInternal() }, - generationConfig?.toInternal(), + (overrideConfig ?: generationConfig)?.toInternal(), tools?.map { it.toInternal() }, toolConfig?.toInternal(), systemInstruction?.copy(role = "system")?.toInternal(), ) - private fun constructRequest(prompt: List) = constructRequest(*prompt.toTypedArray()) + private fun constructRequest(prompt: List) = + constructRequest(null, *prompt.toTypedArray()) private fun constructCountTokensRequest(vararg prompt: Content) = when (generativeBackend.backend) { - GenerativeBackendEnum.GOOGLE_AI -> CountTokensRequest.forGoogleAI(constructRequest(*prompt)) - GenerativeBackendEnum.VERTEX_AI -> CountTokensRequest.forVertexAI(constructRequest(*prompt)) + GenerativeBackendEnum.GOOGLE_AI -> + CountTokensRequest.forGoogleAI(constructRequest(null, *prompt)) + GenerativeBackendEnum.VERTEX_AI -> + CountTokensRequest.forVertexAI(constructRequest(null, *prompt)) } private fun GenerateContentResponse.validate() = apply { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Generable.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Generable.kt new file mode 100644 index 00000000000..64b18ca071b --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Generable.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.annotations + +/** + * This annotation is used with the firebase-ai-ksp-processor plugin to generate JsonSchema that + * match an existing kotlin class structure. For more info see: + * https://github.com/firebase/firebase-android-sdk/blob/main/firebase-ai-ksp-processor/README.md + * + * @property description a description of the class + */ +@Target(AnnotationTarget.CLASS) +@Retention(AnnotationRetention.SOURCE) +public annotation class Generable(public val description: String = "") diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Guide.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Guide.kt new file mode 100644 index 00000000000..ddc8e892757 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/annotations/Guide.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.annotations + +/** + * This annotation is used with the firebase-ai-ksp-processor plugin to provide extra information on + * generated classes and fields. + * @property description a description of the field + * @property minimum the minimum value (exclusive) which the numeric field may contain + * @property maximum the maximum value (exclusive) which the numeric field may contain + * @property minItems the minimum number of items in a list + * @property maxItems the maximum number of items in a list + * @property format the format that a field must conform to + * @property pattern the regular expression that a string field must conform to + */ +@Target(AnnotationTarget.PROPERTY) +@Retention(AnnotationRetention.SOURCE) +public annotation class Guide( + public val description: String = "", + public val minimum: Double = -1.0, + public val maximum: Double = -1.0, + public val minItems: Int = -1, + public val maxItems: Int = -1, + public val format: String = "", + public val pattern: String = "", +) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt index e992f92e674..9d64cfb54df 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt @@ -163,6 +163,7 @@ internal constructor( } catch (e: Throwable) { throw FirebaseAIException.from(e) } + suspend fun generateObject() suspend fun templateGenerateContent( templateId: String, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AutoFunctionDeclaration.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AutoFunctionDeclaration.kt new file mode 100644 index 00000000000..9b85ed8312f --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AutoFunctionDeclaration.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlinx.serialization.json.JsonObject + +/** + * Defines a function that the model can use as a tool. Including a function references to enable + * automatic function calling. + * + * When generating responses, the model might need external information or require the application + * to perform an action. `AutoFunctionDeclaration` provides the necessary information for the model + * to create a [FunctionCallPart], which instructs the client to execute the corresponding function. + * The client then sends the result back to the model as a [FunctionResponsePart]. + * + * For example + * + * ``` + * val getExchangeRate = AutoFunctionDeclaration.create( + * name = "getExchangeRate", + * description = "Get the exchange rate for currencies between countries.", + * inputSchema = CurrencyRequest.schema, + * outputSchema = CurrencyResponse.schema, + * ) { + * // make an api request to convert currencies and return the result + * } + * ``` + * @see JsonSchema + */ +public class AutoFunctionDeclaration +internal constructor( + public val name: String, + public val description: String, + public val inputSchema: JsonSchema, + public val outputSchema: JsonSchema? = null, + public val functionReference: (suspend (I) -> O)? = null +) { + public companion object { + + /** + * Create a strongly typed function declaration with an associated function reference. + * + * @param functionName the name of the function (to the model) + * @param description the description of the function + * @param inputSchema the object the model must provide to you as input + * @param outputSchema the type that will be return to the model when the function is executed + * @param functionReference the function that will be executed when requested by the model. + */ + public fun create( + functionName: String, + description: String, + inputSchema: JsonSchema, + outputSchema: JsonSchema, + functionReference: ((I) -> O)? = null + ): AutoFunctionDeclaration { + return AutoFunctionDeclaration( + functionName, + description, + inputSchema, + outputSchema, + functionReference + ) + } + + /** + * Create a strongly typed function declaration with an associated function reference. This + * version allows an arbitrary JsonObject as output rather than a strict schema. + * + * @param functionName the name of the function (to the model) + * @param description the description of the function + * @param inputSchema the object the model must provide to you as input + * @param functionReference the function that will be executed when requested by the model. + */ + public fun create( + functionName: String, + description: String, + inputSchema: JsonSchema, + functionReference: ((I) -> JsonObject)? = null + ): AutoFunctionDeclaration { + return AutoFunctionDeclaration( + functionName, + description, + inputSchema, + null, + functionReference + ) + } + } + + internal fun toInternal(): FunctionDeclaration.Internal { + return FunctionDeclaration.Internal( + name, + description, + null, + JsonSchema.obj(mapOf("param" to inputSchema)).toInternalJson(), + outputSchema?.toInternalJson() + ) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/FunctionDeclaration.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/FunctionDeclaration.kt index 65b753efda7..7c7eb338ead 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/FunctionDeclaration.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/FunctionDeclaration.kt @@ -61,12 +61,14 @@ public class FunctionDeclaration( internal val schema: Schema = Schema.obj(properties = parameters, optionalProperties = optionalParameters, nullable = false) - internal fun toInternal() = Internal(name, description, schema.toInternalOpenApi()) + internal fun toInternal() = Internal(name, description, schema.toInternalOpenApi(), null, null) @Serializable internal data class Internal( val name: String, val description: String, - val parameters: Schema.InternalOpenAPI + val parameters: Schema.InternalOpenAPI?, + val parametersJsonSchema: Schema.InternalJson?, + val responseJsonSchema: Schema.InternalJson?, ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateObjectResponse.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateObjectResponse.kt new file mode 100644 index 00000000000..8fd54a7616e --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateObjectResponse.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlin.reflect.KClass +import kotlinx.serialization.InternalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.serializerOrNull + +public class GenerateObjectResponse +internal constructor(public val response: GenerateContentResponse, internal val clazz: KClass) { + + @OptIn(InternalSerializationApi::class) + public fun getObject(candidateIndex: Int = 0): T? { + val candidate = response.candidates[candidateIndex] + val deserializer = + clazz.serializerOrNull() + ?: throw RuntimeException("Object type ${clazz.qualifiedName} is not @Serializable") + val text = + candidate.content.parts + .filter { !it.isThought } + .filterIsInstance() + .joinToString(" ") { it.text } + if (text.isEmpty()) { + return null + } + return Json.decodeFromString(deserializer, text) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt index a496098787f..23a29c6d9f7 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt @@ -67,7 +67,10 @@ import kotlinx.serialization.Serializable * - `application/json`: JSON response in the candidates. * * @property responseSchema Output schema of the generated candidate text. If set, a compatible - * [responseMimeType] must also be set. + * [responseMimeType] must also be set. This is mutually exclusive with [responseJsonSchema]. + * + * @property responseJsonSchema Output schema of the generated candidate text. If set, a compatible + * [responseMimeType] must also be set. This is mutually exclusive with [responseSchema]. * * Compatible MIME types: * - `application/json`: Schema for JSON response. @@ -90,6 +93,7 @@ private constructor( internal val stopSequences: List?, internal val responseMimeType: String?, internal val responseSchema: Schema?, + internal val responseJsonSchema: JsonSchema<*>?, internal val responseModalities: List?, internal val thinkingConfig: ThinkingConfig?, ) { @@ -120,6 +124,8 @@ private constructor( * * @property responseSchema See [GenerationConfig.responseSchema]. * + * @property responseJsonSchema See [GenerationConfig.responseJsonSchema] + * * @property responseModalities See [GenerationConfig.responseModalities]. * * @see [generationConfig] @@ -135,9 +141,40 @@ private constructor( @JvmField public var stopSequences: List? = null @JvmField public var responseMimeType: String? = null @JvmField public var responseSchema: Schema? = null + @JvmField public var responseJsonSchema: JsonSchema<*>? = null @JvmField public var responseModalities: List? = null @JvmField public var thinkingConfig: ThinkingConfig? = null + internal constructor( + temperature: Float? = null, + topK: Int? = null, + topP: Float? = null, + candidateCount: Int? = null, + maxOutputTokens: Int? = null, + presencePenalty: Float? = null, + frequencyPenalty: Float? = null, + stopSequences: List? = null, + responseMimeType: String? = null, + responseSchema: Schema? = null, + responseJsonSchema: JsonSchema<*>? = null, + responseModalities: List? = null, + thinkingConfig: ThinkingConfig? = null, + ) { + this.temperature = temperature + this.topK = topK + this.topP = topP + this.candidateCount = candidateCount + this.maxOutputTokens = maxOutputTokens + this.stopSequences = stopSequences + this.presencePenalty = presencePenalty + this.frequencyPenalty = frequencyPenalty + this.responseMimeType = responseMimeType + this.responseSchema = responseSchema + this.responseJsonSchema = responseJsonSchema + this.responseModalities = responseModalities + this.thinkingConfig = thinkingConfig + } + public fun setTemperature(temperature: Float?): Builder = apply { this.temperature = temperature } @@ -164,6 +201,9 @@ private constructor( public fun setResponseSchema(responseSchema: Schema?): Builder = apply { this.responseSchema = responseSchema } + public fun setResponseSchemaJson(responseSchemaJson: JsonSchema<*>?): Builder = apply { + this.responseJsonSchema = responseSchemaJson + } public fun setResponseModalities(responseModalities: List?): Builder = apply { this.responseModalities = responseModalities } @@ -172,8 +212,11 @@ private constructor( } /** Create a new [GenerationConfig] with the attached arguments. */ - public fun build(): GenerationConfig = - GenerationConfig( + public fun build(): GenerationConfig { + if (responseSchema != null && responseJsonSchema != null) { + throw InvalidStateException("responseSchema and responseJsonSchema are mutually exclusive.") + } + return GenerationConfig( temperature = temperature, topK = topK, topP = topP, @@ -184,11 +227,30 @@ private constructor( frequencyPenalty = frequencyPenalty, responseMimeType = responseMimeType, responseSchema = responseSchema, + responseJsonSchema = responseJsonSchema, responseModalities = responseModalities, thinkingConfig = thinkingConfig ) + } } + public fun toBuilder(): Builder = + Builder( + temperature = temperature, + topK = topK, + topP = topP, + candidateCount = candidateCount, + maxOutputTokens = maxOutputTokens, + stopSequences = stopSequences, + presencePenalty = presencePenalty, + frequencyPenalty = frequencyPenalty, + responseMimeType = responseMimeType, + responseSchema = responseSchema, + responseJsonSchema = responseJsonSchema, + responseModalities = responseModalities, + thinkingConfig = thinkingConfig + ) + internal fun toInternal() = Internal( temperature = temperature, @@ -201,6 +263,7 @@ private constructor( presencePenalty = presencePenalty, responseMimeType = responseMimeType, responseSchema = responseSchema?.toInternalOpenApi(), + responseJsonSchema = responseJsonSchema?.toInternalJson(), responseModalities = responseModalities?.map { it.toInternal() }, thinkingConfig = thinkingConfig?.toInternal() ) @@ -217,6 +280,7 @@ private constructor( @SerialName("presence_penalty") val presencePenalty: Float? = null, @SerialName("frequency_penalty") val frequencyPenalty: Float? = null, @SerialName("response_schema") val responseSchema: Schema.InternalOpenAPI? = null, + @SerialName("response_json_schema") val responseJsonSchema: Schema.InternalJson? = null, @SerialName("response_modalities") val responseModalities: List? = null, @SerialName("thinking_config") val thinkingConfig: ThinkingConfig.Internal? = null ) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/JsonSchema.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/JsonSchema.kt new file mode 100644 index 00000000000..c59bb4a673b --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/JsonSchema.kt @@ -0,0 +1,492 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlin.reflect.KClass +import kotlinx.serialization.json.JsonObject + +/** + * Definition of a data type. + * + * These types can be objects, but also primitives and arrays. Represents a select subset of an + * [JsonSchema object](https://json-schema.org/specification). + * + * **Note:** While optional, including a `description` field in your `JsonSchema` is strongly + * encouraged. The more information the model has about what it's expected to generate, the better + * the results. + */ +public class JsonSchema +internal constructor( + public val type: String, + public val clazz: KClass, + public val description: String? = null, + public val format: String? = null, + public val pattern: String? = null, + public val nullable: Boolean? = null, + public val enum: List? = null, + public val properties: Map>? = null, + public val required: List? = null, + public val items: JsonSchema<*>? = null, + public val title: String? = null, + public val minItems: Int? = null, + public val maxItems: Int? = null, + public val minimum: Double? = null, + public val maximum: Double? = null, + public val anyOf: List>? = null, +) { + + public companion object { + /** + * Returns a [JsonSchema] representing a boolean value. + * + * @param description An optional description of what the boolean should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun boolean( + description: String? = null, + nullable: Boolean = false, + title: String? = null, + ): JsonSchema = + JsonSchema( + description = description, + nullable = nullable, + type = "BOOLEAN", + title = title, + clazz = Boolean::class + ) + + /** + * Returns a [JsonSchema] for a 32-bit signed integer number. + * + * **Important:** This [JsonSchema] provides a hint to the model that it should generate a + * 32-bit integer, but only guarantees that the value will be an integer. Therefore it's + * *possible* that decoding it as an `Int` variable (or `int` in Java) could overflow. + * + * @param description An optional description of what the integer should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmName("numInt") + @JvmOverloads + public fun integer( + description: String? = null, + nullable: Boolean = false, + title: String? = null, + minimum: Double? = null, + maximum: Double? = null, + ): JsonSchema = + JsonSchema( + description = description, + format = "int32", + nullable = nullable, + type = "INTEGER", + title = title, + minimum = minimum, + maximum = maximum, + clazz = Integer::class + ) + + /** + * Returns a [JsonSchema] for a 64-bit signed integer number. + * + * @param description An optional description of what the number should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmName("numLong") + @JvmOverloads + public fun long( + description: String? = null, + nullable: Boolean = false, + title: String? = null, + minimum: Double? = null, + maximum: Double? = null, + ): JsonSchema = + JsonSchema( + description = description, + nullable = nullable, + type = "INTEGER", + title = title, + minimum = minimum, + maximum = maximum, + clazz = Long::class + ) + + /** + * Returns a [JsonSchema] for a double-precision floating-point number. + * + * @param description An optional description of what the number should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmName("numDouble") + @JvmOverloads + public fun double( + description: String? = null, + nullable: Boolean = false, + title: String? = null, + minimum: Double? = null, + maximum: Double? = null, + ): JsonSchema = + JsonSchema( + description = description, + nullable = nullable, + type = "NUMBER", + title = title, + minimum = minimum, + maximum = maximum, + clazz = Double::class + ) + + /** + * Returns a [JsonSchema] for a single-precision floating-point number. + * + * **Important:** This [JsonSchema] provides a hint to the model that it should generate a + * single-precision floating-point number, but only guarantees that the value will be a number. + * Therefore it's *possible* that decoding it as a `Float` variable (or `float` in Java) could + * overflow. + * + * @param description An optional description of what the number should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmName("numFloat") + @JvmOverloads + public fun float( + description: String? = null, + nullable: Boolean = false, + title: String? = null, + minimum: Double? = null, + maximum: Double? = null, + ): JsonSchema = + JsonSchema( + description = description, + nullable = nullable, + type = "NUMBER", + format = "float", + title = title, + minimum = minimum, + maximum = maximum, + clazz = Float::class + ) + + /** + * Returns a [JsonSchema] for a string. + * + * @param description An optional description of what the string should contain or represent. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + * @param format An optional pattern that values need to adhere to. + */ + @JvmStatic + @JvmName("str") + @JvmOverloads + public fun string( + description: String? = null, + nullable: Boolean = false, + format: StringFormat? = null, + title: String? = null, + ): JsonSchema = + JsonSchema( + description = description, + format = format?.value, + nullable = nullable, + type = "STRING", + title = title, + clazz = String::class + ) + + /** + * Returns a [JsonSchema] for a complex data type. + * + * This schema instructs the model to produce data of type object, which has keys of type + * `String` and values of type [JsonSchema]. + * + * **Example:** A `city` could be represented with the following object `JsonSchema`. + * + * ``` + * JsonSchema.obj(mapOf( + * "name" to JsonSchema.string(), + * "population" to JsonSchema.integer() + * )) + * ``` + * + * @param properties The map of the object's property names to their [JsonSchema]s. + * @param optionalProperties The list of optional properties. They must correspond to the keys + * provided in the `properties` map. By default it's empty, signaling the model that all + * properties are to be included. + * @param description An optional description of what the object represents. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun obj( + properties: Map>, + optionalProperties: List = emptyList(), + description: String? = null, + nullable: Boolean = false, + title: String? = null, + ): JsonSchema { + if (!properties.keys.containsAll(optionalProperties)) { + throw IllegalArgumentException( + "All optional properties must be present in properties. Missing: ${optionalProperties.minus(properties.keys)}" + ) + } + return JsonSchema( + description = description, + nullable = nullable, + properties = properties, + required = properties.keys.minus(optionalProperties.toSet()).toList(), + type = "OBJECT", + title = title, + clazz = JsonObject::class + ) + } + + /** + * Returns a [JsonSchema] for a complex data type. + * + * This schema instructs the model to produce data of type object, which has keys of type + * `String` and values of type [JsonSchema]. + * + * **Example:** A `city` could be represented with the following object `JsonSchema`. + * + * ``` + * JsonSchema.obj(mapOf( + * "name" to JsonSchema.string(), + * "population" to JsonSchema.integer() + * ), + * City::class + * ) + * ``` + * + * @param properties The map of the object's property names to their [JsonSchema]s. + * @param clazz the real class that this schema represents + * @param optionalProperties The list of optional properties. They must correspond to the keys + * provided in the `properties` map. By default it's empty, signaling the model that all + * properties are to be included. + * @param description An optional description of what the object represents. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun obj( + properties: Map>, + clazz: KClass, + optionalProperties: List = emptyList(), + description: String? = null, + nullable: Boolean = false, + title: String? = null, + ): JsonSchema { + if (!properties.keys.containsAll(optionalProperties)) { + throw IllegalArgumentException( + "All optional properties must be present in properties. Missing: ${optionalProperties.minus(properties.keys)}" + ) + } + return JsonSchema( + description = description, + nullable = nullable, + properties = properties, + required = properties.keys.minus(optionalProperties.toSet()).toList(), + type = "OBJECT", + title = title, + clazz = clazz + ) + } + + /** + * Returns a [JsonSchema] for an array. + * + * @param items The [JsonSchema] of the elements stored in the array. + * @param description An optional description of what the array represents. + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun array( + items: JsonSchema<*>, + description: String? = null, + nullable: Boolean = false, + title: String? = null, + minItems: Int? = null, + maxItems: Int? = null, + ): JsonSchema> = + JsonSchema( + description = description, + nullable = nullable, + items = items, + type = "ARRAY", + title = title, + minItems = minItems, + maxItems = maxItems, + clazz = List::class + ) + + /** + * Returns a [JsonSchema] for an enumeration. + * + * For example, the cardinal directions can be represented as: + * ``` + * JsonSchema.enumeration(listOf("north", "east", "south", "west"), "Cardinal directions") + * ``` + * + * @param values The list of valid values for this enumeration + * @param description The description of what the parameter should contain or represent + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun enumeration( + values: List, + description: String? = null, + nullable: Boolean = false, + title: String? = null, + ): JsonSchema = + JsonSchema( + description = description, + format = "enum", + nullable = nullable, + enum = values, + type = "STRING", + title = title, + clazz = String::class + ) + + /** + * Returns a [JsonSchema] for an enumeration. + * + * For example, the cardinal directions can be represented as: + * ``` + * JsonSchema.enumeration( + * listOf("north", "east", "south", "west"), + * Direction::class, + * "Cardinal directions" + * ) + * ``` + * + * @param values The list of valid values for this enumeration + * @param clazz the real class that this schema represents + * @param description The description of what the parameter should contain or represent + * @param nullable Indicates whether the value can be `null`. Defaults to `false`. + */ + @JvmStatic + @JvmOverloads + public fun enumeration( + values: List, + clazz: KClass, + description: String? = null, + nullable: Boolean = false, + title: String? = null, + ): JsonSchema = + JsonSchema( + description = description, + format = "enum", + nullable = nullable, + enum = values, + type = "STRING", + title = title, + clazz = clazz + ) + + /** + * Returns a [JsonSchema] representing a value that must conform to *any* (one of) the provided + * sub-schema. + * + * Example: A field that can hold either a simple userID or a more detailed user object. + * + * ``` + * JsonSchema.anyOf( listOf( JsonSchema.integer(description = "User ID"), JsonSchema.obj( mapOf( + * "userID" to JsonSchema.integer(description = "User ID"), + * "username" to JsonSchema.string(description = "Username") + * ))) + * ``` + * + * @param schemas The list of valid schemas which could be here + */ + @JvmStatic + public fun anyOf(schemas: List>): JsonSchema = + JsonSchema(type = "ANYOF", anyOf = schemas, clazz = String::class) + } + + internal fun toInternalJson(): Schema.InternalJson { + val outType = + if (type == "ANYOF" || (type == "STRING" && format == "enum")) { + null + } else { + type.lowercase() + } + + val (outMinimum, outMaximum) = + if (outType == "integer" && format == "int32") { + (minimum ?: Integer.MIN_VALUE.toDouble()) to (maximum ?: Integer.MAX_VALUE.toDouble()) + } else { + minimum to maximum + } + + val outFormat = + if ( + (outType == "integer" && format == "int32") || + (outType == "number" && format == "float") || + format == "enum" + ) { + null + } else { + format + } + + if (nullable == true) { + return Schema.InternalJsonNullable( + outType?.let { listOf(it, "null") }, + description, + outFormat, + pattern, + enum?.let { + buildList { + addAll(it) + add("null") + } + }, + properties?.mapValues { it.value.toInternalJson() }, + required, + items?.toInternalJson(), + title, + minItems, + maxItems, + outMinimum, + outMaximum, + anyOf?.map { it.toInternalJson() }, + ) + } + return Schema.InternalJsonNonNull( + outType, + description, + outFormat, + pattern, + enum, + properties?.mapValues { it.value.toInternalJson() }, + required, + items?.toInternalJson(), + title, + minItems, + maxItems, + outMinimum, + outMaximum, + anyOf?.map { it.toInternalJson() }, + ) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Schema.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Schema.kt index 1dfa4ddecb0..9f728adbbd4 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Schema.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Schema.kt @@ -378,6 +378,7 @@ internal constructor( outType?.let { listOf(it, "null") }, description, outFormat, + null, enum?.let { buildList { addAll(it) @@ -399,6 +400,7 @@ internal constructor( outType, description, outFormat, + null, enum, properties?.mapValues { it.value.toInternalJson() }, required, @@ -437,6 +439,7 @@ internal constructor( val type: String? = null, val description: String? = null, val format: String? = null, + val pattern: String? = null, val enum: List? = null, val properties: Map? = null, val required: List? = null, @@ -454,6 +457,7 @@ internal constructor( val type: List? = null, val description: String? = null, val format: String? = null, + val pattern: String? = null, val enum: List? = null, val properties: Map? = null, val required: List? = null, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt index 43a66a10d62..ad8aeb40b07 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt @@ -27,6 +27,7 @@ public class Tool @OptIn(PublicPreviewAPI::class) internal constructor( internal val functionDeclarations: List?, + internal val autoFunctionDeclarations: List>?, internal val googleSearch: GoogleSearch?, internal val codeExecution: JsonObject?, @property:PublicPreviewAPI internal val urlContext: UrlContext?, @@ -35,7 +36,10 @@ internal constructor( @OptIn(PublicPreviewAPI::class) internal fun toInternal() = Internal( - functionDeclarations?.map { it.toInternal() } ?: emptyList(), + buildList { + functionDeclarations?.let { addAll(it.map { it.toInternal() }) } + autoFunctionDeclarations?.let { addAll(it.map { it.toInternal() }) } + }, googleSearch = this.googleSearch?.toInternal(), codeExecution = this.codeExecution, urlContext = this.urlContext?.toInternal() @@ -53,16 +57,36 @@ internal constructor( public companion object { @OptIn(PublicPreviewAPI::class) - private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap()), null) } + private val codeExecutionInstance by lazy { + Tool(null, null, null, JsonObject(emptyMap()), null) + } + + /** + * Creates a [Tool] instance that provides the model with access to the [functionDeclarations]. + * + * @param functionDeclarations The list of functions that this tool allows the model access to. + */ + @JvmStatic + public fun functionDeclarations( + functionDeclarations: List? = null, + ): Tool { + @OptIn(PublicPreviewAPI::class) return Tool(functionDeclarations, null, null, null, null) + } /** * Creates a [Tool] instance that provides the model with access to the [functionDeclarations]. * * @param functionDeclarations The list of functions that this tool allows the model access to. + * @param autoFunctionDeclarations The list of functions that this tool has access to which + * should be executed automatically */ @JvmStatic - public fun functionDeclarations(functionDeclarations: List): Tool { - @OptIn(PublicPreviewAPI::class) return Tool(functionDeclarations, null, null, null) + public fun functionDeclarations( + functionDeclarations: List? = null, + autoFunctionDeclarations: List>? + ): Tool { + @OptIn(PublicPreviewAPI::class) + return Tool(functionDeclarations, autoFunctionDeclarations, null, null, null) } /** Creates a [Tool] instance that allows the model to use code execution. */ @@ -82,7 +106,7 @@ internal constructor( @PublicPreviewAPI @JvmStatic public fun urlContext(urlContext: UrlContext = UrlContext()): Tool { - return Tool(null, null, null, urlContext) + return Tool(null, null, null, null, urlContext) } /** @@ -103,7 +127,7 @@ internal constructor( */ @JvmStatic public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool { - @OptIn(PublicPreviewAPI::class) return Tool(null, googleSearch, null, null) + @OptIn(PublicPreviewAPI::class) return Tool(null, null, googleSearch, null, null) } } } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/type/FunctionDeclarationTest.kt b/firebase-ai/src/test/java/com/google/firebase/ai/type/FunctionDeclarationTest.kt index 7719044b498..575dff886c7 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/type/FunctionDeclarationTest.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/type/FunctionDeclarationTest.kt @@ -48,7 +48,9 @@ internal class FunctionDeclarationTest { "required": [ "userID" ] - } + }, + "parametersJsonSchema": null, + "responseJsonSchema": null } """ .trimIndent() @@ -90,7 +92,9 @@ internal class FunctionDeclarationTest { "required": [ "userID" ] - } + }, + "parametersJsonSchema": null, + "responseJsonSchema": null } """ .trimIndent() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index be6fe853be7..9209a59f27f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,6 +45,7 @@ jsonassert = "1.5.0" kotest = "5.9.0" # Do not use 5.9.1 because it reverts the fix for https://github.com/kotest/kotest/issues/3981 kotestAssertionsCore = "5.8.1" kotlin = "2.0.21" +kotlinpoetKsp = "2.2.0" ktorVersion = "3.0.3" legacySupportV4 = "1.0.0" lifecycleProcess = "2.3.1" @@ -69,6 +70,7 @@ rxjava = "2.2.21" serialization = "1.7.3" slf4jNop = "2.0.17" spotless = "7.0.4" +symbolProcessingApi = "2.2.10-2.0.2" testServices = "1.6.0" truth = "1.4.5" truthProtoExtension = "1.0" @@ -142,6 +144,7 @@ kotlin-bom = { module = "org.jetbrains.kotlin:kotlin-bom", version.ref = "kotlin kotlin-coroutines-tasks = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-play-services", version.ref = "coroutines" } kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" } kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } +kotlinpoet-ksp = { module = "com.squareup:kotlinpoet-ksp", version.ref = "kotlinpoetKsp" } kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref = "coroutines" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-reactive = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-reactive", version.ref = "coroutines" } @@ -203,6 +206,7 @@ rxandroid = { module = "io.reactivex.rxjava2:rxandroid", version.ref = "rxandroi rxjava = { module = "io.reactivex.rxjava2:rxjava", version.ref = "rxjava" } slf4j-nop = { module = "org.slf4j:slf4j-nop", version.ref = "slf4jNop" } spotless-plugin-gradle = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } +symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "symbolProcessingApi" } truth = { module = "com.google.truth:truth", version.ref = "truth" } truth-liteproto-extension = { module = "com.google.truth.extensions:truth-liteproto-extension", version.ref = "truth" } truth-proto-extension = { module = "com.google.truth.extensions:truth-proto-extension", version.ref = "truthProtoExtension" } diff --git a/subprojects.cfg b/subprojects.cfg index f8505ecf8e7..a167e6eb58c 100644 --- a/subprojects.cfg +++ b/subprojects.cfg @@ -74,3 +74,5 @@ transport:transport-runtime-testing # sdk #firebase-storage:test-app #appcheck:firebase-appcheck:test-app #firebase-appdistribution:test-app + +firebase-ai-ksp-processor # buildtools