From a42d00b3f704594344f0a4cabf2eb1aa2bfd6eb9 Mon Sep 17 00:00:00 2001 From: rosariopf Date: Wed, 8 May 2024 17:02:27 +0100 Subject: [PATCH 1/2] chore(vertexai): add storage sdk --- gradle/libs.versions.toml | 2 ++ vertexai/app/build.gradle.kts | 1 + 2 files changed, 3 insertions(+) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 222fddd5c..583551640 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,6 @@ [versions] agp = "8.4.0" +firebaseStorage = "21.0.0" guava = "32.0.1-android" kotlin = "1.9.23" coreKtx = "1.13.1" @@ -14,6 +15,7 @@ vertexAI = "16.0.0-alpha03" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } +firebase-storage = { module = "com.google.firebase:firebase-storage", version.ref = "firebaseStorage" } guava = { module = "com.google.guava:guava", version.ref = "guava" } junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } diff --git a/vertexai/app/build.gradle.kts b/vertexai/app/build.gradle.kts index 4b50b6a0f..4598e5161 100644 --- a/vertexai/app/build.gradle.kts +++ b/vertexai/app/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { implementation(libs.androidx.material3) implementation(libs.firebase.vertex.ai) + implementation(libs.firebase.storage) // Required for one-shot operations (to use `ListenableFuture` from Reactive Streams) implementation(libs.reactive.streams) From 577248a2ddb36afe5b9af28001b6269d7ce554f7 Mon Sep 17 00:00:00 2001 From: rosariopf Date: Wed, 8 May 2024 18:13:29 +0100 Subject: [PATCH 2/2] feat(vertexai): add storage snippets --- .../example/vertexai/java/FilesViewModel.java | 92 +++++++++++++++++++ .../example/vertexai/kotlin/FilesViewModel.kt | 66 +++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 vertexai/app/src/main/java/com/google/firebase/example/vertexai/java/FilesViewModel.java create mode 100644 vertexai/app/src/main/java/com/google/firebase/example/vertexai/kotlin/FilesViewModel.kt diff --git a/vertexai/app/src/main/java/com/google/firebase/example/vertexai/java/FilesViewModel.java b/vertexai/app/src/main/java/com/google/firebase/example/vertexai/java/FilesViewModel.java new file mode 100644 index 000000000..bfdc12114 --- /dev/null +++ b/vertexai/app/src/main/java/com/google/firebase/example/vertexai/java/FilesViewModel.java @@ -0,0 +1,92 @@ +package com.google.firebase.example.vertexai.java; + +import android.net.Uri; + +import androidx.annotation.NonNull; +import androidx.lifecycle.ViewModel; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.firebase.storage.FirebaseStorage; +import com.google.firebase.storage.StorageReference; +import com.google.firebase.vertexai.FirebaseVertexAI; +import com.google.firebase.vertexai.GenerativeModel; +import com.google.firebase.vertexai.java.GenerativeModelFutures; +import com.google.firebase.vertexai.type.Content; +import com.google.firebase.vertexai.type.GenerateContentResponse; + +import java.io.File; +import java.util.concurrent.Executor; + +public class FilesViewModel extends ViewModel { + GenerativeModel generativeModel = FirebaseVertexAI.getInstance() + .generativeModel("gemini-1.5-pro-preview-0409"); + GenerativeModelFutures model = GenerativeModelFutures.from(generativeModel); + + void uploadAndUseReference(Executor executor) { + // [START vertexai_storage_upload] + // Upload raw data to storage reference + StorageReference storage = FirebaseStorage.getInstance().getReference(); + Uri fileUri = Uri.fromFile(new File("images/image.jpg")); + + storage.putFile(fileUri).addOnSuccessListener(taskSnapshot -> { + // Get the MIME type and Cloud Storage URL + String mimeType = taskSnapshot.getMetadata().getContentType(); + String bucket = taskSnapshot.getMetadata().getBucket(); + String filePath = taskSnapshot.getMetadata().getPath(); + + if (mimeType != null && bucket != null) { + String storageUrl = "gs://" + bucket + "/" + filePath; + Content prompt = new Content.Builder() + .addFilePart(mimeType, storageUrl) + .addText("What's in this picture?") + .build(); + + // To generate text output, call generateContent with the prompt + ListenableFuture response = model.generateContent(prompt); + Futures.addCallback(response, new FutureCallback<>() { + @Override + public void onSuccess(GenerateContentResponse result) { + String resultText = result.getText(); + System.out.println(resultText); + } + + @Override + public void onFailure(@NonNull Throwable t) { + t.printStackTrace(); + } + }, executor); + } + }).addOnFailureListener(e -> { + // An error occurred while uploading the file + e.printStackTrace(); + }); + // [END vertexai_storage_upload] + } + + void useObjectReference(Executor executor) { + // [START vertexai_storage_ref] + // Explicitly include the MIME type and Cloud Storage URL + Content prompt = new Content.Builder() + .addFilePart("image/jpeg", "gs://bucket-name/path/image.jpg") + .addText("What's in this picture?") + .build(); + + // To generate text output, call generateContent with the prompt + ListenableFuture response = model.generateContent(prompt); + Futures.addCallback(response, new FutureCallback<>() { + @Override + public void onSuccess(GenerateContentResponse result) { + String resultText = result.getText(); + System.out.println(resultText); + } + + @Override + public void onFailure(@NonNull Throwable t) { + t.printStackTrace(); + } + }, executor); + // [END vertexai_storage_ref] + } +} diff --git a/vertexai/app/src/main/java/com/google/firebase/example/vertexai/kotlin/FilesViewModel.kt b/vertexai/app/src/main/java/com/google/firebase/example/vertexai/kotlin/FilesViewModel.kt new file mode 100644 index 000000000..9b19100c2 --- /dev/null +++ b/vertexai/app/src/main/java/com/google/firebase/example/vertexai/kotlin/FilesViewModel.kt @@ -0,0 +1,66 @@ +package com.google.firebase.example.vertexai.kotlin + +import android.net.Uri +import android.util.Log +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.google.firebase.Firebase +import com.google.firebase.storage.StorageException +import com.google.firebase.storage.storage +import com.google.firebase.vertexai.type.GoogleGenerativeAIException +import com.google.firebase.vertexai.type.content +import com.google.firebase.vertexai.vertexAI +import java.io.File +import kotlinx.coroutines.launch +import kotlinx.coroutines.tasks.await + +class FilesViewModel : ViewModel() { + private val TAG = "FilesViewModel" + private val generativeModel = + Firebase.vertexAI.generativeModel("gemini-1.5-pro-preview-0409") + + fun uploadAndUseReference() { + viewModelScope.launch { + // [START vertexai_storage_upload] + // Upload raw data to storage reference + val storageRef = Firebase.storage.reference + val fileUri = Uri.fromFile(File("images/image.jpg")) + try { + val taskSnapshot = storageRef.putFile(fileUri).await() + + // Get the MIME type and Cloud Storage URL + val mimeType = taskSnapshot.metadata?.contentType + val bucket = taskSnapshot.metadata?.bucket + val filePath = taskSnapshot.metadata?.path + + if (mimeType != null && bucket != null) { + val storageUrl = "gs://$bucket/$filePath" + val prompt = content { + fileData(mimeType, storageUrl) + text("What's in this picture?") + } + val response = generativeModel.generateContent(prompt) + Log.d(TAG, response.text) + } + } catch (e: StorageException) { + // An error occurred while uploading the file + } catch (e: GoogleGenerativeAIException) { + // An error occurred while generating text + } + // [END vertexai_storage_upload] + } + } + + fun useObjectReference() { + viewModelScope.launch { + // [START vertexai_storage_ref] + val prompt = content { + fileData("image/jpeg", "gs://bucket-name/path/image.jpg") + text("What's in this picture?") + } + val response = generativeModel.generateContent(prompt) + Log.d(TAG, response.text) + // [END vertexai_storage_ref] + } + } +} \ No newline at end of file