From 79e8f1911f3fa1d2a0ed5f83611aa92f029ba996 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:04:43 +0000 Subject: [PATCH] Extend resolution search schema and agent system prompt for AI-generated contextual labels and semantic GeoJSON annotations. - Updated \`resolutionSearchSchema\` to include \`mapboxImageLabel\`, \`googleImageLabel\`, and \`analysisFocus\`. - Enhanced GeoJSON features with \`featureCategory\` and \`displayLabel\`. - Updated agent system prompt to support dynamic labeling and semantic map annotations. - Wired dynamic labels through \`ResolutionCarousel\` and \`CompareSlider\` components. - Integrated detected features context into \`querySuggestor\` for improved follow-up queries. Co-authored-by: ngoiyaeric <115367894+ngoiyaeric@users.noreply.github.com> --- app/actions.tsx | 22 +++++++++++++++++++--- components/compare-slider.tsx | 12 +++++++----- components/resolution-carousel.tsx | 29 ++++++++++++++++++++++++----- lib/agents/query-suggestor.tsx | 24 +++++++++++++++--------- lib/agents/resolution-search.tsx | 4 ++++ lib/schema/resolution-search.ts | 10 +++++++++- 6 files changed, 78 insertions(+), 23 deletions(-) diff --git a/app/actions.tsx b/app/actions.tsx index 8b693603..784d3b0b 100644 --- a/app/actions.tsx +++ b/app/actions.tsx @@ -141,7 +141,9 @@ async function submit(formData?: FormData, skip?: boolean) { }, properties: { name: f.name, - description: f.description + description: f.description, + featureCategory: f.featureCategory, + displayLabel: f.displayLabel } })) }; @@ -180,7 +182,9 @@ async function submit(formData?: FormData, skip?: boolean) { } return m }); - const relatedQueries = await querySuggestor(uiStream, sanitizedMessages); + const detectedFeatures = analysisResult.geoJson?.features?.map((f: any) => `${f.name} (${f.featureCategory || "other"})`).join(", "); + const detectedFeaturesSummary = detectedFeatures ? `Detected: ${detectedFeatures}` : undefined; + const relatedQueries = await querySuggestor(uiStream, sanitizedMessages, detectedFeaturesSummary); uiStream.append(
@@ -207,7 +211,10 @@ async function submit(formData?: FormData, skip?: boolean) { geoJson: geoJson, // Use reconstructed GeoJSON for storage/UI image: dataUrl, mapboxImage: mapboxDataUrl, - googleImage: googleDataUrl + googleImage: googleDataUrl, + mapboxImageLabel: analysisResult.mapboxImageLabel, + googleImageLabel: analysisResult.googleImageLabel, + analysisFocus: analysisResult.analysisFocus }), type: 'resolution_search_result' }, @@ -242,6 +249,9 @@ async function submit(formData?: FormData, skip?: boolean) { mapboxImage={mapboxDataUrl || undefined} googleImage={googleDataUrl || undefined} initialImage={dataUrl} + mapboxImageLabel={undefined} + googleImageLabel={undefined} + analysisFocus={undefined} />
@@ -836,6 +846,9 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => { const image = analysisResult.image as string; const mapboxImage = analysisResult.mapboxImage as string; const googleImage = analysisResult.googleImage as string; + const mapboxImageLabel = analysisResult.mapboxImageLabel as string; + const googleImageLabel = analysisResult.googleImageLabel as string; + const analysisFocus = analysisResult.analysisFocus as string; return { id, @@ -845,6 +858,9 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => { mapboxImage={mapboxImage} googleImage={googleImage} initialImage={image} + mapboxImageLabel={mapboxImageLabel} + googleImageLabel={googleImageLabel} + analysisFocus={analysisFocus} /> {geoJson && ( diff --git a/components/compare-slider.tsx b/components/compare-slider.tsx index 34877148..944019bc 100644 --- a/components/compare-slider.tsx +++ b/components/compare-slider.tsx @@ -7,9 +7,11 @@ interface CompareSliderProps { leftImage: string rightImage: string className?: string + leftLabel?: string + rightLabel?: string } -export function CompareSlider({ leftImage, rightImage, className }: CompareSliderProps) { +export function CompareSlider({ leftImage, rightImage, className, leftLabel, rightLabel }: CompareSliderProps) { const [sliderPosition, setSliderPosition] = useState(50) const containerRef = useRef(null) const [containerWidth, setContainerWidth] = useState(0) @@ -80,11 +82,11 @@ export function CompareSlider({ leftImage, rightImage, className }: CompareSlide {/* Labels */} -
- MAPBOX +
+ {leftLabel || 'MAPBOX'}
-
- GOOGLE SATELLITE +
+ {rightLabel || 'GOOGLE SATELLITE'}
) diff --git a/components/resolution-carousel.tsx b/components/resolution-carousel.tsx index 1b28cbf7..2f9ec611 100644 --- a/components/resolution-carousel.tsx +++ b/components/resolution-carousel.tsx @@ -22,9 +22,19 @@ interface ResolutionCarouselProps { mapboxImage?: string | null googleImage?: string | null initialImage?: string | null + mapboxImageLabel?: string + googleImageLabel?: string + analysisFocus?: string } -export function ResolutionCarousel({ mapboxImage, googleImage, initialImage }: ResolutionCarouselProps) { +export function ResolutionCarousel({ + mapboxImage, + googleImage, + initialImage, + mapboxImageLabel, + googleImageLabel, + analysisFocus +}: ResolutionCarouselProps) { const actions = useActions() as any const [, setMessages] = useUIState() const [isAnalyzing, setIsAnalyzing] = React.useState(false) @@ -75,12 +85,12 @@ export function ResolutionCarousel({ mapboxImage, googleImage, initialImage }: R } // Individual slides - if (mapboxImage) slides.push({ type: 'image', src: mapboxImage, showAnalysis: false, label: 'MAPBOX' }) - if (googleImage) slides.push({ type: 'image', src: googleImage, showAnalysis: true, label: 'GOOGLE SATELLITE' }) + if (mapboxImage) slides.push({ type: 'image', src: mapboxImage, showAnalysis: false, label: mapboxImageLabel || 'MAPBOX' }) + if (googleImage) slides.push({ type: 'image', src: googleImage, showAnalysis: true, label: googleImageLabel || 'GOOGLE SATELLITE' }) // Fallback if (slides.length === 0 && initialImage) { - slides.push({ type: 'image', src: initialImage, showAnalysis: false, label: 'MAP CAPTURE' }) + slides.push({ type: 'image', src: initialImage, showAnalysis: false, label: analysisFocus || 'MAP CAPTURE' }) } if (slides.length === 0) return null @@ -102,6 +112,9 @@ export function ResolutionCarousel({ mapboxImage, googleImage, initialImage }: R {isAnalyzing ? 'ANALYZING...' : 'QCX-TERRA ANALYSIS'} )} +
+ {item.label} +
) } @@ -115,7 +128,13 @@ export function ResolutionCarousel({ mapboxImage, googleImage, initialImage }: R
{slide.type === 'compare' ? ( - + ) : ( <> diff --git a/lib/agents/query-suggestor.tsx b/lib/agents/query-suggestor.tsx index 7cb8e50c..8c39bc1a 100644 --- a/lib/agents/query-suggestor.tsx +++ b/lib/agents/query-suggestor.tsx @@ -14,23 +14,27 @@ interface CacheEntry { const queryCache = new Map(); const CACHE_TTL = 5 * 60 * 1000; // 5 minutes -function getCacheKey(messages: CoreMessage[]): string { - // Create a simple hash of the last few messages to use as cache key +function getCacheKey(messages: CoreMessage[], detectedFeatures?: string): string { + // Create a simple hash of the last few messages and detected features to use as cache key const recentMessages = messages.slice(-3); - return JSON.stringify(recentMessages.map(m => ({ - role: m.role, - content: typeof m.content === 'string' ? m.content : '[complex content]' - }))); + return JSON.stringify({ + messages: recentMessages.map(m => ({ + role: m.role, + content: typeof m.content === 'string' ? m.content : '[complex content]' + })), + detectedFeatures + }); } export async function querySuggestor( uiStream: ReturnType, - messages: CoreMessage[] + messages: CoreMessage[], + detectedFeatures?: string ) { const objectStream = createStreamableValue() // OPTIMIZATION: Check cache first - const cacheKey = getCacheKey(messages); + const cacheKey = getCacheKey(messages, detectedFeatures); const cachedEntry = queryCache.get(cacheKey) as CacheEntry | undefined; if (cachedEntry && Date.now() - cachedEntry.timestamp < CACHE_TTL) { @@ -54,10 +58,12 @@ export async function querySuggestor( let finalRelatedQueries: PartialRelated = {} + const systemPrompt = `Generate 3 follow-up queries that explore the subject matter deeper. Format as JSON with an "items" array containing objects with "query" fields. Keep queries concise and relevant.${detectedFeatures ? `\n\nContext - Detected Features: ${detectedFeatures}. Incorporate these features into the suggested queries (e.g., "Tell me more about [Feature Name]" or "What is the significance of [Feature Name]?").` : ''}`; + // OPTIMIZATION: Use a more concise system prompt to reduce token usage const result = await streamObject({ model: (await getModel()) as LanguageModel, - system: `Generate 3 follow-up queries that explore the subject matter deeper. Format as JSON with an "items" array containing objects with "query" fields. Keep queries concise and relevant.`, + system: systemPrompt, messages, schema: relatedSchema, temperature: 0.7, // Lower temperature for more consistent results diff --git a/lib/agents/resolution-search.tsx b/lib/agents/resolution-search.tsx index 9985712d..04946bd4 100644 --- a/lib/agents/resolution-search.tsx +++ b/lib/agents/resolution-search.tsx @@ -128,9 +128,13 @@ Use these user-drawn areas/lines as primary areas of interest for your analysis. 5. **COG Applicability:** Determine if this location would benefit from Cloud Optimized GeoTIFF (COG) analysis for high-precision temporal or spectral data. 6. **News Integration:** Reference any recent news or events that may be relevant to the current state of the location. 7. **Structured Output:** Return your findings in a structured JSON format including summary, geoJson (if any), news context, and any extracted coordinates or COG information. Use the provided schema. +8. **Contextual Labeling:** Produce location-specific and feature-aware labels for the images (\`mapboxImageLabel\`, \`googleImageLabel\`, \`analysisFocus\`). Reference user-drawn features in these labels when they are present. Labels should be concise, uppercase-friendly, and descriptive of the analysis focus. Examples: "Analysis of drawn area: [feature type]", "[Location name] satellite view". +9. **Semantic Annotations:** When creating \`geoJson\` features, use Point features for POIs and landmarks, and Polygon features for land areas or regions of interest. Each feature must have a \`name\`, \`featureCategory\`, and \`displayLabel\` matching findings in your summary. Example: Create a Point for identified bridges with name 'Main Street Bridge', category 'infrastructure'. Your analysis should be based on the visual information in the image, the temporal context provided, and your general knowledge. Do not attempt to access external websites or perform web searches beyond what has been provided. +In your summary, reference annotated features by name (e.g., "I've marked the identified commercial district as a polygon", "The red marker indicates the location of..."). + Analyze the user's prompt and the image to provide a holistic understanding of the location with full temporal and contextual awareness. `; diff --git a/lib/schema/resolution-search.ts b/lib/schema/resolution-search.ts index e7908abd..5df63046 100644 --- a/lib/schema/resolution-search.ts +++ b/lib/schema/resolution-search.ts @@ -19,7 +19,11 @@ export const resolutionSearchSchema = z.object({ .or(z.array(z.array(z.array(z.number())))) .describe('Coordinates for the geometry'), name: z.string().describe('Name of the feature or point of interest'), - description: z.string().optional().describe('Description of the feature') + description: z.string().optional().describe('Description of the feature'), + featureCategory: z.enum(['poi', 'land_feature', 'infrastructure', 'drawn_area', 'other']) + .optional() + .describe('The category of the feature. Use poi for landmarks, land_feature for natural elements, infrastructure for man-made structures, and drawn_area for user-defined regions.'), + displayLabel: z.string().optional().describe('A short label for map display or tooltips') })) }).optional().describe('A collection of points of interest and classified land features to be overlaid on the map.'), @@ -27,6 +31,10 @@ export const resolutionSearchSchema = z.object({ extractedLatitude: z.number().optional().describe('The extracted latitude of the center of the image.'), extractedLongitude: z.number().optional().describe('The extracted longitude of the center of the image.'), + mapboxImageLabel: z.string().optional().describe('A location-specific label for the Mapbox image (e.g., "DOWNTOWN SAN FRANCISCO").'), + googleImageLabel: z.string().optional().describe('A location-specific label for the Google Satellite image (e.g., "SATELLITE VIEW - MISSION DISTRICT").'), + analysisFocus: z.string().optional().describe('A description of the overall analysis focus or user-drawn feature (e.g., "ANALYSIS OF COASTAL EROSION").'), + cogApplicable: z.boolean().optional().describe('Whether Cloud Optimized GeoTIFF (COG) data is applicable for this area.'), cogDescription: z.string().optional().describe('Description of COG data availability or benefits.'),