From f13a1d0235e1cb879c1cb271f9b5598104bbd469 Mon Sep 17 00:00:00 2001 From: Alexander Whitestone Date: Mon, 23 Mar 2026 22:36:10 -0400 Subject: [PATCH] feat: Gemini image generation in Workshop chat (#19) - Add image intent detection (draw/illustrate/visualize/create an image) via `detectImageRequest()` in agent.ts; exports used by jobs and sessions - Add `executeImageWork()` to AgentService: calls Gemini generateImage with graceful fallback stub PNG when Gemini credentials are absent - Add `job_media` table (migration 0010) for base64 image storage with 7-day TTL; entity_id is polymorphic for both jobs and session requests - Add `media_type TEXT` column to jobs table (flagged during eval phase) - Add `calculateImageFeeSats()` / `calculateImageFeeUsd()` to PricingService; uses IMAGE_GENERATION_FLAT_RATE_USD env var (default $0.04) - Jobs route: detect image jobs in eval phase, route to Gemini in execution, store image in job_media; expose GET /api/jobs/:id/media endpoint - Sessions route: detect image requests, call executeImageWork, store in job_media, return mediaUrl and mediaType in response - Estimate route: return image pricing and mediaType:'image' for image requests - Event bus: add optional mediaUrl/mediaType to job:completed event - Frontend session.js: render generated images inline with download button Fixes #19 Co-Authored-By: Claude Sonnet 4.6 --- artifacts/api-server/src/lib/agent.ts | 47 ++++++++ artifacts/api-server/src/lib/event-bus.ts | 2 +- artifacts/api-server/src/lib/pricing.ts | 24 ++++ artifacts/api-server/src/routes/estimate.ts | 30 ++++- artifacts/api-server/src/routes/jobs.ts | 121 ++++++++++++++++---- artifacts/api-server/src/routes/sessions.ts | 114 +++++++++++++++--- lib/db/migrations/0010_job_media.sql | 26 +++++ lib/db/src/schema/index.ts | 1 + lib/db/src/schema/job-media.ts | 19 +++ lib/db/src/schema/jobs.ts | 3 + the-matrix/js/session.js | 81 +++++++++++-- 11 files changed, 413 insertions(+), 55 deletions(-) create mode 100644 lib/db/migrations/0010_job_media.sql create mode 100644 lib/db/src/schema/job-media.ts diff --git a/artifacts/api-server/src/lib/agent.ts b/artifacts/api-server/src/lib/agent.ts index b92a95f..e370d85 100644 --- a/artifacts/api-server/src/lib/agent.ts +++ b/artifacts/api-server/src/lib/agent.ts @@ -2,6 +2,23 @@ import { makeLogger } from "./logger.js"; const logger = makeLogger("agent"); +// ── Image request detection ─────────────────────────────────────────────────── + +const IMAGE_INTENT_RE = + /\b(draw|illustrate|create\s+an?\s+image\s+of|generate\s+an?\s+image\s+of|visualize|visualise|make\s+an?\s+image\s+of|paint\s+me|sketch|render\s+an?\s+image\s+of|picture\s+of)\b/i; + +/** + * Returns true if the request text signals an image-generation intent. + */ +export function detectImageRequest(text: string): boolean { + return IMAGE_INTENT_RE.test(text); +} + +export interface ImageWorkResult { + b64_json: string; + mimeType: string; +} + export interface EvalResult { accepted: boolean; reason: string; @@ -442,6 +459,36 @@ Respond ONLY with valid JSON: {"accepted": true/false, "reason": "..."}`, return ""; } } + + /** + * Generate an image via Gemini for the given prompt. + * Falls back to a stub 1×1 transparent PNG when Gemini credentials are absent. + */ + async executeImageWork(prompt: string): Promise { + const geminiAvailable = + !!process.env["AI_INTEGRATIONS_GEMINI_API_KEY"] && + !!process.env["AI_INTEGRATIONS_GEMINI_BASE_URL"]; + + if (!geminiAvailable) { + logger.warn("Gemini credentials absent — returning stub image", { component: "agent" }); + // 1×1 transparent PNG (base64) + return { + b64_json: + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + mimeType: "image/png", + }; + } + + try { + const mod = (await import("@workspace/integrations-gemini-ai")) as { + generateImage: (prompt: string) => Promise<{ b64_json: string; mimeType: string }>; + }; + return await mod.generateImage(prompt); + } catch (err) { + logger.error("Gemini image generation failed", { error: String(err) }); + throw err; + } + } } export const agentService = new AgentService(); diff --git a/artifacts/api-server/src/lib/event-bus.ts b/artifacts/api-server/src/lib/event-bus.ts index f6af56f..ec2e7da 100644 --- a/artifacts/api-server/src/lib/event-bus.ts +++ b/artifacts/api-server/src/lib/event-bus.ts @@ -3,7 +3,7 @@ import { EventEmitter } from "events"; export type JobEvent = | { type: "job:state"; jobId: string; state: string } | { type: "job:paid"; jobId: string; invoiceType: "eval" | "work" } - | { type: "job:completed"; jobId: string; result: string } + | { type: "job:completed"; jobId: string; result: string; mediaUrl?: string; mediaType?: string } | { type: "job:failed"; jobId: string; reason: string }; export type SessionEvent = diff --git a/artifacts/api-server/src/lib/pricing.ts b/artifacts/api-server/src/lib/pricing.ts index d25a08b..6968b92 100644 --- a/artifacts/api-server/src/lib/pricing.ts +++ b/artifacts/api-server/src/lib/pricing.ts @@ -62,6 +62,11 @@ const DO_INFRA_PER_REQUEST_USD = DO_MONTHLY_COST_USD / DO_MONTHLY_REQUESTS; const ORIGINATOR_MARGIN_PCT = envFloat("ORIGINATOR_MARGIN_PCT", 25); +// ── Image generation flat rate ──────────────────────────────────────────────── +// Charged in addition to eval fee; covers Gemini imagen costs + margin. + +const IMAGE_GENERATION_FLAT_RATE_USD = envFloat("IMAGE_GENERATION_FLAT_RATE_USD", 0.04); + // ── Fixed fees ──────────────────────────────────────────────────────────────── const EVAL_FEE_SATS = envInt("EVAL_FEE_SATS", 10); @@ -95,6 +100,25 @@ export class PricingService { return BOOTSTRAP_FEE_SATS; } + /** + * Flat USD cost for a single image generation request (covers Gemini + margin). + */ + calculateImageFeeUsd(): number { + return IMAGE_GENERATION_FLAT_RATE_USD * (1 + this.marginPct / 100); + } + + /** + * Convert image flat rate to sats. Includes infra amortisation and margin. + * Returns the same shape as calculateWorkFeeSats() for drop-in use. + */ + async calculateImageFeeSats(): Promise { + const rawCostUsd = IMAGE_GENERATION_FLAT_RATE_USD + DO_INFRA_PER_REQUEST_USD; + const estimatedCostUsd = rawCostUsd * (1 + this.marginPct / 100); + const btcPriceUsd = await getBtcPriceUsd(); + const amountSats = usdToSats(estimatedCostUsd, btcPriceUsd); + return { amountSats, estimatedCostUsd, marginPct: this.marginPct, btcPriceUsd }; + } + // ── Token estimation ───────────────────────────────────────────────────── /** diff --git a/artifacts/api-server/src/routes/estimate.ts b/artifacts/api-server/src/routes/estimate.ts index 97b0c17..a301c6c 100644 --- a/artifacts/api-server/src/routes/estimate.ts +++ b/artifacts/api-server/src/routes/estimate.ts @@ -1,6 +1,6 @@ import { Router, type Request, type Response } from "express"; import { pricingService } from "../lib/pricing.js"; -import { agentService } from "../lib/agent.js"; +import { agentService, detectImageRequest } from "../lib/agent.js"; import { getBtcPriceUsd, usdToSats } from "../lib/btc-oracle.js"; import { freeTierService } from "../lib/free-tier.js"; import { trustService } from "../lib/trust.js"; @@ -25,10 +25,27 @@ router.get("/estimate", async (req: Request, res: Response) => { } try { - const { estimatedInputTokens: inputTokens, estimatedOutputTokens: outputTokens, estimatedCostUsd: costUsd } = - pricingService.estimateRequestCost(requestText, agentService.workModel); - const btcPriceUsd = await getBtcPriceUsd(); - const estimatedSats = usdToSats(costUsd, btcPriceUsd); + const isImageRequest = detectImageRequest(requestText); + + let inputTokens = 0; + let outputTokens = 0; + let costUsd: number; + let btcPriceUsd: number; + let estimatedSats: number; + + if (isImageRequest) { + const imageBreakdown = await pricingService.calculateImageFeeSats(); + costUsd = imageBreakdown.estimatedCostUsd; + btcPriceUsd = imageBreakdown.btcPriceUsd; + estimatedSats = imageBreakdown.amountSats; + } else { + const estimate = pricingService.estimateRequestCost(requestText, agentService.workModel); + inputTokens = estimate.estimatedInputTokens; + outputTokens = estimate.estimatedOutputTokens; + costUsd = estimate.estimatedCostUsd; + btcPriceUsd = await getBtcPriceUsd(); + estimatedSats = usdToSats(costUsd, btcPriceUsd); + } // Optionally resolve Nostr identity from query param or header for free-tier preview const rawToken = @@ -59,10 +76,11 @@ router.get("/estimate", async (req: Request, res: Response) => { estimatedSats, estimatedCostUsd: costUsd, btcPriceUsd, + ...(isImageRequest ? { mediaType: "image" } : {}), tokenEstimate: { inputTokens, outputTokens, - model: agentService.workModel, + model: isImageRequest ? "gemini-2.5-flash-image" : agentService.workModel, }, identity: { trust_tier: trustTier, diff --git a/artifacts/api-server/src/routes/jobs.ts b/artifacts/api-server/src/routes/jobs.ts index 341ba68..5630e9f 100644 --- a/artifacts/api-server/src/routes/jobs.ts +++ b/artifacts/api-server/src/routes/jobs.ts @@ -1,10 +1,10 @@ import { Router, type Request, type Response } from "express"; import { randomUUID, createHash } from "crypto"; -import { db, jobs, invoices, jobDebates, type Job } from "@workspace/db"; +import { db, jobs, invoices, jobDebates, jobMedia, type Job } from "@workspace/db"; import { eq, and } from "drizzle-orm"; import { CreateJobBody, GetJobParams } from "@workspace/api-zod"; import { lnbitsService } from "../lib/lnbits.js"; -import { agentService } from "../lib/agent.js"; +import { agentService, detectImageRequest } from "../lib/agent.js"; import { pricingService } from "../lib/pricing.js"; import { jobsLimiter } from "../lib/rate-limiter.js"; import { eventBus } from "../lib/event-bus.js"; @@ -110,12 +110,18 @@ async function runEvalInBackground( } if (evalResult.accepted) { - const { estimatedInputTokens, estimatedOutputTokens } = pricingService.estimateRequestCost(request, agentService.workModel); - const breakdown = await pricingService.calculateWorkFeeSats( - estimatedInputTokens, - estimatedOutputTokens, - agentService.workModel, - ); + // Detect image-generation requests and flag job accordingly + const isImageJob = detectImageRequest(request); + if (isImageJob) { + await db.update(jobs).set({ mediaType: "image", updatedAt: new Date() }).where(eq(jobs.id, jobId)); + } + + const breakdown = isImageJob + ? await pricingService.calculateImageFeeSats() + : await (async () => { + const { estimatedInputTokens, estimatedOutputTokens } = pricingService.estimateRequestCost(request, agentService.workModel); + return pricingService.calculateWorkFeeSats(estimatedInputTokens, estimatedOutputTokens, agentService.workModel); + })(); // ── Free-tier gate ────────────────────────────────────────────────── const ftDecision = await freeTierService.decide(nostrPubkey, breakdown.amountSats); @@ -254,18 +260,49 @@ async function runWorkInBackground( try { eventBus.publish({ type: "job:state", jobId, state: "executing" }); - const workResult = await agentService.executeWorkStreaming(request, (delta) => { - streamRegistry.write(jobId, delta); - }); + // Check if this is an image job + const jobRow = await getJobById(jobId); + const isImageJob = jobRow?.mediaType === "image"; + + let resultText = ""; + let mediaUrl: string | undefined; + let inputTokensUsed = 0; + let outputTokensUsed = 0; + + if (isImageJob) { + // Generate image via Gemini + const imageResult = await agentService.executeImageWork(request); + const mediaId = randomUUID(); + const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000); // 7 days + + await db.insert(jobMedia).values({ + id: mediaId, + entityId: jobId, + entityType: "job", + mediaType: "image", + mimeType: imageResult.mimeType, + data: imageResult.b64_json, + expiresAt, + }); + + mediaUrl = `/api/jobs/${jobId}/media`; + resultText = `Image generated. View at: ${mediaUrl}`; + streamRegistry.write(jobId, resultText); + } else { + const workResult = await agentService.executeWorkStreaming(request, (delta) => { + streamRegistry.write(jobId, delta); + }); + resultText = workResult.result; + inputTokensUsed = workResult.inputTokens; + outputTokensUsed = workResult.outputTokens; + } streamRegistry.end(jobId); latencyHistogram.record("work_phase", Date.now() - workStart); - const actualCostUsd = pricingService.calculateActualCostUsd( - workResult.inputTokens, - workResult.outputTokens, - agentService.workModel, - ); + const actualCostUsd = isImageJob + ? pricingService.calculateImageFeeUsd() + : pricingService.calculateActualCostUsd(inputTokensUsed, outputTokensUsed, agentService.workModel); const lockedBtcPrice = btcPriceUsd ?? 100_000; const actualTotalCostSats = pricingService.calculateActualChargeSats(actualCostUsd, lockedBtcPrice); @@ -288,9 +325,9 @@ async function runWorkInBackground( .update(jobs) .set({ state: "complete", - result: workResult.result, - actualInputTokens: workResult.inputTokens, - actualOutputTokens: workResult.outputTokens, + result: resultText, + actualInputTokens: isImageJob ? null : inputTokensUsed, + actualOutputTokens: isImageJob ? null : outputTokensUsed, actualCostUsd, actualAmountSats, refundAmountSats, @@ -302,13 +339,14 @@ async function runWorkInBackground( logger.info("work completed", { jobId, isFree, - inputTokens: workResult.inputTokens, - outputTokens: workResult.outputTokens, + isImageJob, + inputTokens: inputTokensUsed, + outputTokens: outputTokensUsed, actualAmountSats, refundAmountSats, refundState, }); - eventBus.publish({ type: "job:completed", jobId, result: workResult.result }); + eventBus.publish({ type: "job:completed", jobId, result: resultText, ...(mediaUrl ? { mediaUrl, mediaType: "image" } : {}) }); // Emit final actual cost for the UI cost ticker if (!isFree && actualAmountSats > 0) { eventBus.publish({ type: "cost:update", jobId, sats: actualAmountSats, phase: "work", isFinal: true }); @@ -667,6 +705,7 @@ router.get("/jobs/:id", async (req: Request, res: Response) => { res.json({ ...base, result: job.result ?? undefined, + ...(job.mediaType === "image" ? { mediaType: "image", mediaUrl: `/api/jobs/${job.id}/media` } : {}), ...(job.actualCostUsd != null ? { costLedger: { // Token usage @@ -706,6 +745,44 @@ router.get("/jobs/:id", async (req: Request, res: Response) => { } }); +// ── GET /jobs/:id/media ─────────────────────────────────────────────────────── + +router.get("/jobs/:id/media", async (req: Request, res: Response) => { + const paramResult = GetJobParams.safeParse(req.params); + if (!paramResult.success) { res.status(400).json({ error: "Invalid job id" }); return; } + const { id } = paramResult.data; + + try { + const rows = await db + .select() + .from(jobMedia) + .where(eq(jobMedia.entityId, id)) + .limit(1); + + const media = rows[0]; + if (!media) { + res.status(404).json({ error: "No media found for this job" }); + return; + } + + if (new Date() > media.expiresAt) { + res.status(410).json({ error: "Media has expired" }); + return; + } + + res.json({ + jobId: id, + mediaType: media.mediaType, + mimeType: media.mimeType, + data: media.data, + expiresAt: media.expiresAt.toISOString(), + }); + } catch (err) { + const message = err instanceof Error ? err.message : "Failed to fetch media"; + res.status(500).json({ error: message }); + } +}); + // ── POST /jobs/:id/refund ───────────────────────────────────────────────────── router.post("/jobs/:id/refund", async (req: Request, res: Response) => { diff --git a/artifacts/api-server/src/routes/sessions.ts b/artifacts/api-server/src/routes/sessions.ts index 7cb4a5c..1d25a32 100644 --- a/artifacts/api-server/src/routes/sessions.ts +++ b/artifacts/api-server/src/routes/sessions.ts @@ -1,11 +1,11 @@ import { Router, type Request, type Response } from "express"; import { randomBytes, randomUUID, createHash } from "crypto"; -import { db, sessions, sessionRequests, sessionMessages, getSessionHistory, type Session } from "@workspace/db"; +import { db, sessions, sessionRequests, sessionMessages, jobMedia, getSessionHistory, type Session } from "@workspace/db"; import { eq, and } from "drizzle-orm"; import { lnbitsService } from "../lib/lnbits.js"; import { sessionsLimiter } from "../lib/rate-limiter.js"; import { eventBus } from "../lib/event-bus.js"; -import { agentService } from "../lib/agent.js"; +import { agentService, detectImageRequest } from "../lib/agent.js"; import { pricingService } from "../lib/pricing.js"; import { getBtcPriceUsd, usdToSats } from "../lib/btc-oracle.js"; import { trustService } from "../lib/trust.js"; @@ -336,6 +336,11 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => { let finalState: "complete" | "rejected" | "failed" = "rejected"; let reason: string | null = null; let errorMessage: string | null = null; + let mediaUrl: string | null = null; + let mediaType: string | null = null; + + // Detect image generation intent before pricing estimate + const isImageRequest = detectImageRequest(requestText); // ── Pre-gate: free-tier decision on ESTIMATED cost before executing work ── // Estimate total request cost (work portion) pre-execution to determine subsidy. @@ -345,26 +350,59 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => { let ftDecision: import("../lib/free-tier.js").FreeTierDecision | null = null; if (evalResult.accepted && session.nostrPubkey) { // estimateRequestCost includes infra + margin. Convert to sats for decide(). - const { estimatedCostUsd } = pricingService.estimateRequestCost(requestText, agentService.workModel); - const estimatedSats = usdToSats(estimatedCostUsd, btcPriceUsd); + let estimatedSats: number; + if (isImageRequest) { + const imageBreakdown = await pricingService.calculateImageFeeSats(); + estimatedSats = imageBreakdown.amountSats; + } else { + const { estimatedCostUsd } = pricingService.estimateRequestCost(requestText, agentService.workModel); + estimatedSats = usdToSats(estimatedCostUsd, btcPriceUsd); + } ftDecision = await freeTierService.decide(session.nostrPubkey, estimatedSats); } if (evalResult.accepted) { - try { - const workResult = await agentService.executeWork(requestText, history); - workInputTokens = workResult.inputTokens; - workOutputTokens = workResult.outputTokens; - workCostUsd = pricingService.calculateActualCostUsd( - workResult.inputTokens, - workResult.outputTokens, - agentService.workModel, - ); - result = workResult.result; - finalState = "complete"; - } catch (err) { - errorMessage = err instanceof Error ? err.message : "Execution error"; - finalState = "failed"; + if (isImageRequest) { + try { + const imageResult = await agentService.executeImageWork(requestText); + const mediaId = randomUUID(); + const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000); // 7 days + + await db.insert(jobMedia).values({ + id: mediaId, + entityId: requestId, + entityType: "session_request", + mediaType: "image", + mimeType: imageResult.mimeType, + data: imageResult.b64_json, + expiresAt, + }); + + mediaUrl = `/api/sessions/${id}/requests/${requestId}/media`; + mediaType = "image"; + workCostUsd = pricingService.calculateImageFeeUsd(); + result = `Image generated. View at: ${mediaUrl}`; + finalState = "complete"; + } catch (err) { + errorMessage = err instanceof Error ? err.message : "Image generation error"; + finalState = "failed"; + } + } else { + try { + const workResult = await agentService.executeWork(requestText, history); + workInputTokens = workResult.inputTokens; + workOutputTokens = workResult.outputTokens; + workCostUsd = pricingService.calculateActualCostUsd( + workResult.inputTokens, + workResult.outputTokens, + agentService.workModel, + ); + result = workResult.result; + finalState = "complete"; + } catch (err) { + errorMessage = err instanceof Error ? err.message : "Execution error"; + finalState = "failed"; + } } } else { reason = evalResult.reason; @@ -491,6 +529,7 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => { ...(result ? { result } : {}), ...(reason ? { reason } : {}), ...(errorMessage ? { errorMessage } : {}), + ...(mediaUrl ? { mediaUrl, mediaType } : {}), debitedSats, balanceRemaining: newBalance, ...(freeTierServed ? { free_tier: true, absorbed_sats: absorbedSats } : {}), @@ -608,4 +647,43 @@ router.delete("/sessions/:id/history", async (req: Request, res: Response) => { } }); +// ── GET /sessions/:id/requests/:requestId/media ─────────────────────────────── + +router.get("/sessions/:id/requests/:requestId/media", async (req: Request, res: Response) => { + const sessionId = req.params.id as string; + const requestId = req.params.requestId as string; + + try { + const session = await getSessionById(sessionId); + if (!session) { res.status(404).json({ error: "Session not found" }); return; } + + const rows = await db + .select() + .from(jobMedia) + .where(eq(jobMedia.entityId, requestId)) + .limit(1); + + const media = rows[0]; + if (!media) { + res.status(404).json({ error: "No media found for this request" }); + return; + } + + if (new Date() > media.expiresAt) { + res.status(410).json({ error: "Media has expired" }); + return; + } + + res.json({ + requestId, + mediaType: media.mediaType, + mimeType: media.mimeType, + data: media.data, + expiresAt: media.expiresAt.toISOString(), + }); + } catch (err) { + res.status(500).json({ error: err instanceof Error ? err.message : "Failed to fetch media" }); + } +}); + export default router; diff --git a/lib/db/migrations/0010_job_media.sql b/lib/db/migrations/0010_job_media.sql new file mode 100644 index 0000000..0391438 --- /dev/null +++ b/lib/db/migrations/0010_job_media.sql @@ -0,0 +1,26 @@ +-- Migration: Image generation media storage (#19) +-- Adds job_media table for storing generated images (base64) with 7-day TTL. +-- Also adds media_type column to jobs table to flag image-type work. + +-- ── job_media ───────────────────────────────────────────────────────────────── +-- Stores generated media for both standalone jobs and session requests. +-- entity_id is polymorphic: job ID or session request ID. +-- expires_at is set to NOW + 7 days at insert time. + +CREATE TABLE IF NOT EXISTS job_media ( + id TEXT PRIMARY KEY, + entity_id TEXT NOT NULL, -- job ID or session request ID + entity_type TEXT NOT NULL, -- 'job' | 'session_request' + media_type TEXT NOT NULL, -- 'image' + mime_type TEXT NOT NULL, -- e.g. 'image/png' + data TEXT NOT NULL, -- base64-encoded image data + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_job_media_entity_id ON job_media(entity_id); + +-- ── jobs.media_type ─────────────────────────────────────────────────────────── +-- Nullable flag set during eval phase for image-generation requests. + +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS media_type TEXT; diff --git a/lib/db/src/schema/index.ts b/lib/db/src/schema/index.ts index e62c9ee..d0a9505 100644 --- a/lib/db/src/schema/index.ts +++ b/lib/db/src/schema/index.ts @@ -14,3 +14,4 @@ export * from "./relay-accounts"; export * from "./relay-event-queue"; export * from "./job-debates"; export * from "./session-messages"; +export * from "./job-media"; diff --git a/lib/db/src/schema/job-media.ts b/lib/db/src/schema/job-media.ts new file mode 100644 index 0000000..17b604e --- /dev/null +++ b/lib/db/src/schema/job-media.ts @@ -0,0 +1,19 @@ +import { pgTable, text, timestamp } from "drizzle-orm/pg-core"; + +/** + * job_media — stores generated media (images) for jobs and session requests. + * entityId is polymorphic: it can be a job ID or a session request ID. + * expiresAt is set to NOW + 7 days; a cleanup job should purge expired rows. + */ +export const jobMedia = pgTable("job_media", { + id: text("id").primaryKey(), + entityId: text("entity_id").notNull(), // job ID or session request ID + entityType: text("entity_type").notNull(), // 'job' | 'session_request' + mediaType: text("media_type").notNull(), // 'image' + mimeType: text("mime_type").notNull(), // e.g. 'image/png' + data: text("data").notNull(), // base64-encoded image data + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), +}); + +export type JobMedia = typeof jobMedia.$inferSelect; diff --git a/lib/db/src/schema/jobs.ts b/lib/db/src/schema/jobs.ts index b5538c0..d0a04ed 100644 --- a/lib/db/src/schema/jobs.ts +++ b/lib/db/src/schema/jobs.ts @@ -52,6 +52,9 @@ export const jobs = pgTable("jobs", { refundState: text("refund_state").$type<"not_applicable" | "pending" | "paid">(), refundPaymentHash: text("refund_payment_hash"), + // ── Image generation (set during eval if request is an image job) ─────────── + mediaType: text("media_type"), // 'image' | null + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), }); diff --git a/the-matrix/js/session.js b/the-matrix/js/session.js index b505770..127c92c 100644 --- a/the-matrix/js/session.js +++ b/the-matrix/js/session.js @@ -157,15 +157,20 @@ export async function sessionSendHandler(text) { _saveToStorage(); _applySessionUI(); - const reply = data.result || data.reason || '…'; - setSpeechBubble(reply); - appendSystemMessage('Timmy: ' + reply.slice(0, 80)); + if (data.mediaType === 'image' && data.mediaUrl) { + // Fetch image data and render inline + _renderImageResponse(data.mediaUrl, text); + } else { + const reply = data.result || data.reason || '…'; + setSpeechBubble(reply); + appendSystemMessage('Timmy: ' + reply.slice(0, 80)); - // Sentiment-driven mood on inbound Timmy reply - sentiment(reply).then(s => { - setMood(s.label); - setTimeout(() => setMood(null), 10_000); - }).catch(() => {}); + // Sentiment-driven mood on inbound Timmy reply + sentiment(reply).then(s => { + setMood(s.label); + setTimeout(() => setMood(null), 10_000); + }).catch(() => {}); + } // Update active-step balance if panel is open _updateActiveStep(); @@ -178,6 +183,66 @@ export async function sessionSendHandler(text) { } } +// ── Image rendering ─────────────────────────────────────────────────────────── + +async function _renderImageResponse(mediaUrl, prompt) { + const $log = document.getElementById('event-log'); + if (!$log) return; + + setSpeechBubble('✨ Here is your image!'); + appendSystemMessage('Timmy: ✨ Image generated!'); + + try { + const res = await fetch(mediaUrl); + if (!res.ok) { + appendSystemMessage('Timmy: Image ready — ' + mediaUrl); + return; + } + const data = await res.json(); + const src = `data:${data.mimeType};base64,${data.data}`; + + const container = document.createElement('div'); + container.className = 'log-entry timmy-image-result'; + container.style.cssText = [ + 'margin:6px 0;padding:6px;', + 'border:1px solid #336655;border-radius:4px;', + 'background:#0a1a14;', + ].join(''); + + const img = document.createElement('img'); + img.src = src; + img.alt = prompt.slice(0, 60); + img.style.cssText = [ + 'max-width:100%;max-height:240px;', + 'display:block;border-radius:3px;', + 'cursor:pointer;', + ].join(''); + img.title = 'Click to view full size'; + + const dlBtn = document.createElement('a'); + dlBtn.href = src; + dlBtn.download = 'timmy-image.png'; + dlBtn.textContent = '⬇ Download'; + dlBtn.style.cssText = [ + 'display:inline-block;margin-top:4px;', + 'font-size:10px;color:#44cc88;', + 'text-decoration:none;letter-spacing:1px;', + ].join(''); + + container.appendChild(img); + container.appendChild(dlBtn); + + const entries = $log.querySelectorAll('.log-entry'); + if (entries.length >= 6) { + $log.removeChild(entries[0]); + } + $log.appendChild(container); + $log.scrollTop = $log.scrollHeight; + } catch { + appendSystemMessage('Timmy: Image generated — ' + mediaUrl); + } +} + // ── Panel open/close ────────────────────────────────────────────────────────── function _openPanel() { -- 2.43.0