1 Commits

Author SHA1 Message Date
Alexander Whitestone
f13a1d0235 feat: Gemini image generation in Workshop chat (#19)
Some checks failed
CI / Typecheck & Lint (pull_request) Failing after 1s
- Add image intent detection (draw/illustrate/visualize/create an image)
  via `detectImageRequest()` in agent.ts; exports used by jobs and sessions
- Add `executeImageWork()` to AgentService: calls Gemini generateImage with
  graceful fallback stub PNG when Gemini credentials are absent
- Add `job_media` table (migration 0010) for base64 image storage with 7-day TTL;
  entity_id is polymorphic for both jobs and session requests
- Add `media_type TEXT` column to jobs table (flagged during eval phase)
- Add `calculateImageFeeSats()` / `calculateImageFeeUsd()` to PricingService;
  uses IMAGE_GENERATION_FLAT_RATE_USD env var (default $0.04)
- Jobs route: detect image jobs in eval phase, route to Gemini in execution,
  store image in job_media; expose GET /api/jobs/:id/media endpoint
- Sessions route: detect image requests, call executeImageWork, store in
  job_media, return mediaUrl and mediaType in response
- Estimate route: return image pricing and mediaType:'image' for image requests
- Event bus: add optional mediaUrl/mediaType to job:completed event
- Frontend session.js: render generated images inline with download button

Fixes #19

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-23 22:36:10 -04:00
15 changed files with 422 additions and 427 deletions

View File

@@ -2,6 +2,23 @@ import { makeLogger } from "./logger.js";
const logger = makeLogger("agent");
// ── Image request detection ───────────────────────────────────────────────────
const IMAGE_INTENT_RE =
/\b(draw|illustrate|create\s+an?\s+image\s+of|generate\s+an?\s+image\s+of|visualize|visualise|make\s+an?\s+image\s+of|paint\s+me|sketch|render\s+an?\s+image\s+of|picture\s+of)\b/i;
/**
* Returns true if the request text signals an image-generation intent.
*/
export function detectImageRequest(text: string): boolean {
return IMAGE_INTENT_RE.test(text);
}
export interface ImageWorkResult {
b64_json: string;
mimeType: string;
}
export interface EvalResult {
accepted: boolean;
reason: string;
@@ -442,6 +459,36 @@ Respond ONLY with valid JSON: {"accepted": true/false, "reason": "..."}`,
return "";
}
}
/**
* Generate an image via Gemini for the given prompt.
* Falls back to a stub 1×1 transparent PNG when Gemini credentials are absent.
*/
async executeImageWork(prompt: string): Promise<ImageWorkResult> {
const geminiAvailable =
!!process.env["AI_INTEGRATIONS_GEMINI_API_KEY"] &&
!!process.env["AI_INTEGRATIONS_GEMINI_BASE_URL"];
if (!geminiAvailable) {
logger.warn("Gemini credentials absent — returning stub image", { component: "agent" });
// 1×1 transparent PNG (base64)
return {
b64_json:
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
mimeType: "image/png",
};
}
try {
const mod = (await import("@workspace/integrations-gemini-ai")) as {
generateImage: (prompt: string) => Promise<{ b64_json: string; mimeType: string }>;
};
return await mod.generateImage(prompt);
} catch (err) {
logger.error("Gemini image generation failed", { error: String(err) });
throw err;
}
}
}
export const agentService = new AgentService();

View File

@@ -3,7 +3,7 @@ import { EventEmitter } from "events";
export type JobEvent =
| { type: "job:state"; jobId: string; state: string }
| { type: "job:paid"; jobId: string; invoiceType: "eval" | "work" }
| { type: "job:completed"; jobId: string; result: string }
| { type: "job:completed"; jobId: string; result: string; mediaUrl?: string; mediaType?: string }
| { type: "job:failed"; jobId: string; reason: string };
export type SessionEvent =

View File

@@ -62,6 +62,11 @@ const DO_INFRA_PER_REQUEST_USD = DO_MONTHLY_COST_USD / DO_MONTHLY_REQUESTS;
const ORIGINATOR_MARGIN_PCT = envFloat("ORIGINATOR_MARGIN_PCT", 25);
// ── Image generation flat rate ────────────────────────────────────────────────
// Charged in addition to eval fee; covers Gemini imagen costs + margin.
const IMAGE_GENERATION_FLAT_RATE_USD = envFloat("IMAGE_GENERATION_FLAT_RATE_USD", 0.04);
// ── Fixed fees ────────────────────────────────────────────────────────────────
const EVAL_FEE_SATS = envInt("EVAL_FEE_SATS", 10);
@@ -95,6 +100,25 @@ export class PricingService {
return BOOTSTRAP_FEE_SATS;
}
/**
* Flat USD cost for a single image generation request (covers Gemini + margin).
*/
calculateImageFeeUsd(): number {
return IMAGE_GENERATION_FLAT_RATE_USD * (1 + this.marginPct / 100);
}
/**
* Convert image flat rate to sats. Includes infra amortisation and margin.
* Returns the same shape as calculateWorkFeeSats() for drop-in use.
*/
async calculateImageFeeSats(): Promise<WorkFeeBreakdown> {
const rawCostUsd = IMAGE_GENERATION_FLAT_RATE_USD + DO_INFRA_PER_REQUEST_USD;
const estimatedCostUsd = rawCostUsd * (1 + this.marginPct / 100);
const btcPriceUsd = await getBtcPriceUsd();
const amountSats = usdToSats(estimatedCostUsd, btcPriceUsd);
return { amountSats, estimatedCostUsd, marginPct: this.marginPct, btcPriceUsd };
}
// ── Token estimation ─────────────────────────────────────────────────────
/**

View File

@@ -1,6 +1,6 @@
import { Router, type Request, type Response } from "express";
import { pricingService } from "../lib/pricing.js";
import { agentService } from "../lib/agent.js";
import { agentService, detectImageRequest } from "../lib/agent.js";
import { getBtcPriceUsd, usdToSats } from "../lib/btc-oracle.js";
import { freeTierService } from "../lib/free-tier.js";
import { trustService } from "../lib/trust.js";
@@ -25,10 +25,27 @@ router.get("/estimate", async (req: Request, res: Response) => {
}
try {
const { estimatedInputTokens: inputTokens, estimatedOutputTokens: outputTokens, estimatedCostUsd: costUsd } =
pricingService.estimateRequestCost(requestText, agentService.workModel);
const btcPriceUsd = await getBtcPriceUsd();
const estimatedSats = usdToSats(costUsd, btcPriceUsd);
const isImageRequest = detectImageRequest(requestText);
let inputTokens = 0;
let outputTokens = 0;
let costUsd: number;
let btcPriceUsd: number;
let estimatedSats: number;
if (isImageRequest) {
const imageBreakdown = await pricingService.calculateImageFeeSats();
costUsd = imageBreakdown.estimatedCostUsd;
btcPriceUsd = imageBreakdown.btcPriceUsd;
estimatedSats = imageBreakdown.amountSats;
} else {
const estimate = pricingService.estimateRequestCost(requestText, agentService.workModel);
inputTokens = estimate.estimatedInputTokens;
outputTokens = estimate.estimatedOutputTokens;
costUsd = estimate.estimatedCostUsd;
btcPriceUsd = await getBtcPriceUsd();
estimatedSats = usdToSats(costUsd, btcPriceUsd);
}
// Optionally resolve Nostr identity from query param or header for free-tier preview
const rawToken =
@@ -59,10 +76,11 @@ router.get("/estimate", async (req: Request, res: Response) => {
estimatedSats,
estimatedCostUsd: costUsd,
btcPriceUsd,
...(isImageRequest ? { mediaType: "image" } : {}),
tokenEstimate: {
inputTokens,
outputTokens,
model: agentService.workModel,
model: isImageRequest ? "gemini-2.5-flash-image" : agentService.workModel,
},
identity: {
trust_tier: trustTier,

View File

@@ -1,10 +1,10 @@
import { Router, type Request, type Response } from "express";
import { randomUUID, createHash } from "crypto";
import { db, jobs, invoices, jobDebates, type Job } from "@workspace/db";
import { db, jobs, invoices, jobDebates, jobMedia, type Job } from "@workspace/db";
import { eq, and } from "drizzle-orm";
import { CreateJobBody, GetJobParams } from "@workspace/api-zod";
import { lnbitsService } from "../lib/lnbits.js";
import { agentService } from "../lib/agent.js";
import { agentService, detectImageRequest } from "../lib/agent.js";
import { pricingService } from "../lib/pricing.js";
import { jobsLimiter } from "../lib/rate-limiter.js";
import { eventBus } from "../lib/event-bus.js";
@@ -110,12 +110,18 @@ async function runEvalInBackground(
}
if (evalResult.accepted) {
const { estimatedInputTokens, estimatedOutputTokens } = pricingService.estimateRequestCost(request, agentService.workModel);
const breakdown = await pricingService.calculateWorkFeeSats(
estimatedInputTokens,
estimatedOutputTokens,
agentService.workModel,
);
// Detect image-generation requests and flag job accordingly
const isImageJob = detectImageRequest(request);
if (isImageJob) {
await db.update(jobs).set({ mediaType: "image", updatedAt: new Date() }).where(eq(jobs.id, jobId));
}
const breakdown = isImageJob
? await pricingService.calculateImageFeeSats()
: await (async () => {
const { estimatedInputTokens, estimatedOutputTokens } = pricingService.estimateRequestCost(request, agentService.workModel);
return pricingService.calculateWorkFeeSats(estimatedInputTokens, estimatedOutputTokens, agentService.workModel);
})();
// ── Free-tier gate ──────────────────────────────────────────────────
const ftDecision = await freeTierService.decide(nostrPubkey, breakdown.amountSats);
@@ -254,18 +260,49 @@ async function runWorkInBackground(
try {
eventBus.publish({ type: "job:state", jobId, state: "executing" });
const workResult = await agentService.executeWorkStreaming(request, (delta) => {
streamRegistry.write(jobId, delta);
});
// Check if this is an image job
const jobRow = await getJobById(jobId);
const isImageJob = jobRow?.mediaType === "image";
let resultText = "";
let mediaUrl: string | undefined;
let inputTokensUsed = 0;
let outputTokensUsed = 0;
if (isImageJob) {
// Generate image via Gemini
const imageResult = await agentService.executeImageWork(request);
const mediaId = randomUUID();
const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000); // 7 days
await db.insert(jobMedia).values({
id: mediaId,
entityId: jobId,
entityType: "job",
mediaType: "image",
mimeType: imageResult.mimeType,
data: imageResult.b64_json,
expiresAt,
});
mediaUrl = `/api/jobs/${jobId}/media`;
resultText = `Image generated. View at: ${mediaUrl}`;
streamRegistry.write(jobId, resultText);
} else {
const workResult = await agentService.executeWorkStreaming(request, (delta) => {
streamRegistry.write(jobId, delta);
});
resultText = workResult.result;
inputTokensUsed = workResult.inputTokens;
outputTokensUsed = workResult.outputTokens;
}
streamRegistry.end(jobId);
latencyHistogram.record("work_phase", Date.now() - workStart);
const actualCostUsd = pricingService.calculateActualCostUsd(
workResult.inputTokens,
workResult.outputTokens,
agentService.workModel,
);
const actualCostUsd = isImageJob
? pricingService.calculateImageFeeUsd()
: pricingService.calculateActualCostUsd(inputTokensUsed, outputTokensUsed, agentService.workModel);
const lockedBtcPrice = btcPriceUsd ?? 100_000;
const actualTotalCostSats = pricingService.calculateActualChargeSats(actualCostUsd, lockedBtcPrice);
@@ -288,9 +325,9 @@ async function runWorkInBackground(
.update(jobs)
.set({
state: "complete",
result: workResult.result,
actualInputTokens: workResult.inputTokens,
actualOutputTokens: workResult.outputTokens,
result: resultText,
actualInputTokens: isImageJob ? null : inputTokensUsed,
actualOutputTokens: isImageJob ? null : outputTokensUsed,
actualCostUsd,
actualAmountSats,
refundAmountSats,
@@ -302,13 +339,14 @@ async function runWorkInBackground(
logger.info("work completed", {
jobId,
isFree,
inputTokens: workResult.inputTokens,
outputTokens: workResult.outputTokens,
isImageJob,
inputTokens: inputTokensUsed,
outputTokens: outputTokensUsed,
actualAmountSats,
refundAmountSats,
refundState,
});
eventBus.publish({ type: "job:completed", jobId, result: workResult.result });
eventBus.publish({ type: "job:completed", jobId, result: resultText, ...(mediaUrl ? { mediaUrl, mediaType: "image" } : {}) });
// Emit final actual cost for the UI cost ticker
if (!isFree && actualAmountSats > 0) {
eventBus.publish({ type: "cost:update", jobId, sats: actualAmountSats, phase: "work", isFinal: true });
@@ -667,6 +705,7 @@ router.get("/jobs/:id", async (req: Request, res: Response) => {
res.json({
...base,
result: job.result ?? undefined,
...(job.mediaType === "image" ? { mediaType: "image", mediaUrl: `/api/jobs/${job.id}/media` } : {}),
...(job.actualCostUsd != null ? {
costLedger: {
// Token usage
@@ -706,6 +745,44 @@ router.get("/jobs/:id", async (req: Request, res: Response) => {
}
});
// ── GET /jobs/:id/media ───────────────────────────────────────────────────────
router.get("/jobs/:id/media", async (req: Request, res: Response) => {
const paramResult = GetJobParams.safeParse(req.params);
if (!paramResult.success) { res.status(400).json({ error: "Invalid job id" }); return; }
const { id } = paramResult.data;
try {
const rows = await db
.select()
.from(jobMedia)
.where(eq(jobMedia.entityId, id))
.limit(1);
const media = rows[0];
if (!media) {
res.status(404).json({ error: "No media found for this job" });
return;
}
if (new Date() > media.expiresAt) {
res.status(410).json({ error: "Media has expired" });
return;
}
res.json({
jobId: id,
mediaType: media.mediaType,
mimeType: media.mimeType,
data: media.data,
expiresAt: media.expiresAt.toISOString(),
});
} catch (err) {
const message = err instanceof Error ? err.message : "Failed to fetch media";
res.status(500).json({ error: message });
}
});
// ── POST /jobs/:id/refund ─────────────────────────────────────────────────────
router.post("/jobs/:id/refund", async (req: Request, res: Response) => {

View File

@@ -1,11 +1,11 @@
import { Router, type Request, type Response } from "express";
import { randomBytes, randomUUID, createHash } from "crypto";
import { db, sessions, sessionRequests, sessionMessages, getSessionHistory, type Session } from "@workspace/db";
import { db, sessions, sessionRequests, sessionMessages, jobMedia, getSessionHistory, type Session } from "@workspace/db";
import { eq, and } from "drizzle-orm";
import { lnbitsService } from "../lib/lnbits.js";
import { sessionsLimiter } from "../lib/rate-limiter.js";
import { eventBus } from "../lib/event-bus.js";
import { agentService } from "../lib/agent.js";
import { agentService, detectImageRequest } from "../lib/agent.js";
import { pricingService } from "../lib/pricing.js";
import { getBtcPriceUsd, usdToSats } from "../lib/btc-oracle.js";
import { trustService } from "../lib/trust.js";
@@ -336,6 +336,11 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => {
let finalState: "complete" | "rejected" | "failed" = "rejected";
let reason: string | null = null;
let errorMessage: string | null = null;
let mediaUrl: string | null = null;
let mediaType: string | null = null;
// Detect image generation intent before pricing estimate
const isImageRequest = detectImageRequest(requestText);
// ── Pre-gate: free-tier decision on ESTIMATED cost before executing work ──
// Estimate total request cost (work portion) pre-execution to determine subsidy.
@@ -345,26 +350,59 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => {
let ftDecision: import("../lib/free-tier.js").FreeTierDecision | null = null;
if (evalResult.accepted && session.nostrPubkey) {
// estimateRequestCost includes infra + margin. Convert to sats for decide().
const { estimatedCostUsd } = pricingService.estimateRequestCost(requestText, agentService.workModel);
const estimatedSats = usdToSats(estimatedCostUsd, btcPriceUsd);
let estimatedSats: number;
if (isImageRequest) {
const imageBreakdown = await pricingService.calculateImageFeeSats();
estimatedSats = imageBreakdown.amountSats;
} else {
const { estimatedCostUsd } = pricingService.estimateRequestCost(requestText, agentService.workModel);
estimatedSats = usdToSats(estimatedCostUsd, btcPriceUsd);
}
ftDecision = await freeTierService.decide(session.nostrPubkey, estimatedSats);
}
if (evalResult.accepted) {
try {
const workResult = await agentService.executeWork(requestText, history);
workInputTokens = workResult.inputTokens;
workOutputTokens = workResult.outputTokens;
workCostUsd = pricingService.calculateActualCostUsd(
workResult.inputTokens,
workResult.outputTokens,
agentService.workModel,
);
result = workResult.result;
finalState = "complete";
} catch (err) {
errorMessage = err instanceof Error ? err.message : "Execution error";
finalState = "failed";
if (isImageRequest) {
try {
const imageResult = await agentService.executeImageWork(requestText);
const mediaId = randomUUID();
const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000); // 7 days
await db.insert(jobMedia).values({
id: mediaId,
entityId: requestId,
entityType: "session_request",
mediaType: "image",
mimeType: imageResult.mimeType,
data: imageResult.b64_json,
expiresAt,
});
mediaUrl = `/api/sessions/${id}/requests/${requestId}/media`;
mediaType = "image";
workCostUsd = pricingService.calculateImageFeeUsd();
result = `Image generated. View at: ${mediaUrl}`;
finalState = "complete";
} catch (err) {
errorMessage = err instanceof Error ? err.message : "Image generation error";
finalState = "failed";
}
} else {
try {
const workResult = await agentService.executeWork(requestText, history);
workInputTokens = workResult.inputTokens;
workOutputTokens = workResult.outputTokens;
workCostUsd = pricingService.calculateActualCostUsd(
workResult.inputTokens,
workResult.outputTokens,
agentService.workModel,
);
result = workResult.result;
finalState = "complete";
} catch (err) {
errorMessage = err instanceof Error ? err.message : "Execution error";
finalState = "failed";
}
}
} else {
reason = evalResult.reason;
@@ -491,6 +529,7 @@ router.post("/sessions/:id/request", async (req: Request, res: Response) => {
...(result ? { result } : {}),
...(reason ? { reason } : {}),
...(errorMessage ? { errorMessage } : {}),
...(mediaUrl ? { mediaUrl, mediaType } : {}),
debitedSats,
balanceRemaining: newBalance,
...(freeTierServed ? { free_tier: true, absorbed_sats: absorbedSats } : {}),
@@ -608,4 +647,43 @@ router.delete("/sessions/:id/history", async (req: Request, res: Response) => {
}
});
// ── GET /sessions/:id/requests/:requestId/media ───────────────────────────────
router.get("/sessions/:id/requests/:requestId/media", async (req: Request, res: Response) => {
const sessionId = req.params.id as string;
const requestId = req.params.requestId as string;
try {
const session = await getSessionById(sessionId);
if (!session) { res.status(404).json({ error: "Session not found" }); return; }
const rows = await db
.select()
.from(jobMedia)
.where(eq(jobMedia.entityId, requestId))
.limit(1);
const media = rows[0];
if (!media) {
res.status(404).json({ error: "No media found for this request" });
return;
}
if (new Date() > media.expiresAt) {
res.status(410).json({ error: "Media has expired" });
return;
}
res.json({
requestId,
mediaType: media.mediaType,
mimeType: media.mimeType,
data: media.data,
expiresAt: media.expiresAt.toISOString(),
});
} catch (err) {
res.status(500).json({ error: err instanceof Error ? err.message : "Failed to fetch media" });
}
});
export default router;

View File

@@ -0,0 +1,26 @@
-- Migration: Image generation media storage (#19)
-- Adds job_media table for storing generated images (base64) with 7-day TTL.
-- Also adds media_type column to jobs table to flag image-type work.
-- ── job_media ─────────────────────────────────────────────────────────────────
-- Stores generated media for both standalone jobs and session requests.
-- entity_id is polymorphic: job ID or session request ID.
-- expires_at is set to NOW + 7 days at insert time.
CREATE TABLE IF NOT EXISTS job_media (
id TEXT PRIMARY KEY,
entity_id TEXT NOT NULL, -- job ID or session request ID
entity_type TEXT NOT NULL, -- 'job' | 'session_request'
media_type TEXT NOT NULL, -- 'image'
mime_type TEXT NOT NULL, -- e.g. 'image/png'
data TEXT NOT NULL, -- base64-encoded image data
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_job_media_entity_id ON job_media(entity_id);
-- ── jobs.media_type ───────────────────────────────────────────────────────────
-- Nullable flag set during eval phase for image-generation requests.
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS media_type TEXT;

View File

@@ -14,3 +14,4 @@ export * from "./relay-accounts";
export * from "./relay-event-queue";
export * from "./job-debates";
export * from "./session-messages";
export * from "./job-media";

View File

@@ -0,0 +1,19 @@
import { pgTable, text, timestamp } from "drizzle-orm/pg-core";
/**
* job_media — stores generated media (images) for jobs and session requests.
* entityId is polymorphic: it can be a job ID or a session request ID.
* expiresAt is set to NOW + 7 days; a cleanup job should purge expired rows.
*/
export const jobMedia = pgTable("job_media", {
id: text("id").primaryKey(),
entityId: text("entity_id").notNull(), // job ID or session request ID
entityType: text("entity_type").notNull(), // 'job' | 'session_request'
mediaType: text("media_type").notNull(), // 'image'
mimeType: text("mime_type").notNull(), // e.g. 'image/png'
data: text("data").notNull(), // base64-encoded image data
createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
});
export type JobMedia = typeof jobMedia.$inferSelect;

View File

@@ -52,6 +52,9 @@ export const jobs = pgTable("jobs", {
refundState: text("refund_state").$type<"not_applicable" | "pending" | "paid">(),
refundPaymentHash: text("refund_payment_hash"),
// ── Image generation (set during eval if request is an image job) ───────────
mediaType: text("media_type"), // 'image' | null
createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(),
});

View File

@@ -16,9 +16,6 @@ let _camera = null;
let _timmyGroup = null;
let _applySlap = null;
// Registered 3D click targets: Array of { group: THREE.Object3D, callback: fn }
const _clickTargets = [];
const _raycaster = new THREE.Raycaster();
const _pointer = new THREE.Vector2();
const _noCtxMenu = e => e.preventDefault();
@@ -36,15 +33,6 @@ export function registerSlapTarget(timmyGroup, applyFn) {
_applySlap = applyFn;
}
/**
* Register a Three.js Object3D as a clickable target.
* @param {THREE.Object3D} group The mesh/group to raycast against.
* @param {Function} callback Called when the user clicks the object.
*/
export function registerClickTarget(group, callback) {
_clickTargets.push({ group, callback });
}
export function initInteraction(camera, renderer) {
_camera = camera;
_canvas = renderer.domElement;
@@ -64,7 +52,6 @@ export function disposeInteraction() {
_camera = null;
_timmyGroup = null;
_applySlap = null;
_clickTargets.length = 0;
}
// ── Internal ──────────────────────────────────────────────────────────────────
@@ -133,20 +120,6 @@ function _tryAgentInspect(clientX, clientY) {
let _tapStart = 0;
let _tapX = 0, _tapY = 0;
function _tryClickTargets(clientX, clientY) {
if (!_clickTargets.length || !_camera || !_canvas) return false;
const rect = _canvas.getBoundingClientRect();
_pointer.x = ((clientX - rect.left) / rect.width) * 2 - 1;
_pointer.y = ((clientY - rect.top) / rect.height) * -2 + 1;
_raycaster.setFromCamera(_pointer, _camera);
for (const target of _clickTargets) {
if (!target.group.visible) continue;
const hits = _raycaster.intersectObject(target.group, true);
if (hits.length > 0) { target.callback(); return true; }
}
return false;
}
function _onPointerDown(event) {
// Record tap start for short-tap detection
_tapStart = Date.now();
@@ -162,30 +135,21 @@ function _onPointerDown(event) {
const dt = Date.now() - _tapStart;
const moved = Math.hypot(upEvent.clientX - _tapX, upEvent.clientY - _tapY);
if (dt < 220 && moved < 18) {
// Short tap — try click targets, then slap, then inspect
if (!_tryClickTargets(upEvent.clientX, upEvent.clientY)) {
if (!_hitTestTimmy(upEvent.clientX, upEvent.clientY)) {
_tryAgentInspect(upEvent.clientX, upEvent.clientY);
}
// Short tap — try slap then inspect
if (!_hitTestTimmy(upEvent.clientX, upEvent.clientY)) {
_tryAgentInspect(upEvent.clientX, upEvent.clientY);
}
}
};
_canvas.addEventListener('pointerup', onUp, { once: true });
} else {
// Desktop click: only fire when pointer lock is already active
// (otherwise navigation.js requests lock on this same click — that's fine,
// both can fire; slap + lock request together is acceptable)
if (document.pointerLockElement === _canvas) {
// FPS mode: cast from screen centre
// Cast from screen center in FPS mode
_pointer.set(0, 0);
_raycaster.setFromCamera(_pointer, _camera);
// Check registered click targets first
for (const target of _clickTargets) {
if (!target.group.visible) continue;
const hits = _raycaster.intersectObject(target.group, true);
if (hits.length > 0) {
target.callback();
event.stopImmediatePropagation();
return;
}
}
if (_timmyGroup && _applySlap) {
const hits = _raycaster.intersectObject(_timmyGroup, true);
if (hits.length > 0) {
@@ -193,9 +157,6 @@ function _onPointerDown(event) {
event.stopImmediatePropagation();
}
}
} else {
// Not in pointer lock — still handle click targets (e.g. power meter)
_tryClickTargets(event.clientX, event.clientY);
}
}
}

View File

@@ -7,10 +7,10 @@ import {
} from './agents.js';
import { initEffects, updateEffects, disposeEffects, updateJobIndicators } from './effects.js';
import { initUI, updateUI } from './ui.js';
import { initInteraction, disposeInteraction, registerSlapTarget, registerClickTarget } from './interaction.js';
import { initInteraction, disposeInteraction, registerSlapTarget } from './interaction.js';
import { initWebSocket, getConnectionState, getJobCount } from './websocket.js';
import { initPaymentPanel } from './payment.js';
import { initSessionPanel, openSessionPanel } from './session.js';
import { initSessionPanel } from './session.js';
import { initNostrIdentity } from './nostr-identity.js';
import { warmup as warmupEdgeWorker, onReady as onEdgeWorkerReady } from './edge-worker-client.js';
import { setEdgeWorkerReady } from './ui.js';
@@ -18,7 +18,6 @@ import { initTimmyId } from './timmy-id.js';
import { AGENT_DEFS } from './agent-defs.js';
import { initNavigation, updateNavigation, disposeNavigation } from './navigation.js';
import { initHudLabels, updateHudLabels, disposeHudLabels } from './hud-labels.js';
import { initPowerMeter, updatePowerMeter, disposePowerMeter, getMeterGroup } from './power-meter.js';
let running = false;
let canvas = null;
@@ -30,7 +29,6 @@ function buildWorld(firstInit, stateSnapshot) {
initEffects(scene);
initAgents(scene);
initPowerMeter(scene);
if (stateSnapshot) applyAgentStates(stateSnapshot);
@@ -39,7 +37,6 @@ function buildWorld(firstInit, stateSnapshot) {
initInteraction(camera, renderer);
registerSlapTarget(getTimmyGroup(), applySlap);
registerClickTarget(getMeterGroup(), openSessionPanel);
// AR floating labels
initHudLabels(camera, AGENT_DEFS, TIMMY_WORLD_POS);
@@ -85,7 +82,6 @@ function buildWorld(firstInit, stateSnapshot) {
updateEffects(now);
updateAgents(now);
updateJobIndicators(now);
updatePowerMeter(now, camera, renderer);
updateUI({
fps: currentFps,
agentCount: getAgentCount(),
@@ -125,7 +121,6 @@ function teardown({ scene, renderer, ac }) {
disposeNavigation();
disposeInteraction();
disposeHudLabels();
disposePowerMeter();
disposeEffects();
disposeAgents();
disposeWorld(renderer, scene);

View File

@@ -1,291 +0,0 @@
/**
* power-meter.js — 3D Session Power Meter
*
* A glowing orb that fills proportionally to the session balance.
* - Pulses brightly on payment received
* - Drains visibly on job cost deduction
* - Visible only when a funded session is active
* - Clicking the meter opens the session panel
*/
import * as THREE from 'three';
// ── World position (bottom-right corner, visible from default camera) ──────────
const METER_WORLD_POS = new THREE.Vector3(4.5, 0.85, 2.0);
const DEFAULT_MAX_SATS = 5000;
// ── Module state ───────────────────────────────────────────────────────────────
let _scene = null;
let _meterGroup = null;
let _innerOrb = null;
let _outerShell = null;
let _boltMesh = null;
let _orbMat = null;
let _shellMat = null;
let _meterLight = null;
let _labelEl = null;
let _fillFraction = 0; // 0..1, currently displayed fill
let _targetFill = 0; // 0..1, desired fill
let _pulseIntensity = 0; // 0..1, extra brightness from pulse
let _pulseDecay = 0; // per-frame decay rate
let _visible = false;
let _maxSats = DEFAULT_MAX_SATS;
let _currentSats = 0;
// ── Public API ─────────────────────────────────────────────────────────────────
export function initPowerMeter(scene) {
_scene = scene;
_labelEl = _createLabelEl();
_meterGroup = new THREE.Group();
_meterGroup.position.copy(METER_WORLD_POS);
_meterGroup.visible = false;
// Outer transparent shell
_shellMat = new THREE.MeshPhysicalMaterial({
color: 0x00aacc,
emissive: 0x003344,
emissiveIntensity: 0.4,
transparent: true,
opacity: 0.18,
roughness: 0.05,
metalness: 0.15,
side: THREE.DoubleSide,
});
_outerShell = new THREE.Mesh(new THREE.SphereGeometry(0.48, 24, 18), _shellMat);
_meterGroup.add(_outerShell);
// Equatorial ring accent
const ringMat = new THREE.MeshBasicMaterial({
color: 0x00ccff,
transparent: true,
opacity: 0.55,
});
const ring = new THREE.Mesh(new THREE.TorusGeometry(0.48, 0.012, 8, 40), ringMat);
ring.rotation.x = Math.PI / 2;
_meterGroup.add(ring);
// Inner fill orb — scales with balance
_orbMat = new THREE.MeshStandardMaterial({
color: 0x00ffcc,
emissive: 0x00ffcc,
emissiveIntensity: 1.4,
roughness: 0.05,
transparent: true,
opacity: 0.90,
});
_innerOrb = new THREE.Mesh(new THREE.SphereGeometry(0.38, 20, 16), _orbMat);
_innerOrb.scale.setScalar(0.01);
_meterGroup.add(_innerOrb);
// Lightning bolt — flat shape rendered in front of sphere
_boltMesh = _makeBolt();
_boltMesh.position.set(0, 0, 0.50);
_meterGroup.add(_boltMesh);
// Point light — intensity tracks balance + pulse
_meterLight = new THREE.PointLight(0x00ccff, 0.0, 6);
_meterGroup.add(_meterLight);
scene.add(_meterGroup);
return _meterGroup;
}
/**
* Update balance and target fill level.
* @param {number} sats Current balance in satoshis
* @param {number} [maxSats] Session deposit cap (sets the "full" reference)
*/
export function setMeterBalance(sats, maxSats) {
_currentSats = Math.max(0, sats);
if (maxSats !== undefined && maxSats > 0) _maxSats = maxSats;
_targetFill = Math.min(1, _currentSats / _maxSats);
if (_labelEl) _labelEl.textContent = `${_currentSats}`;
}
/**
* Fire a pulse animation on the meter.
* @param {'fill'|'drain'} type
*/
export function triggerMeterPulse(type) {
if (type === 'fill') {
_pulseIntensity = 1.0;
_pulseDecay = 0.022; // slow bright flash
} else {
_pulseIntensity = 0.45;
_pulseDecay = 0.040; // quick drain flicker
}
}
/** Show or hide the power meter and its text label. */
export function setMeterVisible(visible) {
_visible = visible;
if (_meterGroup) _meterGroup.visible = visible;
if (_labelEl) _labelEl.style.display = visible ? 'block' : 'none';
if (!visible) {
_pulseIntensity = 0;
_fillFraction = 0;
_targetFill = 0;
if (_innerOrb) _innerOrb.scale.setScalar(0.01);
if (_meterLight) _meterLight.intensity = 0;
}
}
/** Returns the Three.js Group for raycasting. */
export function getMeterGroup() {
return _meterGroup;
}
/**
* Called every animation frame.
* @param {number} now performance.now() value
* @param {THREE.Camera} camera
* @param {THREE.WebGLRenderer} renderer
*/
export function updatePowerMeter(now, camera, renderer) {
if (!_meterGroup || !_visible) return;
const t = now * 0.001;
// Smooth fill toward target
_fillFraction += (_targetFill - _fillFraction) * 0.06;
const s = Math.max(0.01, _fillFraction * 0.94);
_innerOrb.scale.setScalar(s);
// Colour interpolation: empty=red → mid=orange/yellow → full=cyan
_updateOrbColor(_fillFraction, _pulseIntensity);
// Pulse decay
if (_pulseIntensity > 0) {
_pulseIntensity = Math.max(0, _pulseIntensity - _pulseDecay);
}
// Point light
_meterLight.intensity = _fillFraction * 1.6 + _pulseIntensity * 4.5;
// Shell emissive pulse
_shellMat.emissiveIntensity = 0.3 + _pulseIntensity * 1.8 + _fillFraction * 0.35;
// Gentle bob + slow rotation
_meterGroup.position.y = METER_WORLD_POS.y + Math.sin(t * 1.4) * 0.06;
_meterGroup.rotation.y = t * 0.35;
// Bolt scale pulses with balance and on payment
const boltS = 1.0 + _pulseIntensity * 0.55 + Math.sin(t * 3.2) * 0.05;
_boltMesh.scale.setScalar(boltS);
// DOM label
if (_labelEl && camera && renderer) {
_updateLabelPosition(camera, renderer);
}
}
export function disposePowerMeter() {
if (_meterGroup && _scene) {
_scene.remove(_meterGroup);
_meterGroup.traverse(child => {
if (child.geometry) child.geometry.dispose();
if (child.material) {
const mats = Array.isArray(child.material) ? child.material : [child.material];
mats.forEach(m => m.dispose());
}
});
_meterGroup = null;
}
if (_labelEl) {
_labelEl.remove();
_labelEl = null;
}
_scene = null;
}
// ── Internals ──────────────────────────────────────────────────────────────────
function _makeBolt() {
const shape = new THREE.Shape();
shape.moveTo( 0.075, 0.28);
shape.lineTo(-0.030, 0.04);
shape.lineTo( 0.040, 0.04);
shape.lineTo(-0.075, -0.28);
shape.lineTo( 0.030, -0.04);
shape.lineTo(-0.040, -0.04);
shape.closePath();
const mat = new THREE.MeshBasicMaterial({
color: 0xffee00,
side: THREE.DoubleSide,
transparent: true,
opacity: 0.95,
});
return new THREE.Mesh(new THREE.ShapeGeometry(shape), mat);
}
function _updateOrbColor(fill, pulse) {
let r, g, b;
if (fill < 0.5) {
const k = fill * 2; // 0→1
r = 1.0 - k * 0.30; // 1.00 → 0.70
g = k * 0.70; // 0.00 → 0.70
b = k * 0.20; // 0.00 → 0.20
} else {
const k = (fill - 0.5) * 2; // 0→1
r = 0.70 - k * 0.70; // 0.70 → 0.00
g = 0.70 + k * 0.30; // 0.70 → 1.00
b = 0.20 + k * 0.80; // 0.20 → 1.00
}
// Pulse brightens toward white
const pr = r + pulse * (1 - r);
const pg = g + pulse * (1 - g);
const pb = b + pulse * (1 - b);
_orbMat.color.setRGB(pr, pg, pb);
_orbMat.emissive.setRGB(pr * 0.8, pg * 0.8, pb * 0.8);
_orbMat.emissiveIntensity = 1.0 + fill * 0.8 + pulse * 2.5;
_meterLight.color.setRGB(pr, pg, pb);
}
function _createLabelEl() {
const el = document.createElement('div');
el.style.cssText = [
'position:fixed',
'pointer-events:none',
'color:#00ffee',
'font-size:11px',
'font-family:monospace',
'font-weight:bold',
'text-align:center',
'text-shadow:0 0 6px #00ffcc,0 0 14px #00aaff',
'display:none',
'z-index:50',
'transform:translateX(-50%)',
'white-space:nowrap',
].join(';');
document.body.appendChild(el);
return el;
}
function _updateLabelPosition(camera, renderer) {
const canvas = renderer.domElement;
const rect = canvas.getBoundingClientRect();
// Project a point slightly above the orb center
const worldPos = METER_WORLD_POS.clone();
worldPos.y += 0.72;
worldPos.project(camera);
if (worldPos.z > 1) { // behind camera
_labelEl.style.display = 'none';
return;
}
_labelEl.style.display = 'block';
const x = rect.left + ( worldPos.x * 0.5 + 0.5) * rect.width;
const y = rect.top + (-worldPos.y * 0.5 + 0.5) * rect.height;
_labelEl.style.left = `${x}px`;
_labelEl.style.top = `${y}px`;
}

View File

@@ -16,7 +16,6 @@ import { setSpeechBubble, setMood } from './agents.js';
import { appendSystemMessage, setSessionSendHandler, setInputBarSessionMode } from './ui.js';
import { getOrRefreshToken } from './nostr-identity.js';
import { sentiment } from './edge-worker-client.js';
import { setMeterBalance, triggerMeterPulse, setMeterVisible } from './power-meter.js';
const API = '/api';
const LS_KEY = 'timmy_session_v1';
@@ -34,7 +33,6 @@ let _pollTimer = null;
let _inFlight = false;
let _selectedSats = 500; // deposit amount selection
let _topupSats = 500; // topup amount selection
let _sessionMax = 500; // max sats for this session (initial deposit)
// ── Public API ────────────────────────────────────────────────────────────────
@@ -107,11 +105,6 @@ export function isSessionActive() {
return _sessionState === 'active' || _sessionState === 'paused';
}
/** Public entry-point used by the 3D power meter click handler. */
export function openSessionPanel() {
_openPanel();
}
// Called by ui.js when user submits the input bar while session is active
export async function sessionSendHandler(text) {
if (!_sessionId || !_macaroon || _inFlight) return;
@@ -163,17 +156,21 @@ export async function sessionSendHandler(text) {
_sessionState = _balanceSats < MIN_BALANCE ? 'paused' : 'active';
_saveToStorage();
_applySessionUI();
triggerMeterPulse('drain');
const reply = data.result || data.reason || '…';
setSpeechBubble(reply);
appendSystemMessage('Timmy: ' + reply.slice(0, 80));
if (data.mediaType === 'image' && data.mediaUrl) {
// Fetch image data and render inline
_renderImageResponse(data.mediaUrl, text);
} else {
const reply = data.result || data.reason || '…';
setSpeechBubble(reply);
appendSystemMessage('Timmy: ' + reply.slice(0, 80));
// Sentiment-driven mood on inbound Timmy reply
sentiment(reply).then(s => {
setMood(s.label);
setTimeout(() => setMood(null), 10_000);
}).catch(() => {});
// Sentiment-driven mood on inbound Timmy reply
sentiment(reply).then(s => {
setMood(s.label);
setTimeout(() => setMood(null), 10_000);
}).catch(() => {});
}
// Update active-step balance if panel is open
_updateActiveStep();
@@ -186,6 +183,66 @@ export async function sessionSendHandler(text) {
}
}
// ── Image rendering ───────────────────────────────────────────────────────────
async function _renderImageResponse(mediaUrl, prompt) {
const $log = document.getElementById('event-log');
if (!$log) return;
setSpeechBubble('✨ Here is your image!');
appendSystemMessage('Timmy: ✨ Image generated!');
try {
const res = await fetch(mediaUrl);
if (!res.ok) {
appendSystemMessage('Timmy: Image ready — ' + mediaUrl);
return;
}
const data = await res.json();
const src = `data:${data.mimeType};base64,${data.data}`;
const container = document.createElement('div');
container.className = 'log-entry timmy-image-result';
container.style.cssText = [
'margin:6px 0;padding:6px;',
'border:1px solid #336655;border-radius:4px;',
'background:#0a1a14;',
].join('');
const img = document.createElement('img');
img.src = src;
img.alt = prompt.slice(0, 60);
img.style.cssText = [
'max-width:100%;max-height:240px;',
'display:block;border-radius:3px;',
'cursor:pointer;',
].join('');
img.title = 'Click to view full size';
const dlBtn = document.createElement('a');
dlBtn.href = src;
dlBtn.download = 'timmy-image.png';
dlBtn.textContent = '⬇ Download';
dlBtn.style.cssText = [
'display:inline-block;margin-top:4px;',
'font-size:10px;color:#44cc88;',
'text-decoration:none;letter-spacing:1px;',
].join('');
container.appendChild(img);
container.appendChild(dlBtn);
const entries = $log.querySelectorAll('.log-entry');
if (entries.length >= 6) {
$log.removeChild(entries[0]);
}
$log.appendChild(container);
$log.scrollTop = $log.scrollHeight;
} catch {
appendSystemMessage('Timmy: Image generated — ' + mediaUrl);
}
}
// ── Panel open/close ──────────────────────────────────────────────────────────
function _openPanel() {
@@ -295,11 +352,9 @@ function _startDepositPolling() {
if (data.state === 'active') {
_macaroon = data.macaroon;
_balanceSats = data.balanceSats;
_sessionMax = _balanceSats; // initial deposit = full bar
_sessionState = 'active';
_saveToStorage();
_applySessionUI();
triggerMeterPulse('fill');
_closePanel(); // panel auto-closes; user types in input bar
appendSystemMessage(`Session active — ${_balanceSats} sats`);
return;
@@ -414,10 +469,8 @@ function _startTopupPolling() {
_balanceSats = data.balanceSats;
_macaroon = data.macaroon || _macaroon;
_sessionState = data.state === 'active' ? 'active' : _sessionState;
_sessionMax = Math.max(_sessionMax, _balanceSats);
_saveToStorage();
_applySessionUI();
triggerMeterPulse('fill');
_setStep('active');
_updateActiveStep();
_setStatus('active', `⚡ Topped up! ${_balanceSats} sats`, '#22aa66');
@@ -524,10 +577,6 @@ function _applySessionUI() {
// Low balance notice above input bar
const $notice = document.getElementById('low-balance-notice');
if ($notice) $notice.style.display = lowBal ? '' : 'none';
// 3D power meter visibility
setMeterVisible(anyActive);
if (anyActive) setMeterBalance(_balanceSats, _sessionMax);
}
function _updateActiveStep() {
@@ -546,9 +595,7 @@ function _clearSession() {
_macaroon = null;
_balanceSats = 0;
_sessionState = null;
_sessionMax = 500;
localStorage.removeItem(LS_KEY);
setMeterVisible(false);
setInputBarSessionMode(false);
setSessionSendHandler(null);
const $hud = document.getElementById('session-hud');

View File

@@ -6,7 +6,6 @@ import { sentiment } from './edge-worker-client.js';
import { setLabelState } from './hud-labels.js';
import { createJobIndicator, dissolveJobIndicator } from './effects.js';
import { getPubkey } from './nostr-identity.js';
import { setMeterBalance, triggerMeterPulse } from './power-meter.js';
function resolveWsUrl() {
const explicit = import.meta.env.VITE_WS_URL;
@@ -191,15 +190,6 @@ function handleMessage(msg) {
break;
}
case 'session_balance_update': {
// Real-time balance push from server: animate fill level and pulse
if (typeof msg.balanceSats === 'number') {
setMeterBalance(msg.balanceSats, msg.maxSats);
triggerMeterPulse(msg.delta > 0 ? 'fill' : 'drain');
}
break;
}
case 'agent_count':
case 'visitor_count':
break;